diff --git a/.github/workflows/manual_release_stable.yaml b/.github/workflows/manual_release_stable.yaml index 5d8c1f9c..c7d86eb4 100644 --- a/.github/workflows/manual_release_stable.yaml +++ b/.github/workflows/manual_release_stable.yaml @@ -102,17 +102,83 @@ jobs: - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + version_docs: + name: Version docs + needs: [release_prepare, changelog_update, pypi_publish] + runs-on: ubuntu-latest + outputs: + version_docs_commitish: ${{ steps.commit_versioned_docs.outputs.commit_long_sha }} + permissions: + contents: write + env: + NODE_VERSION: 22 + PYTHON_VERSION: 3.14 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + ref: ${{ needs.changelog_update.outputs.changelog_commitish }} + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Python dependencies + run: uv run poe install-dev + + - name: Install website dependencies + run: | + cd website + corepack enable + yarn install + + - name: Snapshot the current version + run: | + cd website + VERSION="$(python -c "import tomllib, pathlib; print(tomllib.loads(pathlib.Path('../pyproject.toml').read_text())['project']['version'])")" + MAJOR_MINOR="$(echo "$VERSION" | cut -d. -f1-2)" + export MAJOR_MINOR + # Remove existing version if present (patch releases override) + rm -rf "versioned_docs/version-${MAJOR_MINOR}" + rm -rf "versioned_sidebars/version-${MAJOR_MINOR}-sidebars.json" + jq 'map(select(. != env.MAJOR_MINOR))' versions.json > tmp.json && mv tmp.json versions.json + # Build API reference and create version snapshots + bash build_api_reference.sh + npx docusaurus docs:version "$MAJOR_MINOR" + npx docusaurus api:version "$MAJOR_MINOR" + + - name: Commit and push versioned docs + id: commit_versioned_docs + uses: EndBug/add-and-commit@v9 + with: + add: "website/versioned_docs website/versioned_sidebars website/versions.json" + message: "docs: version ${{ needs.release_prepare.outputs.version_number }} docs [skip ci]" + default_author: github_actions + doc_release: name: Doc release - needs: [changelog_update, pypi_publish] + needs: [changelog_update, pypi_publish, version_docs] permissions: contents: write pages: write id-token: write uses: ./.github/workflows/_release_docs.yaml with: - # Use the ref from the changelog update to include the updated changelog. - ref: ${{ needs.changelog_update.outputs.changelog_commitish }} + # Use the version_docs commit to include both changelog and versioned docs. + ref: ${{ needs.version_docs.outputs.version_docs_commitish }} secrets: inherit trigger_docker_build: diff --git a/pyproject.toml b/pyproject.toml index 7e61dd41..c1c30017 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,9 @@ allow-direct-references = true [tool.ruff] line-length = 120 include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"] +exclude = [ + "website/versioned_docs/**", +] [tool.ruff.lint] select = ["ALL"] @@ -201,6 +204,7 @@ python-version = "3.10" [tool.ty.src] include = ["src", "tests", "scripts", "docs", "website"] +exclude = ["website/versioned_docs"] [[tool.ty.overrides]] include = [ diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 92406f80..dfb3ef6a 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -3,6 +3,7 @@ const { join, resolve } = require('node:path'); const { config } = require('@apify/docs-theme'); const { externalLinkProcessor } = require('./tools/utils/externalLink'); +const versions = require('./versions.json'); const GROUP_ORDER = [ 'Actor', @@ -18,9 +19,14 @@ const GROUP_ORDER = [ ]; const groupSort = (g1, g2) => { - if (GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)) { - return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2); - } + const i1 = GROUP_ORDER.indexOf(g1); + const i2 = GROUP_ORDER.indexOf(g2); + // Both known – sort by defined order + if (i1 !== -1 && i2 !== -1) return i1 - i2; + // Unknown groups go after known ones + if (i1 !== -1) return -1; + if (i2 !== -1) return 1; + // Both unknown – alphabetical return g1.localeCompare(g2); }; @@ -66,19 +72,21 @@ module.exports = { title: 'SDK for Python', items: [ { - to: 'docs/overview', + type: 'doc', + docId: 'introduction/introduction', label: 'Docs', position: 'left', activeBaseRegex: '/docs(?!/changelog)', }, { - to: '/reference', + type: 'custom-versioned-reference', label: 'Reference', position: 'left', activeBaseRegex: '/reference', }, { - to: 'docs/changelog', + type: 'doc', + docId: 'changelog', label: 'Changelog', position: 'left', activeBaseRegex: '/docs/changelog', @@ -88,6 +96,17 @@ module.exports = { label: 'GitHub', position: 'left', }, + { + type: 'docsVersionDropdown', + position: 'left', + className: 'navbar__item', + 'data-api-links': JSON.stringify([ + 'reference/next', + ...versions.map((version, i) => (i === 0 ? 'reference' : `reference/${version}`)), + ]), + dropdownItemsBefore: [], + dropdownItemsAfter: [], + }, ], }, }, @@ -280,6 +299,12 @@ module.exports = { includeGeneratedIndex: false, includePages: true, relativePaths: false, + excludeRoutes: [ + '/sdk/python/reference/[0-9]*/**', + '/sdk/python/reference/[0-9]*', + '/sdk/python/reference/next/**', + '/sdk/python/reference/next', + ], }, }, ], @@ -287,6 +312,7 @@ module.exports = { ], themeConfig: { ...config.themeConfig, + versions, tableOfContents: { ...config.themeConfig.tableOfContents, maxHeadingLevel: 5, diff --git a/website/src/theme/NavbarItem/ComponentTypes.js b/website/src/theme/NavbarItem/ComponentTypes.js new file mode 100644 index 00000000..b6b4f89d --- /dev/null +++ b/website/src/theme/NavbarItem/ComponentTypes.js @@ -0,0 +1,7 @@ +import OriginalComponentTypes from '@theme-original/NavbarItem/ComponentTypes'; +import VersionedReferenceNavbarItem from './VersionedReferenceNavbarItem'; + +export default { + ...OriginalComponentTypes, + 'custom-versioned-reference': VersionedReferenceNavbarItem, +}; diff --git a/website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js b/website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js new file mode 100644 index 00000000..d42eb46a --- /dev/null +++ b/website/src/theme/NavbarItem/DocsVersionDropdownNavbarItem.js @@ -0,0 +1,91 @@ +import React, { useMemo } from 'react'; +import { useVersions, useActiveDocContext, useDocsVersionCandidates } from '@docusaurus/plugin-content-docs/client'; +import { useDocsPreferredVersion } from '@docusaurus/theme-common'; +import { translate } from '@docusaurus/Translate'; +import { useLocation } from '@docusaurus/router'; +import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; +import DefaultNavbarItem from '@theme/NavbarItem/DefaultNavbarItem'; +import DropdownNavbarItem from '@theme/NavbarItem/DropdownNavbarItem'; + +const getVersionMainDoc = (version) => version.docs.find((doc) => doc.id === version.mainDocId); + +/* eslint-disable react/prop-types */ +export default function DocsVersionDropdownNavbarItem({ + mobile, + docsPluginId, + dropdownActiveClassDisabled, + dropdownItemsBefore, + dropdownItemsAfter, + ...props +}) { + const { search, hash, pathname } = useLocation(); + const { siteConfig } = useDocusaurusContext(); + const baseUrl = siteConfig.baseUrl.endsWith('/') ? siteConfig.baseUrl : `${siteConfig.baseUrl}/`; + const apiLinks = useMemo(() => { + if (!pathname.startsWith(`${baseUrl}reference`)) { + return []; + } + try { + return JSON.parse(props['data-api-links']); + } catch { + return []; + } + }, [props['data-api-links'], pathname, baseUrl]); + + const activeDocContext = useActiveDocContext(docsPluginId); + const versions = useVersions(docsPluginId); + const { savePreferredVersionName } = useDocsPreferredVersion(docsPluginId); + const versionLinks = versions.map((version, idx) => { + // We try to link to the same doc, in another version + // When not possible, fallback to the "main doc" of the version + const versionDoc = activeDocContext.alternateDocVersions[version.name] ?? getVersionMainDoc(version); + return { + label: version.label, + // preserve ?search#hash suffix on version switches + to: `${apiLinks[idx] ?? versionDoc.path}${search}${hash}`, + isActive: () => version === activeDocContext.activeVersion, + onClick: () => savePreferredVersionName(version.name), + }; + }); + const items = [...dropdownItemsBefore, ...versionLinks, ...dropdownItemsAfter]; + const dropdownVersion = useDocsVersionCandidates(docsPluginId)[0]; + // Mobile dropdown is handled a bit differently + const dropdownLabel = + mobile && items.length > 1 + ? translate({ + id: 'theme.navbar.mobileVersionsDropdown.label', + message: 'Versions', + description: 'The label for the navbar versions dropdown on mobile view', + }) + : dropdownVersion.label; + let dropdownTo = mobile && items.length > 1 ? undefined : getVersionMainDoc(dropdownVersion).path; + + if (dropdownTo && pathname.startsWith(`${baseUrl}reference`)) { + dropdownTo = versionLinks.find((v) => v.label === dropdownVersion.label)?.to; + } + + // We don't want to render a version dropdown with 0 or 1 item. If we build + // the site with a single docs version (onlyIncludeVersions: ['1.0.0']), + // We'd rather render a button instead of a dropdown + if (items.length <= 1) { + return ( + false : undefined} + /> + ); + } + return ( + false : undefined} + /> + ); +} diff --git a/website/src/theme/NavbarItem/VersionedReferenceNavbarItem.js b/website/src/theme/NavbarItem/VersionedReferenceNavbarItem.js new file mode 100644 index 00000000..1614a4a8 --- /dev/null +++ b/website/src/theme/NavbarItem/VersionedReferenceNavbarItem.js @@ -0,0 +1,16 @@ +import React from 'react'; +import { useDocsVersionCandidates } from '@docusaurus/plugin-content-docs/client'; +import DefaultNavbarItem from '@theme/NavbarItem/DefaultNavbarItem'; + +/* eslint-disable react/prop-types */ +export default function VersionedReferenceNavbarItem({ docsPluginId, ...props }) { + const [version] = useDocsVersionCandidates(docsPluginId); + + // Latest version → /reference, "current" (next) → /reference/next, others → /reference/{name} + let to = '/reference'; + if (!version.isLast) { + to = `/reference/${version.name === 'current' ? 'next' : version.name}`; + } + + return ; +} diff --git a/website/versioned_docs/version-1.7/.gitignore b/website/versioned_docs/version-1.7/.gitignore new file mode 100644 index 00000000..1a13c363 --- /dev/null +++ b/website/versioned_docs/version-1.7/.gitignore @@ -0,0 +1 @@ +changelog.md diff --git a/website/versioned_docs/version-1.7/01-introduction/index.mdx b/website/versioned_docs/version-1.7/01-introduction/index.mdx new file mode 100644 index 00000000..d4cc4cd3 --- /dev/null +++ b/website/versioned_docs/version-1.7/01-introduction/index.mdx @@ -0,0 +1,50 @@ +--- +id: introduction +title: Overview +sidebar_label: Overview +slug: /overview +description: 'The official library for creating Apify Actors in Python, providing tools for web scraping, automation, and data storage integration.' +--- + +The Apify SDK for Python is the official library for creating [Apify Actors](https://docs.apify.com/platform/actors) in Python. + +```python +from apify import Actor +from bs4 import BeautifulSoup +import requests + +async def main(): + async with Actor: + actor_input = await Actor.get_input() + response = requests.get(actor_input['url']) + soup = BeautifulSoup(response.content, 'html.parser') + await Actor.push_data({ 'url': actor_input['url'], 'title': soup.title.string }) +``` + +## What are Actors? + +Actors are serverless cloud programs that can do almost anything a human can do in a web browser. They can do anything from small tasks such as filling in forms or unsubscribing from online services, all the way up to scraping and processing vast numbers of web pages. + +Actors can be run either locally, or on the [Apify platform](https://docs.apify.com/platform/), where you can run them at scale, monitor them, schedule them, and even publish and monetize them. + +If you're new to Apify, learn [what is Apify](https://docs.apify.com/platform/about) in the Apify platform documentation. + +## Quick start + +To create and run Actors through Apify Console, see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template). For creating and running Python Actors locally, refer to the [quick start guide](./quick-start). + +Explore the Guides section in the sidebar for a deeper understanding of the SDK's features and best practices. + +## Installation + +The Apify SDK for Python requires Python version 3.8 or above. It is typically installed when you create a new Actor project using the [Apify CLI](https://docs.apify.com/cli). To install it manually in an existing project, use: + +```bash +pip install apify +``` + +:::note API client alternative + +If you need to interact with the Apify API programmatically without creating Actors, use the [Apify API client for Python](https://docs.apify.com/api/client/python) instead. + +::: diff --git a/website/versioned_docs/version-1.7/01-introduction/quick-start.mdx b/website/versioned_docs/version-1.7/01-introduction/quick-start.mdx new file mode 100644 index 00000000..1d039fc9 --- /dev/null +++ b/website/versioned_docs/version-1.7/01-introduction/quick-start.mdx @@ -0,0 +1,140 @@ +--- +id: quick-start +title: Quick start +sidebar_label: Quick start +description: 'Get started with the Apify SDK for Python by creating your first Actor and learning the basics.' +--- + +Learn how to create and run Actors using the Apify SDK for Python. + +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +## Step 1: Create Actors + +To create and run Actors through Apify Console, see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template). + +To create a new Apify Actor on your computer, you can use the [Apify CLI](https://docs.apify.com/cli), and select one of the [Python Actor templates](https://apify.com/templates?category=python). + +For example, to create an Actor from the "[beta] Python SDK" template, you can use the [`apify create` command](https://docs.apify.com/cli/docs/reference#apify-create-actorname). + +```bash +apify create my-first-actor --template python-start +``` + +This will create a new folder called `my-first-actor`, download and extract the "Getting started with Python" Actor template there, create a virtual environment in `my-first-actor/.venv`, and install the Actor dependencies in it. + +## Step 2: Run Actors + +To run the Actor, you can use the [`apify run` command](https://docs.apify.com/cli/docs/reference#apify-run): + +```bash +cd my-first-actor +apify run +``` + +This command: + +- Activates the virtual environment in `.venv` (if no other virtual environment is activated yet) +- Starts the Actor with the appropriate environment variables for local running +- Configures it to use local storages from the `storage` folder + +The Actor input, for example, will be in `storage/key_value_stores/default/INPUT.json`. + +## Step 3: Understand Actor structure + +All Python Actor templates follow the same structure. + +The `.actor` directory contains the [Actor configuration](https://docs.apify.com/platform/actors/development/actor-config), such as the Actor's definition and input schema, and the Dockerfile necessary to run the Actor on the Apify platform. + +The Actor's runtime dependencies are specified in the `requirements.txt` file, which follows the [standard requirements file format](https://pip.pypa.io/en/stable/reference/requirements-file-format/). + +The Actor's source code is in the `src` folder. This folder contains two important files: + +- `main.py` - which contains the main function of the Actor +- `__main__.py` - which is the entrypoint of the Actor package, setting up the Actor [logger](../concepts/logging) and executing the Actor's main function via [`asyncio.run()`](https://docs.python.org/3/library/asyncio-runner.html#asyncio.run). + + + + { +`from apify import Actor +${''} +async def main(): + async with Actor: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!')` + } + + + { +`import asyncio +import logging +${''} +from apify.log import ActorLogFormatter +${''} +from .main import main +${''} +handler = logging.StreamHandler() +handler.setFormatter(ActorLogFormatter()) +${''} +apify_logger = logging.getLogger('apify') +apify_logger.setLevel(logging.DEBUG) +apify_logger.addHandler(handler) +${''} +asyncio.run(main())` + } + + + +If you want to modify the Actor structure, you need to make sure that your Actor is executable as a module, via `python -m src`, as that is the command started by `apify run` in the Apify CLI. We recommend keeping the entrypoint for the Actor in the `src/__main__.py` file. + +## Step 4: Add dependencies {#adding-dependencies} + +Adding dependencies into the Actor is simple. + +First, add them in the [`requirements.txt`](https://pip.pypa.io/en/stable/reference/requirements-file-format/) file in the Actor source folder. + +Then activate the virtual environment in `.venv`: + + + + { +`source .venv/bin/activate` + } + + + { +`.venv\\Scripts\\activate` + } + + + +Then install the dependencies: + +```bash +python -m pip install -r requirements.txt +``` + +## Next steps + +### Guides + +To see how you can integrate the Apify SDK with some of the most popular web scraping libraries, check out our guides for working with: + +- [Requests or HTTPX](../guides/requests-and-httpx) +- [Beautiful Soup](../guides/beautiful-soup) +- [Playwright](../guides/playwright) +- [Selenium](../guides/selenium) +- [Scrapy](../guides/scrapy) + +### Concepts + +To learn more about the features of the Apify SDK and how to use them, check out the Concepts section in the sidebar, especially the guides for: + +- [Actor lifecycle](../concepts/actor-lifecycle) +- [Working with storages](../concepts/storages) +- [Handling Actor events](../concepts/actor-events) +- [How to use proxies](../concepts/proxy-management) diff --git a/website/versioned_docs/version-1.7/02-guides/01-requests-and-httpx.mdx b/website/versioned_docs/version-1.7/02-guides/01-requests-and-httpx.mdx new file mode 100644 index 00000000..21d72f4f --- /dev/null +++ b/website/versioned_docs/version-1.7/02-guides/01-requests-and-httpx.mdx @@ -0,0 +1,100 @@ +--- +title: Using Requests and HTTPX +sidebar_label: Using Requests and HTTPX +--- + +To use either of the libraries mentioned below in your Actors, +you can start from the [Start with Python](https://apify.com/templates?category=python) Actor template. + +## Requests + +The [`requests`](https://requests.readthedocs.io) library is one of the most popular Python libraries for making HTTP requests. + +To use it in your Actors, no special configuration is needed. +Just put `requests` in your `requirements.txt` file, +[reinstall dependencies](../introduction/quick-start#adding-dependencies) if you're running the Actor locally, +and you're good to go. + +```python title="src/main.py" +import requests +from apify import Actor + +async def main(): + async with Actor: + response = requests.get('http://example.com') + print(response.text) +``` + +### Using proxies with requests + +To use Apify Proxy with `requests`, +you can just generate a proxy URL through [`Actor.create_proxy_configuration()`](../../reference/class/Actor#create_proxy_configuration), +and pass it to `requests` using the [`proxies` argument](https://requests.readthedocs.io/en/latest/user/advanced/#proxies): + +```python title="src/main.py" +import requests +from apify import Actor + +async def main(): + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration() + proxy_url = await proxy_configuration.new_url() + proxies = { + 'http': proxy_url, + 'https': proxy_url, + } + + response = requests.get('http://example.com', proxies=proxies) + print(response.text) +``` + +To learn more about using proxies in your Actor with `requests`, check the [documentation for proxy management](../concepts/proxy-management). + +## HTTPX + +Another very popular Python library for performing HTTP requests is [`HTTPX`](https://www.python-httpx.org/). +Its main advantage over `requests` is the ability to [perform asynchronous HTTP requests](https://www.python-httpx.org/async/), +making it ideal for large-scale, parallel web scraping. + +To use it in your Actors, no special configuration is needed. +Just put `httpx` in your `requirements.txt` file, +[reinstall dependencies](../introduction/quick-start#adding-dependencies) if you're running the Actor locally, +and you're good to go. + +```python title="src/main.py" +import asyncio +import httpx +from apify import Actor + +async def main(): + async with Actor: + async with httpx.AsyncClient() as httpx_client: + # This will perform all the requests in parallel + http_requests = [] + for i in range(10): + http_requests.append(httpx_client.get(f'http://example.com/{i}')) + + responses = await asyncio.gather(*http_requests) + print(responses) +``` + +### Using proxies with HTTPX + +To use Apify Proxy with `httpx`, +you can just generate a proxy URL through [`Actor.create_proxy_configuration()`](../../reference/class/Actor#create_proxy_configuration), +and pass it to `httpx` using the [`proxies` argument](https://requests.readthedocs.io/en/latest/user/advanced/#proxies): + +```python title="src/main.py" +import httpx +from apify import Actor + +async def main(): + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration() + proxy_url = await proxy_configuration.new_url() + async with httpx.AsyncClient(proxies=proxy_url) as httpx_client: + response = httpx_client.get(f'http://example.com'), + print(response) +``` + +To learn more about using proxies in your Actor with `httpx`, check the [documentation for proxy management](../concepts/proxy-management). diff --git a/website/versioned_docs/version-1.7/02-guides/02-beautiful-soup.mdx b/website/versioned_docs/version-1.7/02-guides/02-beautiful-soup.mdx new file mode 100644 index 00000000..a625741f --- /dev/null +++ b/website/versioned_docs/version-1.7/02-guides/02-beautiful-soup.mdx @@ -0,0 +1,81 @@ +--- +title: Using Beautiful Soup +sidebar_label: Using Beautiful Soup +--- + +[Beautiful Soup](https://www.crummy.com/software/BeautifulSoup/) is a Python library for pulling data out of HTML and XML files. +It provides simple methods and Pythonic idioms for navigating, searching, and modifying a website's element tree, +allowing you to quickly extract the data you need. + +## Using BeautifulSoup in Actors + +To create Actors which use BeautifulSoup, start from the [BeautifulSoup & Python](https://apify.com/templates?category=python) Actor template. + +This Actor template already contains the BeautifulSoup library preinstalled, which means you can start using it right away. + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, +up to a maximum depth, starting from URLs in the Actor input. + +It uses `requests` to fetch the pages, +and BeautifulSoup to parse their content and read the page title and links to other pages. + +```python title="src/main.py" +from urllib.parse import urljoin + +import requests +from apify import Actor +from bs4 import BeautifulSoup + +async def main(): + async with Actor: + # Read the Actor input + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{ 'url': 'https://apify.com' }]) + max_depth = actor_input.get('max_depth', 1) + + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Enqueue the starting URLs in the default request queue + default_queue = await Actor.open_request_queue() + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url}...') + await default_queue.add_request({ 'url': url, 'userData': { 'depth': 0 }}) + + # Process the requests in the queue one by one + while request := await default_queue.fetch_next_request(): + url = request['url'] + depth = request['userData']['depth'] + Actor.log.info(f'Scraping {url}...') + + try: + # Fetch the URL using `requests` and parse it using `BeautifulSoup` + response = requests.get(url) + soup = BeautifulSoup(response.content, 'html.parser') + + # If we haven't reached the max depth, + # look for nested links and enqueue their targets + if depth < max_depth: + for link in soup.find_all('a'): + link_href = link.get('href') + link_url = urljoin(url, link_href) + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url}...') + await default_queue.add_request({ + 'url': link_url, + 'userData': {'depth': depth + 1 }, + }) + + # Push the title of the page into the default dataset + title = soup.title.string if soup.title else None + await Actor.push_data({ 'url': url, 'title': title }) + except: + Actor.log.exception(f'Cannot extract data from {url}.') + finally: + # Mark the request as handled so it's not processed again + await default_queue.mark_request_as_handled(request) +``` diff --git a/website/versioned_docs/version-1.7/02-guides/03-playwright.mdx b/website/versioned_docs/version-1.7/02-guides/03-playwright.mdx new file mode 100644 index 00000000..8094e621 --- /dev/null +++ b/website/versioned_docs/version-1.7/02-guides/03-playwright.mdx @@ -0,0 +1,120 @@ +--- +title: Using Playwright +sidebar_label: Using Playwright +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +[Playwright](https://playwright.dev) is a tool for web automation and testing that can also be used for web scraping. +It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Playwright for web scraping include: + +- **Cross-browser support** - Playwright supports the latest versions of major browsers like Chrome, Firefox, and Safari, +so you can choose the one that suits your needs the best. +- **Headless mode** - Playwright can run in headless mode, +meaning that the browser window is not visible on your screen while it is scraping, +which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Playwright provides a variety of powerful selectors that allow you to target specific elements on a web page, +including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Playwright allows you to emulate user interactions like clicking, scrolling, filling out forms, +and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Playwright in Actors + +To create Actors which use Playwright, start from the [Playwright & Python](https://apify.com/templates?category=python) Actor template. + +On the Apify platform, the Actor will already have Playwright and the necessary browsers preinstalled in its Docker image, +including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to finish the Playwright setup yourself before you can run the actor. + + + + { +`source .venv/bin/activate +playwright install --with-deps` + } + + + { +`.venv\\Scripts\\activate +playwright install --with-deps` + } + + + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, +up to a maximum depth, starting from URLs in the Actor input. + +It uses Playwright to open the pages in an automated Chrome browser, +and to extract the title and anchor elements after the pages load. + +```python title="src/main.py" +from urllib.parse import urljoin + +from apify import Actor +from playwright.async_api import async_playwright + + +async def main(): + async with Actor: + # Read the Actor input + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{ 'url': 'https://apify.com' }]) + max_depth = actor_input.get('max_depth', 1) + + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Enqueue the starting URLs in the default request queue + default_queue = await Actor.open_request_queue() + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + await default_queue.add_request({ 'url': url, 'userData': { 'depth': 0 }}) + + # Launch Playwright an open a new browser context + Actor.log.info('Launching Playwright...') + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=Actor.config.headless) + context = await browser.new_context() + + # Process the requests in the queue one by one + while request := await default_queue.fetch_next_request(): + url = request['url'] + depth = request['userData']['depth'] + Actor.log.info(f'Scraping {url} ...') + + try: + # Open the URL in a new Playwright page + page = await context.new_page() + await page.goto(url) + + # If we haven't reached the max depth, + # look for nested links and enqueue their targets + if depth < max_depth: + for link in await page.locator('a').all(): + link_href = await link.get_attribute('href') + link_url = urljoin(url, link_href) + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + await default_queue.add_request({ + 'url': link_url, + 'userData': {'depth': depth + 1 }, + }) + + # Push the title of the page into the default dataset + title = await page.title() + await Actor.push_data({ 'url': url, 'title': title }) + except: + Actor.log.exception(f'Cannot extract data from {url}.') + finally: + await page.close() + await default_queue.mark_request_as_handled(request) +``` diff --git a/website/versioned_docs/version-1.7/02-guides/04-selenium.mdx b/website/versioned_docs/version-1.7/02-guides/04-selenium.mdx new file mode 100644 index 00000000..3efa5149 --- /dev/null +++ b/website/versioned_docs/version-1.7/02-guides/04-selenium.mdx @@ -0,0 +1,110 @@ +--- +title: Using Selenium +sidebar_label: Using Selenium +--- + +[Selenium](https://www.selenium.dev/) is a tool for web automation and testing that can also be used for web scraping. +It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Selenium for web scraping include: + +- **Cross-browser support** - Selenium supports the latest versions of major browsers like Chrome, Firefox, and Safari, +so you can choose the one that suits your needs the best. +- **Headless mode** - Selenium can run in headless mode, +meaning that the browser window is not visible on your screen while it is scraping, +which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Selenium provides a variety of powerful selectors that allow you to target specific elements on a web page, +including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Selenium allows you to emulate user interactions like clicking, scrolling, filling out forms, +and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Selenium in Actors + +To create Actors which use Selenium, start from the [Selenium & Python](https://apify.com/templates?category=python) Actor template. + +On the Apify platform, the Actor will already have Selenium and the necessary browsers preinstalled in its Docker image, +including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to install the Selenium browser drivers yourself. +Refer to the [Selenium documentation](https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/) for installation instructions. + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, +up to a maximum depth, starting from URLs in the Actor input. + +It uses Selenium ChromeDriver to open the pages in an automated Chrome browser, +and to extract the title and anchor elements after the pages load. + +```python title="src/main.py" +from urllib.parse import urljoin + +from apify import Actor +from selenium import webdriver +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.common.by import By + + +async def main(): + async with Actor: + # Read the Actor input + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{ 'url': 'https://apify.com' }]) + max_depth = actor_input.get('max_depth', 1) + + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Enqueue the starting URLs in the default request queue + default_queue = await Actor.open_request_queue() + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + await default_queue.add_request({ 'url': url, 'userData': { 'depth': 0 }}) + + # Launch a new Selenium Chrome WebDriver + Actor.log.info('Launching Chrome WebDriver...') + chrome_options = ChromeOptions() + if Actor.config.headless: + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + driver = webdriver.Chrome(options=chrome_options) + + driver.get('http://www.example.com') + assert driver.title == 'Example Domain' + + # Process the requests in the queue one by one + while request := await default_queue.fetch_next_request(): + url = request['url'] + depth = request['userData']['depth'] + Actor.log.info(f'Scraping {url} ...') + + try: + # Open the URL in the Selenium WebDriver + driver.get(url) + + # If we haven't reached the max depth, + # look for nested links and enqueue their targets + if depth < max_depth: + for link in driver.find_elements(By.TAG_NAME, 'a'): + link_href = link.get_attribute('href') + link_url = urljoin(url, link_href) + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + await default_queue.add_request({ + 'url': link_url, + 'userData': {'depth': depth + 1 }, + }) + + # Push the title of the page into the default dataset + title = driver.title + await Actor.push_data({ 'url': url, 'title': title }) + except: + Actor.log.exception(f'Cannot extract data from {url}.') + finally: + await default_queue.mark_request_as_handled(request) + + driver.quit() +``` diff --git a/website/versioned_docs/version-1.7/02-guides/05-scrapy.mdx b/website/versioned_docs/version-1.7/02-guides/05-scrapy.mdx new file mode 100644 index 00000000..f73c4a3c --- /dev/null +++ b/website/versioned_docs/version-1.7/02-guides/05-scrapy.mdx @@ -0,0 +1,111 @@ +--- +title: Using Scrapy +sidebar_label: Using Scrapy +--- + +:::tip + +Our CLI now has native support for running Scrapy spiders on Apify! Check out the [Scrapy migration guide](https://docs.apify.com/cli/docs/integrating-scrapy) for more information. + +::: + +Scrapy is an open-source web scraping framework written in Python. +It provides a complete set of tools for web scraping, including the ability to define how to extract data from websites, handle pagination and navigation. + +Some of the key features of Scrapy for web scraping include: + +- **Request and response handling** - Scrapy provides an easy-to-use interface for making HTTP requests and handling responses, +allowing you to navigate through web pages and extract data. +- **Robust Spider framework** - Scrapy has a spider framework that allows you to define how to scrape data from websites, +including how to follow links, how to handle pagination, and how to parse the data. +- **Built-in data extraction** - Scrapy includes built-in support for data extraction using XPath and CSS selectors, +allowing you to easily extract data from HTML and XML documents. +- **Integration with other tool** - Scrapy can be integrated with other Python tools like BeautifulSoup and Selenium for more advanced scraping tasks. + +## Using Scrapy in Actors + +To create Actors which use Scrapy, start from the [Scrapy & Python](https://apify.com/templates?category=python) Actor template. + +This template already contains the structure and setup necessary to integrate Scrapy into your Actors, +setting up the Scrapy settings, asyncio reactor, Actor logger and item pipeline +as necessary to make Scrapy spiders run in Actors and save their outputs in Apify datasets. + +### Manual setup + +If you don't want to use the template, there are several things you need to set up. + +#### Event loop & reactor + +Since the `Actor` class uses `asyncio` under the hood, +Scrapy has to use the [`AsyncioSelectorReactor`](https://docs.scrapy.org/en/latest/topics/asyncio.html) reactor. +And to be able to run the Scrapy engine in an already running loop, +you have to use the [`nest_asyncio`](https://pypi.org/project/nest-asyncio/) package. + +#### Item pipeline + +To push the results into the Actor's default dataset, +the engine has to use a custom [`ItemPipeline`](https://docs.scrapy.org/en/latest/topics/item-pipeline.html) +that calls `Actor.push_data()` on the scraped items. + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, +up to a maximum depth, starting from URLs in the Actor input. + +It uses Scrapy download the pages, extract the results from each page, and continue recursively through the website pagination. + +```python title="src/main.py" +from urllib.parse import urljoin +import nest_asyncio +import scrapy +from itemadapter import ItemAdapter +from scrapy.crawler import CrawlerProcess +from scrapy.utils.project import get_project_settings +from scrapy.utils.reactor import install_reactor + +from apify import Actor + +# This is necessary so that twisted and asyncio work well together +install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +nest_asyncio.apply() + +# Scrapes titles pages and enqueues all links it finds on the page +class TitleSpider(scrapy.Spider): + name = 'title_spider' + + def __init__(self, start_urls, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_urls = start_urls + + def parse(self, response): + yield { + 'url': response.url, + 'title': response.css('title::text').extract_first(), + } + for link_href in response.css('a::attr("href")'): + link_url = urljoin(response.url, link_href.get()) + if link_url.startswith(('http://', 'https://')): + yield scrapy.Request(link_url) + +# Pushes the scraped items into the actor's default dataset +class ActorDatasetPushPipeline: + async def process_item(self, item, spider): + item_dict = ItemAdapter(item).asdict() + await Actor.push_data(item_dict) + return item + +async def main(): + async with Actor: + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{ 'url': 'https://apify.com' }]) + start_urls = [start_url.get('url') for start_url in start_urls] + + settings = get_project_settings() + settings['ITEM_PIPELINES'] = { ActorDatasetPushPipeline: 1 } + settings['TWISTED_REACTOR'] = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' + settings['DEPTH_LIMIT'] = actor_input.get('max_depth', 1) + + process = CrawlerProcess(settings) + process.crawl(TitleSpider, start_urls=start_urls) + process.start() +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/01-actor-lifecycle.mdx b/website/versioned_docs/version-1.7/03-concepts/01-actor-lifecycle.mdx new file mode 100644 index 00000000..cfbc6154 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/01-actor-lifecycle.mdx @@ -0,0 +1,117 @@ +--- +title: Actor lifecycle +sidebar_label: Actor lifecycle +--- + +## Lifecycle methods + +### Initialization and cleanup + +At the start of its runtime, the Actor needs to initialize itself, its event manager and its storages, +and at the end of the runtime it needs to close these cleanly. +The Apify SDK provides several options on how to manage this. + +#### `Actor.init()` and `Actor.exit()` + +The [`Actor.init()`](../../reference/class/Actor#init) method initializes the Actor, +the event manager which processes the Actor events from the platform event websocket, +and the storage client used in the execution environment. +It should be called before performing any other Actor operations. + +The [`Actor.exit()`](../../reference/class/Actor#exit) method then exits the Actor cleanly, +tearing down the event manager and the storage client. +There is also the [`Actor.fail()`](../../reference/class/Actor#fail) method, which exits the Actor while marking it as failed. + +```python title="src/main.py" +from apify import Actor +from apify_shared.consts import ActorExitCodes + +async def main(): + await Actor.init() + try: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') + await Actor.exit() + except Exception as e: + Actor.log.exception('Error while running Actor') + await Actor.fail(exit_code=ActorExitCodes.ERROR_USER_FUNCTION_THREW, exception=e) +``` + +#### Context manager + +So that you don't have to call the lifecycle methods manually, the [`Actor`](../../reference/class/Actor) class provides a context manager, +which calls the [`Actor.init()`](../../reference/class/Actor#init) method on enter, +the [`Actor.exit()`](../../reference/class/Actor#exit) method on a clean exit, +and the [`Actor.fail()`](../../reference/class/Actor#fail) method when there is an exception during the run of the Actor. + +This is the recommended way to work with the `Actor` class. + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') +``` + +#### Main function + +Another option is to pass a function to the Actor via the [`Actor.main(main_func)`](../../reference/class/Actor#main) method, +which causes the Actor to initialize, run the main function, and exit, catching any runtime errors in the passed function. + +```python title="src/main.py" +from apify import Actor + +async def actor_main_func(): + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') + +async def main(): + await Actor.main(actor_main_func) +``` + +### Rebooting an Actor + +Sometimes, you want to restart your Actor to make it run from the beginning again. +To do that, you can use the [`Actor.reboot()`](../../reference/class/Actor#reboot) method. +When you call it, the Apify platform stops the container of the run, +and starts a new container of the same Actor with the same run ID and storages. + +Don't do it unconditionally, or you might get the Actor in a reboot loop. + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + # TODO: figure out a good reason why to reboot + await Actor.reboot() +``` + +## Actor status message + +To inform you or the users running your Actors about the progress of their runs, +you can set the status message for the run, which will then be visible in the run detail in Apify Console, +or accessible through the Apify API. + +To set the status message for the Actor run, you can use the [`Actor.set_status_message()`](../../reference/class/Actor#set_status_message) method. + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + await Actor.set_status_message('Here we go!') + ... + await Actor.set_status_message('So far so good...') + ... + await Actor.set_status_message('Steady as she goes...') + ... + await Actor.set_status_message('Almost there...') + ... + await Actor.set_status_message('Phew! That was not that hard!') +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/02-actor-input.mdx b/website/versioned_docs/version-1.7/03-concepts/02-actor-input.mdx new file mode 100644 index 00000000..7d9e89b3 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/02-actor-input.mdx @@ -0,0 +1,27 @@ +--- +title: Actor input +sidebar_label: Actor input +--- + +The Actor gets its [input](https://docs.apify.com/platform/actors/running/input) from the input record in its default key-value store. + +To access it, instead of reading the record manually, +you can use the [`Actor.get_input()`](../../reference/class/Actor#get_input) convenience method. +It will get the input record key from the Actor configuration, +read the record from the default key-value store, +and decrypt any [secret input fields](https://docs.apify.com/platform/actors/development/secret-input). + +For example, if an Actor received a JSON input with two fields, +`{ "firstNumber": 1, "secondNumber": 2 }`, +this is how you might process it: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + actor_input = await Actor.get_input() or {} + first_number = actor_input.get('firstNumber') + second_number = actor_input.get('secondNumber') + Actor.log.info(f'Sum: {first_number + second_number}') +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/03-storages.mdx b/website/versioned_docs/version-1.7/03-concepts/03-storages.mdx new file mode 100644 index 00000000..e229e14b --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/03-storages.mdx @@ -0,0 +1,284 @@ +--- +title: Working with storages +sidebar_label: Working with storages +--- + +The `Actor` class provides methods to work either with the default storages of the Actor, or with any other storage, named or unnamed. + +## Types of storages + +There are three types of storages available to Actors. + +First are [datasets](https://docs.apify.com/platform/storage/dataset), which are append-only tables for storing the results of your Actors. +You can open a dataset through the [`Actor.open_dataset()`](../../reference/class/Actor#open_dataset) method, +and work with it through the resulting [`Dataset`](../../reference/class/Dataset) class instance. + +Next there are [key-value stores](https://docs.apify.com/platform/storage/key-value-store), +which function as a read/write storage for storing file-like objects, typically the Actor state or binary results. +You can open a key-value store through the [`Actor.open_key_value_store()`](../../reference/class/Actor#open_key_value_store) method, +and work with it through the resulting [`KeyValueStore`](../../reference/class/KeyValueStore) class instance. + +Finally, there are [request queues](https://docs.apify.com/platform/storage/request-queue). +These are queues into which you can put the URLs you want to scrape, +and from which the Actor can dequeue them and process them. +You can open a request queue through the [`Actor.open_request_queue()`](../../reference/class/Actor#open_request_queue) method, +and work with it through the resulting [`RequestQueue`](../../reference/class/RequestQueue) class instance. + +Each Actor run has its default dataset, default key-value store and default request queue. + +## Local storage emulation + +To be able to develop Actors locally, +the storages that the Apify platform provides are emulated on the local filesystem. + +The storage contents are loaded from and saved to the `storage` folder in the Actor's main folder. +Each storage type is stored in its own subfolder, so for example datasets are stored in the `storage/datasets` folder. + +Each storage is then stored in its own folder, named after the storage, or called `default` if it's the default storage. +For example, a request queue with the name `my-queue` would be stored in `storage/request_queues/my-queue`. + +Each dataset item, key-value store record, or request in a request queue is then stored in its own file in the storage folder. +Dataset items and request queue requests are always JSON files, and key-value store records can be any file type, based on its content type. +For example, the Actor input is typically stored in `storage/key_value_stores/default/INPUT.json`. + +### Local storage persistence + +By default, the storage contents are persisted across multiple Actor runs. +To clean up the Actor storages before the running the Actor, +use the `--purge` flag of the [`apify run`](https://docs.apify.com/cli/docs/reference#apify-run) command of the Apify CLI. + +```bash +apify run --purge +``` + +## Convenience methods for working with default storages + +There are several methods for directly working with the default key-value store or default dataset of the Actor. + +[`Actor.get_value('my-record')`](../../reference/class/Actor#get_value) reads a record from the default key-value store of the Actor. + +[`Actor.set_value('my-record', 'my-value')`](../../reference/class/Actor#set_value) saves a new value to the record in the default key-value store. + +[`Actor.get_input()`](../../reference/class/Actor#get_input) reads the Actor input from the default key-value store of the Actor. + +[`Actor.push_data([{'result': 'Hello, world!'}, ...])`](../../reference/class/Actor#push_data) saves results to the default dataset of the Actor. + +## Opening named and unnamed storages + +The [`Actor.open_dataset()`](../../reference/class/Actor#open_dataset), +[`Actor.open_key_value_store()`](../../reference/class/Actor#open_key_value_store) +and [`Actor.open_request_queue()`](../../reference/class/Actor#open_request_queue) methods +can be used to open any storage for reading and writing. +You can either use them without arguments to open the default storages, +or you can pass a storage ID or name to open another storage. + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + # Work with the default dataset of the Actor + dataset = await Actor.open_dataset() + await dataset.push_data({'result': 'Hello, world!'}) + + # Work with the key-value store with ID 'mIJVZsRQrDQf4rUAf' + key_value_store = await Actor.open_key_value_store(id='mIJVZsRQrDQf4rUAf') + await key_value_store.set_value('record', 'Hello, world!') + + # Work with the request queue with the name 'my-queue' + request_queue = await Actor.open_request_queue(name='my-queue') + await request_queue.add_request({ 'uniqueKey': 'v0Nr', 'url': 'https://example.com' }) +``` + +## Deleting storages + +To delete a storage, you can use the +[`Dataset.drop()`](../../reference/class/Dataset#drop), +[`KeyValueStore.drop()`](../../reference/class/KeyValueStore#drop) +or [`RequestQueue.drop()`](../../reference/class/RequestQueue#drop) method. + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + # Open a key-value store with the name 'my-cool-store' + key_value_store = await Actor.open_key_value_store(name='my-cool-store') + await key_value_store.set_value('record', 'Hello, world!') + ... + + # Now we don't want it anymore + await key_value_store.drop() +``` + +## Working with datasets + +### Reading & writing items + +To write data into a dataset, you can use the [`Dataset.push_data()`](../../reference/class/Dataset#push_data) method. + +To read data from a dataset, you can use the [`Dataset.get_data()`](../../reference/class/Dataset#get_data) method. + +To get an iterator of the data, you can use the [`Dataset.iterate_items()`](../../reference/class/Dataset#iterate_items) method. + +```python +# Open a dataset and write some data in it +dataset = await Actor.open_dataset(name='my-cool-dataset') +await dataset.push_data([{'itemNo': i} for i in range(1000)]) + +# Read back the first half of the data +first_half = await dataset.get_data(limit=500) +print(first_half['items']) + +# Iterate over the second half +second_half = [] +async for item in dataset.iterate_items(offset=500): + second_half.append(item) +print(second_half) +``` + +### Exporting items + +You can also export the dataset items into a key-value store, as either a CSV or a JSON record, +using the [`Dataset.export_to_csv()`](../../reference/class/Dataset#export_to_csv) +or [`Dataset.export_to_json()`](../../reference/class/Dataset#export_to_json) method. + +```python +# Open a dataset and write some data in it +dataset = await Actor.open_dataset(name='my-cool-dataset') +await dataset.push_data([{'itemNo': i} for i in range(1000)]) + +# Export the data as CSV and JSON +await dataset.export_to_csv('data.csv', to_key_value_store_name='my-cool-key-value-store') +await dataset.export_to_json('data.json', to_key_value_store_name='my-cool-key-value-store') + +# Print the exported records +store = await Actor.open_key_value_store(name='my-cool-key-value-store') +print(await store.get_value('data.csv')) +print(await store.get_value('data.json')) +``` + +## Working with key-value stores + +### Reading and writing records + +To read records from a key-value store, you can use the [`KeyValueStore.get_value()`](../../reference/class/KeyValueStore#get_value) method. + +To write records into a key-value store, you can use the [`KeyValueStore.set_value()`](../../reference/class/KeyValueStore#set_value) method. +You can set the content type of a record with the `content_type` argument. +To delete a record, set its value to `None`. + +```python +# Open a key-value store and write some data in it +store = await Actor.open_key_value_store(name='my-cool-key-value-store') +await store.set_value('automatic_text', 'abcd') +await store.set_value('automatic_json', {'ab': 'cd'}) +await store.set_value('explicit_csv', 'a,b\nc,d', content_type='text/csv') + +# Try that the values are read correctly +print(await store.get_value('automatic_text')) +print(await store.get_value('automatic_json')) +print(await store.get_value('explicit_csv')) + +# Delete the `automatic_text` value +await store.set_value('automatic_text', None) +``` + +### Iterating keys + +To get an iterator of the key-value store record keys, +you can use the [`KeyValueStore.iterate_keys()`](../../reference/class/KeyValueStore#iterate_keys) method. + +```python +# Print the info for each record +print('Records in store:') +async for (key, info) in store.iterate_keys(): + print(f'{key=}, {info=}') +``` + +### Public URLs of records + +To get a publicly accessible URL of a key-value store record, +you can use the [`KeyValueStore.get_public_url()`](../../reference/class/KeyValueStore#get_public_url) method. + +```python +print(f'"my_record" record URL: {await store.get_public_url('my_record')}') +``` + +## Working with request queues + +### Adding requests to a queue + +To add a request into the queue, you can use the [`RequestQueue.add_request()`](../../reference/class/RequestQueue#add_request) method. + +You can use the `forefront` boolean argument to specify whether the request should go to the beginning of the queue, or to the end. + +You can use the `uniqueKey` of the request to uniquely identify a request. If you try to add more requests with the same unique key, +only the first one will be added. + +### Reading requests + +To fetch the next request from the queue for processing, +you can use the [`RequestQueue.fetch_next_request()`](../../reference/class/RequestQueue#fetch_next_request) method. + +To get info about a specific request from the queue, +you can use the [`RequestQueue.get_request()`](../../reference/class/RequestQueue#get_request) method. + +### Handling requests + +To mark a request as handled, you can use the [`RequestQueue.mark_request_as_handled()`](../../reference/class/RequestQueue#mark_request_as_handled) method. + +To mark a request as not handled, so that it gets retried, +you can use the [`RequestQueue.reclaim_request()`](../../reference/class/RequestQueue#reclaim_request) method. + +To check if all the requests in the queue are handled, +you can use the [`RequestQueue.is_finished()`](../../reference/class/RequestQueue#is_finished) method. + +### Full example + +```python title="src/main.py" +import asyncio +import random +from apify import Actor + + +async def main(): + async with Actor: + # Open the queue + queue = await Actor.open_request_queue() + + # Add some requests to the queue + for i in range(1, 10): + await queue.add_request({ 'uniqueKey': f'{i}', 'url': f'http://example.com/{i}' }) + + # Add a request to the start of the queue, for priority processing + await queue.add_request({ 'uniqueKey': '0', 'url': 'http://example.com/0' }, forefront=True) + + # If you try to add an existing request again, it will not do anything + operation_info = await queue.add_request({ 'uniqueKey': '5', 'url': 'http://different-example.com/5' }) + print(operation_info) + print(await queue.get_request(operation_info['requestId'])) + + # Finally, process the queue until all requests are handled + while not await queue.is_finished(): + # Fetch the next unhandled request in the queue + request = await queue.fetch_next_request() + # This can happen due to the eventual consistency of the underlying request queue storage, + # best solution is just to sleep a bit + if request is None: + await asyncio.sleep(1) + continue + + Actor.log.info(f'Processing request {request["uniqueKey"]}...') + Actor.log.info(f'Scraping URL {request["url"]}...') + + # Do some fake work, which fails 30% of the time + await asyncio.sleep(1) + if random.random() > 0.3: + # If processing the request was successful, mark it as handled + Actor.log.info('Request successful.') + await queue.mark_request_as_handled(request) + else: + # If processing the request was unsuccessful, reclaim it so it can be processed again + Actor.log.warning('Request failed, will retry!') + await queue.reclaim_request(request) diff --git a/website/versioned_docs/version-1.7/03-concepts/04-actor-events.mdx b/website/versioned_docs/version-1.7/03-concepts/04-actor-events.mdx new file mode 100644 index 00000000..8d795cab --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/04-actor-events.mdx @@ -0,0 +1,113 @@ +--- +title: Handling Actor events & persisting state +sidebar_label: Actor events & state persistence +--- + +During its runtime, the Actor receives Actor events sent by the Apify platform or generated by the Apify SDK itself. + +## Event types + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EventDataDescription
SYSTEM_INFO
{`{
+  "createdAt": datetime,
+  "cpuCurrentUsage": float,
+  "memCurrentBytes": int,
+  "isCpuOverloaded": bool
+}`}
+
+

This event is emitted regularly and it indicates the current resource usage of the Actor.

+ The isCpuOverloaded argument indicates whether the current CPU usage is higher than Config.max_used_cpu_ratio +
MIGRATINGNone +

Emitted when the Actor running on the Apify platform + is going to be migrated + {' '}to another worker server soon.

+ You can use it to persist the state of the Actor so that once it is executed again on the new server, + it doesn't have to start over from the beginning. +
ABORTINGNone + When a user aborts an Actor run on the Apify platform, + they can choose to abort gracefully to allow the Actor some time before getting killed. + This graceful abort emits the ABORTING event which you can use to finish all running tasks and do cleanup. +
PERSIST_STATE
{`{ "isMigrating": bool }`}
 +

Emitted in regular intervals (by default 60 seconds) to notify the Actor that it should persist its state, + in order to avoid repeating all work when the Actor restarts.

+

This event is also emitted automatically when the MIGRATING event happens, + in which case the isMigrating flag is set to True.

+ Note that the PERSIST_STATE event is provided merely for user convenience, + you can achieve the same effect by persisting the state regularly in an interval and listening for the migrating event. +
+ + + +## Adding handlers to events + +To add handlers to these events, you use the [`Actor.on()`](../../reference/class/Actor#on) method, +and to remove them, you use the [`Actor.off()`](../../reference/class/Actor#off) method. + +```python title="src/main.py" +import asyncio +from apify import Actor +from apify_shared.consts import ActorEventTypes + +async def main(): + async with Actor: + total_items = 1000 + + # Load the state if it's saved from some previous execution + processed_items = 0 + actor_state = await Actor.get_value('STATE') + if actor_state is not None: + processed_items = actor_state + + # Save the state when the `PERSIST_STATE` event happens + async def save_state(event_data): + nonlocal processed_items + Actor.log.info('Saving actor state', extra=event_data) + await Actor.set_value('STATE', processed_items) + + Actor.on(ActorEventTypes.PERSIST_STATE, save_state) + + # Do some fake work + for i in range(processed_items, total_items): + Actor.log.info(f'Processing item {i}...') + processed_items = i + await asyncio.sleep(0.1) + + # Suppose we can stop saving the state now + Actor.off(ActorEventTypes.PERSIST_STATE, save_state) + + # Do some more fake work, this time something that can't be restarted, + # so no point persisting the state + for j in range(0, 10): + Actor.log.info(f'Processing item {j} of another kind...') + await asyncio.sleep(1) +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/05-proxy-management.mdx b/website/versioned_docs/version-1.7/03-concepts/05-proxy-management.mdx new file mode 100644 index 00000000..e71c170b --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/05-proxy-management.mdx @@ -0,0 +1,194 @@ +--- +id: proxy-management +title: Proxy management +--- + +[IP address blocking](https://en.wikipedia.org/wiki/IP_address_blocking) +is one of the oldest and most effective ways of preventing access to a website. +It is therefore paramount for a good web scraping library +to provide easy to use but powerful tools which can work around IP blocking. +The most powerful weapon in your anti IP blocking arsenal +is a [proxy server](https://en.wikipedia.org/wiki/Proxy_server). + +With the Apify SDK, you can use your own proxy servers, +proxy servers acquired from third-party providers, +or you can rely on [Apify Proxy](https://apify.com/proxy) for your scraping needs. + +## Quick start + +If you want to use Apify Proxy locally, +make sure that you run your Actors via the Apify CLI +and that you are [logged in](https://docs.apify.com/cli/docs/installation#login-with-your-apify-account) with your Apify account in the CLI. + +### Using Apify Proxy + +```python +proxy_configuration = await Actor.create_proxy_configuration() +proxy_url = await proxy_configuration.new_url() +``` + +### Using your own proxies + +```python +proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], +) +proxy_url = await proxy_configuration.new_url() +``` + +## Proxy Configuration + +All your proxy needs are managed by the [`ProxyConfiguration`](../../reference/class/ProxyConfiguration) class. +You create an instance using the [`Actor.create_proxy_configuration()`](../../reference/class/Actor#create_proxy_configuration) method. +Then you generate proxy URLs using the [`ProxyConfiguration.new_url()`](../../reference/class/ProxyConfiguration#new_url) method. + +### Apify Proxy vs. your own proxies + +The `ProxyConfiguration` class covers both Apify Proxy and custom proxy URLs, +so that you can easily switch between proxy providers. +However, some features of the class are available only to Apify Proxy users, +mainly because Apify Proxy is what one would call a super-proxy. +It's not a single proxy server, but an API endpoint that allows connection +through millions of different IP addresses. +So the class essentially has two modes: Apify Proxy or Your proxy. + +The difference is easy to remember. +Using the `proxy_url` or `new_url_function` arguments enables use of your custom proxy URLs, +whereas all the other options are there to configure Apify Proxy. +Visit the [Apify Proxy docs](https://docs.apify.com/proxy) for more info on how these parameters work. + +### IP Rotation and session management + +`proxyConfiguration.new_url()` allows you to pass a `session_id` parameter. +It will then be used to create a `session_id`-`proxy_url` pair, +and subsequent `new_url()` calls with the same `session_id` will always return the same `proxy_url`. +This is extremely useful in scraping, because you want to create the impression of a real user. + +When no `session_id` is provided, your custom proxy URLs are rotated round-robin, +whereas Apify Proxy manages their rotation using black magic to get the best performance. + +```python +proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], +) +proxy_url = await proxy_configuration.new_url() # http://proxy-1.com +proxy_url = await proxy_configuration.new_url() # http://proxy-2.com +proxy_url = await proxy_configuration.new_url() # http://proxy-1.com +proxy_url = await proxy_configuration.new_url() # http://proxy-2.com +proxy_url = await proxy_configuration.new_url(session_id='a') # http://proxy-1.com +proxy_url = await proxy_configuration.new_url(session_id='b') # http://proxy-2.com +proxy_url = await proxy_configuration.new_url(session_id='b') # http://proxy-2.com +proxy_url = await proxy_configuration.new_url(session_id='a') # http://proxy-1.com +``` + +### Apify Proxy Configuration + +With Apify Proxy, you can select specific proxy groups to use, or countries to connect from. +This allows you to get better proxy performance after some initial research. + +```python +proxy_configuration = await Actor.create_proxy_configuration( + groups=['RESIDENTIAL'], + country_code='US', +) +proxy_url = await proxy_configuration.new_url() +``` + +Now your connections using proxy_url will use only Residential proxies from the US. +Note that you must first get access to a proxy group before you are able to use it. +You can find your available proxy groups in the [proxy dashboard](https://console.apify.com/proxy). + +If you don't specify any proxy groups, automatic proxy selection will be used. + +### Your own proxy configuration + +There are two options how to make `ProxyConfiguration` work with your own proxies. + +Either you can pass it a list of your own proxy servers: + +```python +proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], +) +proxy_url = await proxy_configuration.new_url() +``` + +Or you can pass it a method (accepting one optional argument, the session ID), +to generate proxy URLs automatically: + +```python +def custom_new_url_function(session_id: Optional[str] = None) -> str: + if session_id is not None: + return f'http://my-custom-proxy-supporting-sessions.com?session-id={session_id} + return 'http://my-custom-proxy-not-supporting-sessions.com' + +proxy_configuration = await Actor.create_proxy_configuration( + new_url_function = custom_new_url_function, +) + +proxy_url_with_session = await proxy_configuration.new_url('a') +proxy_url_without_Session = await proxy_configuration.new_url() +``` + +### Configuring proxy based on Actor input + +To make selecting the proxies that the Actor uses easier, +you can use an input field with the editor [`proxy` in your input schema](https://docs.apify.com/platform/actors/development/input-schema#object). +This input will then be filled with a dictionary containing the proxy settings you or the users of your Actor selected for the Actor run. + +You can then use that input to create the proxy configuration: + +```python +actor_input = await Actor.get_input() or {} +proxy_settings = actor_input.get('proxySettings') +proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings) +proxy_url = await proxy_configuration.new_url() +``` + +## Using the generated proxy URLs + +### Requests + +To use the generated proxy URLs with the `requests` library, +use the [`proxies` argument](https://requests.readthedocs.io/en/latest/user/advanced/#proxies): + +```python +proxy_configuration = await Actor.create_proxy_configuration() +proxy_url = await proxy_configuration.new_url() +proxies = { + 'http': proxy_url, + 'https': proxy_url, +} + +response = requests.get('http://example.com', proxies=proxies) +# --- OR --- +with requests.Session() as session: + session.proxies.update(proxies) + response = session.get('http://example.com') +``` + +### HTTPX + +To use the generated proxy URLs with the `httpx` library, +use the [`proxies` argument](https://www.python-httpx.org/advanced/#http-proxying): + +```python +proxy_configuration = await Actor.create_proxy_configuration() +proxy_url = await proxy_configuration.new_url() + +response = httpx.get('http://example.com', proxies=proxy_url) +# --- OR --- +async with httpx.AsyncClient(proxies=proxy_url) as httpx_client: + response = await httpx_client.get('http://example.com') +``` + + diff --git a/website/versioned_docs/version-1.7/03-concepts/06-interacting-with-other-actors.mdx b/website/versioned_docs/version-1.7/03-concepts/06-interacting-with-other-actors.mdx new file mode 100644 index 00000000..59a1017d --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/06-interacting-with-other-actors.mdx @@ -0,0 +1,96 @@ +--- +title: Interacting with other Actors +sidebar_label: Interacting with other Actors +--- + +There are several methods that interact with other Actors and Actor tasks on the Apify platform. + +## Actor.start() + +The [`Actor.start()`](../../reference/class/Actor#start) method starts another Actor on the Apify platform, +and immediately returns the details of the started Actor run. + +```python +# Start your own Actor named 'my-fancy-actor' +actor_run_details = await Actor.start('~my-fancy-actor', {'foo': 'bar'}) +print(f'Started run ID: {actor_run_details["id"]}') +``` + +## Actor.call() + +The [`Actor.call()`](../../reference/class/Actor#call) method starts another Actor on the Apify platform, +and waits for the started Actor run to finish. + +```python +# Start the `apify/screenshot-url`, wait for it to finish, and get its output +actor_run_details = await Actor.call( + 'apify/screenshot-url', + {'url': 'http://example.com', 'delay': 10000 }, +) +run_client = Actor.apify_client.run(actor_run_details['id']) +screenshot = await run_client().key_value_store().get_value('OUTPUT') +``` + +## Actor.call_task() + +The [`Actor.call_task()`](../../reference/class/Actor#call_task) method +starts an [Actor task](https://docs.apify.com/platform/actors/tasks) on the Apify platform, +and waits for the started Actor run to finish. + +```python +# Start the Actor task with ID `Z3m6FPSj0GYZ25rQc`, +# wait for it to finish, and get its dataset items +task_run_details = await Actor.call_task('Z3m6FPSj0GYZ25rQc') +run_client = Actor.apify_client.run(task_run_details['id']) +task_run_dataset_items = await run_client().dataset().list_items() +``` + +## Actor.metamorph() + +The [`Actor.metamorph()`](../../reference/class/Actor#metamorph) operation transforms an Actor run into a run of another Actor with a new input. +This feature is useful if you want to use another Actor to finish the work of your current Actor, +instead of internally starting a new Actor run and waiting for its finish. +With metamorph, you can easily create new Actors on top of existing ones, +and give your users nicer input structure and user interface for the final Actor. +For the users of your Actors, the metamorph operation is completely transparent; +they will just see your Actor got the work done. + +Internally, the system stops the container corresponding to the original Actor run +and starts a new container using a different container image. +All the default storages are preserved, +and the new Actor input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store. + +To make you Actor compatible with the metamorph operation, +use [`Actor.get_input()`](../../reference/class/Actor#get_input) +instead of [`Actor.get_value('INPUT')`](../../reference/class/Actor#get_value) to read your Actor input. +This method will fetch the input using the right key in a case of metamorphed run. + +For example, imagine you have an Actor that accepts a hotel URL on input, +and then internally uses the [`apify/web-scraper`](https://apify.com/apify/web-scraper) public Actor to scrape all the hotel reviews. +The metamorphing code would look as follows: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + # Get the original Actor input + actor_input = await Actor.get_input() or {} + hotel_url = actor_input.get('hotel_url') + + # Create new input for `apify/web-scraper` + web_scraper_input = { + 'startUrls': [{ url: hotelUrl }], + 'pageFunction': """async function pageFunction(context) { + // Here you pass the JavaScript page function + // that scrapes all the reviews from the hotel's URL + }""", + } + + # Metamorph the Actor run to `apify/web-scraper` with the new input + await Actor.metamorph('apify/web-scraper', web_scraper_input) + + # This code will not be called, + # since the `metamorph` action terminates the current Actor run container + print('You will not see this!') +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/07-webhooks.mdx b/website/versioned_docs/version-1.7/03-concepts/07-webhooks.mdx new file mode 100644 index 00000000..b8b6480a --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/07-webhooks.mdx @@ -0,0 +1,50 @@ +--- +title: Creating webhooks +sidebar_label: Creating webhooks +--- + +Webhooks allow you to configure the Apify platform to perform an action when a certain event occurs. +For example, you can use them to start another Actor when the current run finishes or fails. + +You can learn more in the [documentation for webhooks](https://docs.apify.com/platform/integrations/webhooks). + +## Creating an ad-hoc webhook dynamically + +Besides creating webhooks manually in Apify Console, or through the Apify API, +you can also create [ad-hoc webhooks](https://docs.apify.com/platform/integrations/webhooks/ad-hoc-webhooks) +dynamically from the code of your Actor using the [`Actor.add_webhook()`](../../reference/class/Actor#add_webhook) method: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + await Actor.add_webhook( + event_types: ['ACTOR.RUN.FAILED'], + request_url: 'https://example.com/run-failed', + ) + raise RuntimeError('I am an error and I know it!') +``` + +Note that webhooks are only supported when running on the Apify platform. +When running the Actor locally, the method will print a warning and have no effect. + +### Preventing duplicate webhooks + +To ensure that duplicate ad-hoc webhooks won't get created in a case of Actor restart, +you can use the `idempotencyKey` parameter. +The idempotency key must be unique across all the webhooks of a user so that only one webhook gets created for a given value. +You can use, for example, the Actor run ID as the idempotency key: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + await Actor.add_webhook( + event_types: ['ACTOR.RUN.FAILED'], + request_url: 'https://example.com/run-failed', + idempotency_key: Actor.config.actor_run_id, + ) + raise RuntimeError('I am an error and I know it!') +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/08-access-apify-api.mdx b/website/versioned_docs/version-1.7/03-concepts/08-access-apify-api.mdx new file mode 100644 index 00000000..83ee6191 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/08-access-apify-api.mdx @@ -0,0 +1,42 @@ +--- +title: Accessing the Apify API +sidebar_label: Accessing Apify API +--- + +The Apify SDK contains many useful features for making Actor development easier. +However, it does not cover all the features the Apify API offers. + +For working with the Apify API directly, +you can use the provided instance of the [Apify API Client](https://docs.apify.com/api/client/python) library. + +## Actor.apify_client + +To access the provided instance of [`ApifyClientAsync`](https://docs.apify.com/api/client/python/reference/class/ApifyClientAsync), +you can use the [`Actor.apify_client`](../../reference/class/Actor#apify_client) property. + +For example, to get the details of your user, you can use this snippet: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + me = await Actor.apify_client.user('me').get() + print(me) +``` + +## Actor.new_client() + +If you want to create a completely new instance of the client, +for example, to get a client for a different user or change the configuration of the client, +you can use the [`Actor.new_client()`](../../reference/class/Actor#new_client) method: + +```python title="src/main.py" +from apify import Actor + +async def main(): + async with Actor: + another_users_client = Actor.new_client(token='ANOTHER_USERS_TOKEN', max_retries=2) + them = await another_users_client.user('me').get() + print(them) +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/09-running-webserver.mdx b/website/versioned_docs/version-1.7/03-concepts/09-running-webserver.mdx new file mode 100644 index 00000000..d1ad9d90 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/09-running-webserver.mdx @@ -0,0 +1,66 @@ +--- +title: Running a webserver in your Actor +sidebar_label: Running a webserver +--- + +Each Actor run on the Apify platform is assigned a unique hard-to-guess URL (for example `https://8segt5i81sokzm.runs.apify.net`), +which enables HTTP access to an optional web server running inside the Actor run's container. + +The URL is available in the following places: + +- In Apify Console, on the Actor run details page as the **Container URL** field. +- In the API as the `containerUrl` property of the [Run object](https://docs.apify.com/api/v2#/reference/actors/run-object/get-run). +- In the Actor as the `Actor.config.container_url` property. + +The web server running inside the container must listen at the port defined by the `Actor.config.container_port` property. +When running Actors locally, the port defaults to `4321`, +so the web server will be accessible at `http://localhost:4321`. + +## Example + +The following example demonstrates how to start a simple web server in your Actor, +which will respond to every GET request with the number of items that the Actor has processed so far: + +```python title="src/main.py" +import asyncio +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from apify import Actor + +processed_items = 0 +http_server = None + +# Just a simple handler that will print the number of processed items so far +# on every GET request +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.log_request() + self.send_response(200) + self.end_headers() + self.wfile.write(bytes(f'Processed items: {processed_items}', encoding='utf-8')) + +def run_server(): + # Start the HTTP server on the provided port, + # and save a reference to the server + global http_server + with ThreadingHTTPServer(('', Actor.config.container_port), RequestHandler) as server: + Actor.log.info(f'Server running on {Actor.config.container_url}') + http_server = server + server.serve_forever() + +async def main(): + global processed_items + async with Actor: + # Start the HTTP server in a separate thread + run_server_task = asyncio.get_running_loop().run_in_executor(None, run_server) + + # Simulate doing some work + for _ in range(100): + await asyncio.sleep(1) + processed_items += 1 + Actor.log.info(f'Processed items: {processed_items}') + + # Signal the HTTP server to shut down, and wait for it to finish + http_server.shutdown() + await run_server_task +``` diff --git a/website/versioned_docs/version-1.7/03-concepts/10-logging.mdx b/website/versioned_docs/version-1.7/03-concepts/10-logging.mdx new file mode 100644 index 00000000..33c98b29 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/10-logging.mdx @@ -0,0 +1,109 @@ +--- +title: Logging +sidebar_label: Logging +--- + +The Apify SDK is logging useful information through the [`logging`](https://docs.python.org/3/library/logging.html) module +from Python's standard library, into the logger with the name `apify`. + +## Automatic configuration + +When you create an Actor from an Apify-provided template, either in Apify Console or through the Apify CLI, +you do not have to configure the logger yourself. +The template already contains initialization code for the logger, +which sets the logger level to `DEBUG` and the log formatter to [`ActorLogFormatter`](../../reference/class/ActorLogFormatter). + +## Manual configuration + +### Configuring the log level + +In Python's default behavior, if you don't configure the logger otherwise, +only logs with level `WARNING` or higher are printed out to the standard output, without any formatting. +To also have logs with `DEBUG` and `INFO` level printed out, +you need to call the [`Logger.setLevel()`](https://docs.python.org/3/library/logging.html#logging.Logger.setLevel) method on the logger, +with the desired minimum level as an argument. + +### Configuring the log formatting + +By default, only the log message is printed out to the output, without any formatting. +To have a nicer output, with the log level printed in color, the messages nicely aligned, and extra log fields printed out, +you can use the [`ActorLogFormatter`](../../reference/class/ActorLogFormatter) class from the `apify.log` module. + +### Example log configuration + +To configure and test the logger, you can use this snippet: + +```python +import logging + +from apify import Actor +from apify.log import ActorLogFormatter + +handler = logging.StreamHandler() +handler.setFormatter(ActorLogFormatter()) + +apify_logger = logging.getLogger('apify') +apify_logger.setLevel(logging.DEBUG) +apify_logger.addHandler(handler) +``` + +This configuration will cause all levels of messages to be printed to the standard output, +with some pretty formatting. + +## Logger usage + +Here you can see how all the log levels would look like. + +You can use the `extra` argument for all log levels, it's not specific to the warning level. +When you use `Logger.exception()`, there is no need to pass the Exception object to the log manually, +it will automatiacally infer it from the current execution context and print the exception details. + +```python +Actor.log.debug('This is a debug message') +Actor.log.info('This is an info message') +Actor.log.warning('This is a warning message', extra={'reason': 'Bad Actor!'}) +Actor.log.error('This is an error message') +try: + raise RuntimeError('Ouch!') +except: + Actor.log.exception('This is an exceptional message') +``` + +Result: + + + + +
+    
DEBUG This is a debug message

+    
INFO  This is an info message

+    
WARN  This is a warning message {`({"reason": "Bad Actor!"})`}

+    
ERROR This is an error message

+    
ERROR This is an exceptional message

+    
      Traceback (most recent call last):

+    
        File "main.py", line 6, in <module>

+    
          raise RuntimeError('Ouch!')

+    
      RuntimeError: Ouch!

+
+ + diff --git a/website/versioned_docs/version-1.7/03-concepts/11-configuration.mdx b/website/versioned_docs/version-1.7/03-concepts/11-configuration.mdx new file mode 100644 index 00000000..70b26858 --- /dev/null +++ b/website/versioned_docs/version-1.7/03-concepts/11-configuration.mdx @@ -0,0 +1,45 @@ +--- +title: Actor configuration and environment variables +sidebar_label: Configuration & env vars +--- + +The [`Actor`](../../reference/class/Actor) class gets configured using the [`Configuration`](../../reference/class/Configuration) class, +which initializes itself based on the provided environment variables. + +If you're using the Apify SDK in your Actors on the Apify platform, or Actors running locally through the Apify CLI, +you don't need to configure the `Actor` class manually, +unless you have some specific requirements, everything will get configured automatically. + +If you need some special configuration, you can adjust it either through the `Configuration` class directly, +or by setting environment variables when running the Actor locally. + +To see the full list of configuration options, check the `Configuration` class +or the list of environment variables that the Actor understands. + +### Configuring from code + +This will cause the Actor to persist its state every 10 seconds: + +```python title="src/main.py" +from apify import Actor +from apify_shared.consts import ActorEventTypes + +async def main(): + global_config = Configuration.get_global_configuration() + global_config.persist_state_interval_millis = 10000 + + async with Actor: + async def save_state(): + await Actor.set_value('STATE', 'Hello, world!') + + # The `save_state` handler will be called every 10 seconds now + Actor.on(ActorEventTypes.PERSIST_STATE, save_state) +``` + +### Configuring via environment variables + +This Actor run will not persist its local storages to the filesystem: + +```bash +APIFY_PERSIST_STORAGE=0 apify run +``` diff --git a/website/versioned_docs/version-1.7/api-packages.json b/website/versioned_docs/version-1.7/api-packages.json new file mode 100644 index 00000000..e2e41f0d --- /dev/null +++ b/website/versioned_docs/version-1.7/api-packages.json @@ -0,0 +1 @@ +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] diff --git a/website/versioned_docs/version-1.7/api-typedoc.json b/website/versioned_docs/version-1.7/api-typedoc.json new file mode 100644 index 00000000..f63cd0da --- /dev/null +++ b/website/versioned_docs/version-1.7/api-typedoc.json @@ -0,0 +1,8083 @@ +{ + "id": 0, + "name": "apify", + "kind": 1, + "kindString": "Project", + "flags": {}, + "originalName": "", + "children": [ + { + "id": 1, + "name": "crypto_random_object_id", + "module": "_crypto", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Python reimplementation of cryptoRandomObjectId from `@apify/utilities`.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_crypto.py", + "line": 128, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_crypto.py#L128" + } + ], + "signatures": [ + { + "id": 2, + "name": "crypto_random_object_id", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Python reimplementation of cryptoRandomObjectId from `@apify/utilities`.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 3, + "name": "length", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "17" + } + ] + } + ] + }, + { + "id": 4, + "name": "decrypt_input_secrets", + "module": "_crypto", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Decrypt input secrets.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_crypto.py", + "line": 134, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_crypto.py#L134" + } + ], + "signatures": [ + { + "id": 5, + "name": "decrypt_input_secrets", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Decrypt input secrets.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [ + { + "id": 6, + "name": "private_key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "rsa.RSAPrivateKey" + } + }, + { + "id": 7, + "name": "input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + } + ] + } + ] + }, + { + "id": 8, + "name": "force_remove", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"JS-like rm(filename, { force: true }).\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 254, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L254" + } + ], + "signatures": [ + { + "id": 9, + "name": "force_remove", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"JS-like rm(filename, { force: true }).\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 10, + "name": "filename", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 11, + "name": "guess_file_extension", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Guess the file extension based on content type.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 270, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L270" + } + ], + "signatures": [ + { + "id": 12, + "name": "guess_file_extension", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Guess the file extension based on content type.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "parameters": [ + { + "id": 13, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 14, + "name": "unique_key_to_request_id", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate request ID based on unique key in a deterministic way.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 296, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L296" + } + ], + "signatures": [ + { + "id": 15, + "name": "unique_key_to_request_id", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate request ID based on unique key in a deterministic way.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 16, + "name": "unique_key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 17, + "name": "force_rename", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Rename a directory. Checks for existence of source directory and removes destination directory if it exists.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 302, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L302" + } + ], + "signatures": [ + { + "id": 18, + "name": "force_rename", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Rename a directory. Checks for existence of source directory and removes destination directory if it exists.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 19, + "name": "src_dir", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 20, + "name": "dst_dir", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 21, + "name": "budget_ow", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Budget version of ow.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 386, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L386" + } + ], + "signatures": [ + { + "id": 22, + "name": "budget_ow", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Budget version of ow.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 23, + "name": "value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "dict | str | float | bool" + } + }, + { + "id": 24, + "name": "predicate", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "dict[str, tuple[type, bool]] | tuple[type, bool]" + } + }, + { + "id": 25, + "name": "value_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 26, + "name": "compute_short_hash", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\\n\", {\"Arguments\": [{\"param\": \"data\", \"desc\": \"The binary data to be hashed.\"}, {\"param\": \"length\", \"desc\": \"The length of the hash to be returned.\\n\"}]}, {\"Returns\": [\"A substring (prefix) of the hexadecimal hash of the data.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 417, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L417" + } + ], + "signatures": [ + { + "id": 27, + "name": "compute_short_hash", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Computes a hexadecimal SHA-256 hash of the provided data and returns a substring (prefix) of it.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 28, + "name": "data", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "bytes" + } + }, + { + "id": 29, + "name": "length", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "8" + } + ] + } + ] + }, + { + "id": 30, + "name": "normalize_url", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Normalizes a URL.\\n\\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\\n\", {\"Arguments\": [{\"param\": \"url\", \"desc\": \"The URL to be normalized.\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"Flag to determine whether the fragment part of the URL should be retained.\\n\"}]}, {\"Returns\": [\"A string containing the normalized URL.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 431, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L431" + } + ], + "signatures": [ + { + "id": 31, + "name": "normalize_url", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Normalizes a URL.\\n\\nThis function cleans and standardizes a URL by removing leading and trailing whitespaces,\\nconverting the scheme and netloc to lower case, stripping unwanted tracking parameters\\n(specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,\\nand optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally\\nidentical but differ in trivial ways (such as parameter order or casing) are treated as the same.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 32, + "name": "url", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 33, + "name": "keep_url_fragment", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 34, + "name": "compute_unique_key", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Computes a unique key for caching & deduplication of requests.\\n\\nThis function computes a unique key by normalizing the provided URL and method.\\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\\nincluded in the key. Otherwise, the unique key is just the normalized URL.\\n\", {\"Arguments\": [{\"param\": \"url\", \"desc\": \"The request URL.\"}, {\"param\": \"method\", \"desc\": \"The HTTP method, defaults to 'GET'.\"}, {\"param\": \"payload\", \"desc\": \"The request payload, defaults to None.\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"A flag indicating whether to keep the URL fragment, defaults to False.\"}, {\"param\": \"use_extended_unique_key\", \"desc\": \"A flag indicating whether to include a hashed payload in the key, defaults to False.\\n\"}]}, {\"Returns\": [\"A string representing the unique key for the request.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_utils.py", + "line": 477, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_utils.py#L477" + } + ], + "signatures": [ + { + "id": 35, + "name": "compute_unique_key", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Computes a unique key for caching & deduplication of requests.\\n\\nThis function computes a unique key by normalizing the provided URL and method.\\nIf 'use_extended_unique_key' is True and a payload is provided, the payload is hashed and\\nincluded in the key. Otherwise, the unique key is just the normalized URL.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 36, + "name": "url", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 37, + "name": "method", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + }, + "defaultValue": "'GET'" + }, + { + "id": 38, + "name": "payload", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "bytes | None" + }, + "defaultValue": "None" + }, + { + "id": 39, + "name": "keep_url_fragment", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 40, + "name": "use_extended_unique_key", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 41, + "name": "Actor", + "module": "actor", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The main class of the SDK, through which all the actor operations should be done.\"]}" + } + ] + }, + "children": [ + { + "id": 42, + "name": "__init__", + "module": "actor", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\", {\"Arguments\": [{\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"The actor configuration to be used. If not passed, a new Configuration instance will be created.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 81, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L81" + } + ], + "signatures": [ + { + "id": 43, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the methods on this class function as classmethods too,\\nand that is their preferred usage.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 44, + "name": "config", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 45, + "name": "apify_client", + "module": "actor", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 174, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L174" + } + ] + }, + { + "id": 46, + "name": "config", + "module": "actor", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 181, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L181" + } + ] + }, + { + "id": 47, + "name": "event_manager", + "module": "actor", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 188, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L188" + } + ] + }, + { + "id": 48, + "name": "log", + "module": "actor", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 196, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L196" + } + ] + }, + { + "id": 49, + "name": "init", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 205, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L205" + } + ], + "signatures": [ + { + "id": 50, + "name": "init", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the actor instance.\\n\\nThis initializes the Actor instance.\\nIt configures the right storage client based on whether the actor is running locally or on the Apify platform,\\nit initializes the event manager for processing actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events,\\nso that the actor can regularly persist its state in response to these events.\\n\\nThis method should be called immediately before performing any additional actor actions,\\nand it should be called only once.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 51, + "name": "get_system_info", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the current system info.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 261, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L261" + } + ], + "signatures": [ + { + "id": 52, + "name": "get_system_info", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the current system info.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [] + } + ] + }, + { + "id": 53, + "name": "exit", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"type\": \"int, optional\", \"desc\": \"The exit code with which the actor should fail (defaults to `0`).\"}, {\"param\": \"event_listeners_timeout_secs\", \"type\": \"float, optional\", \"desc\": \"How long should the actor wait for actor event listeners to finish before exiting.\"}, {\"param\": \"status_message\", \"type\": \"str, optional\", \"desc\": \"The final status message that the actor should display.\"}, {\"param\": \"cleanup_timeout\", \"type\": \"timedelta, optional\", \"desc\": \"How long we should wait for event listeners.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 298, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L298" + } + ], + "signatures": [ + { + "id": 54, + "name": "exit", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Exit the actor instance.\\n\\nThis stops the Actor instance.\\nIt cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event,\\nwaits for all the event listeners to finish,\\nand stops the event manager.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 55, + "name": "exit_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "0" + }, + { + "id": 56, + "name": "event_listeners_timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "float | None" + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" + }, + { + "id": 57, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 58, + "name": "cleanup_timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta" + }, + "defaultValue": "timedelta(seconds=30)" + } + ] + } + ] + }, + { + "id": 59, + "name": "fail", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"type\": \"int, optional\", \"desc\": \"The exit code with which the actor should fail (defaults to `1`).\"}, {\"param\": \"exception\", \"type\": \"BaseException, optional\", \"desc\": \"The exception with which the actor failed.\"}, {\"param\": \"status_message\", \"type\": \"str, optional\", \"desc\": \"The final status message that the actor should display.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 372, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L372" + } + ], + "signatures": [ + { + "id": 60, + "name": "fail", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fail the actor instance.\\n\\nThis performs all the same steps as Actor.exit(),\\nbut it additionally sets the exit code to `1` (by default).\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 61, + "name": "exit_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "1" + }, + { + "id": 62, + "name": "exception", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "BaseException | None" + }, + "defaultValue": "None" + }, + { + "id": 63, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 64, + "name": "main", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\nit sets up a connection to listen for platform events.\\nFor example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\nit prints error details to console so that they are stored to the log,\\nand finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\", {\"Arguments\": [{\"param\": \"main_actor_function\", \"type\": \"Callable\", \"desc\": \"The user function which should be run in the actor\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 412, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L412" + } + ], + "signatures": [ + { + "id": 65, + "name": "main", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the actor, run the passed function and finish the actor cleanly.\\n\\n**The `Actor.main()` function is optional** and is provided merely for your convenience.\\nIt is mainly useful when you're running your code as an actor on the [Apify platform](https://apify.com/actors).\\n\\nThe `Actor.main()` function performs the following actions:\\n\\n- When running on the Apify platform (i.e. `APIFY_IS_AT_HOME` environment variable is set),\\nit sets up a connection to listen for platform events.\\nFor example, to get a notification about an imminent migration to another server.\\n- It invokes the user function passed as the `main_actor_function` parameter.\\n- If the user function was an async function, it awaits it.\\n- If the user function throws an exception or some other error is encountered,\\nit prints error details to console so that they are stored to the log,\\nand finishes the actor cleanly.\\n- Finally, it exits the Python process, with zero exit code on success and non-zero on errors.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "MainReturnType | None" + }, + "parameters": [ + { + "id": 66, + "name": "main_actor_function", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Callable[[], MainReturnType]" + } + } + ] + } + ] + }, + { + "id": 67, + "name": "new_client", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\", {\"Arguments\": [{\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token\"}, {\"param\": \"api_url\", \"type\": \"str, optional\", \"desc\": \"The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com\"}, {\"param\": \"max_retries\", \"type\": \"int, optional\", \"desc\": \"How many times to retry a failed request at most\"}, {\"param\": \"min_delay_between_retries_millis\", \"type\": \"int, optional\", \"desc\": \"How long will the client wait between retrying requests\\n(increases exponentially from this value)\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"The socket timeout of the HTTP requests sent to the Apify API\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 457, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L457" + } + ], + "signatures": [ + { + "id": 68, + "name": "new_client", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python) package,\\nand it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment variables.\\n\\nYou can override the token via the available options.\\nThat's useful if you want to use the client as a different Apify user than the SDK internals are using.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyClientAsync" + }, + "parameters": [ + { + "id": 69, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 70, + "name": "api_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 71, + "name": "max_retries", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 72, + "name": "min_delay_between_retries_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 73, + "name": "timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 74, + "name": "open_dataset", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"Dataset\", \"desc\": \"An instance of the `Dataset` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 513, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L513" + } + ], + "signatures": [ + { + "id": 75, + "name": "open_dataset", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Dataset", + "id": 292 + }, + "parameters": [ + { + "id": 76, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 77, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 78, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 79, + "name": "open_key_value_store", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"KeyValueStore\", \"desc\": \"An instance of the `KeyValueStore` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 552, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L552" + } + ], + "signatures": [ + { + "id": 80, + "name": "open_key_value_store", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "KeyValueStore", + "id": 352 + }, + "parameters": [ + { + "id": 81, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 82, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 83, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 84, + "name": "open_request_queue", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to `True` then the Apify cloud storage is always used.\\nThis way it is possible to combine local and cloud storage.\\n\"}]}, {\"Returns\": [{\"param\": \"RequestQueue\", \"desc\": \"An instance of the `RequestQueue` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 590, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L590" + } + ], + "signatures": [ + { + "id": 85, + "name": "open_request_queue", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "RequestQueue", + "id": 376 + }, + "parameters": [ + { + "id": 86, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 87, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 88, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 89, + "name": "push_data", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\", {\"Arguments\": [{\"param\": \"data\", \"type\": \"object or list of objects, optional\", \"desc\": \"The data to push to the default dataset.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 629, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L629" + } + ], + "signatures": [ + { + "id": 90, + "name": "push_data", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 91, + "name": "data", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + } + ] + } + ] + }, + { + "id": 92, + "name": "get_input", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 647, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L647" + } + ], + "signatures": [ + { + "id": 93, + "name": "get_input", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the actor input value from the default key-value store associated with the current actor run.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [] + } + ] + }, + { + "id": 94, + "name": "get_value", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key of the record which to retrieve.\"}, {\"param\": \"default_value\", \"type\": \"Any, optional\", \"desc\": \"Default value returned in case the record does not exist.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 667, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L667" + } + ], + "signatures": [ + { + "id": 95, + "name": "get_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the default key-value store associated with the current actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [ + { + "id": 96, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 97, + "name": "default_value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 98, + "name": "set_value", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key of the record which to set.\"}, {\"param\": \"value\", \"type\": \"any\", \"desc\": \"The value of the record which to set, or None, if the record should be deleted.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type which should be set to the value.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 683, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L683" + } + ], + "signatures": [ + { + "id": 99, + "name": "set_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 100, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 101, + "name": "value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + }, + { + "id": 102, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 103, + "name": "on", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n- `ActorEventTypes.SYSTEM_INFO`:\\nEmitted every minute, the event data contains info about the resource usage of the actor.\\n- `ActorEventTypes.MIGRATING`:\\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\\nso that they are not interrupted by the migration..\\n- `ActorEventTypes.PERSIST_STATE`:\\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\nin order to avoid repeating all work when the actor restarts.\\nThis event is automatically emitted together with the migrating event,\\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\nNote that this event is provided merely for your convenience,\\nyou can achieve the same effect using an interval and listening for the migrating event.\\n- `ActorEventTypes.ABORTING`:\\nWhen a user aborts an actor run on the Apify platform,\\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"type\": \"ActorEventTypes\", \"desc\": \"The actor event for which to listen to.\"}, {\"param\": \"listener\", \"type\": \"Callable\", \"desc\": \"The function which is to be called when the event is emitted (can be async).\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 716, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L716" + } + ], + "signatures": [ + { + "id": 104, + "name": "on", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add an event listener to the actor's event manager.\\n\\nThe following events can be emitted:\\n- `ActorEventTypes.SYSTEM_INFO`:\\nEmitted every minute, the event data contains info about the resource usage of the actor.\\n- `ActorEventTypes.MIGRATING`:\\nEmitted when the actor running on the Apify platform is going to be migrated to another worker server soon.\\nYou can use it to persist the state of the actor and gracefully stop your in-progress tasks,\\nso that they are not interrupted by the migration..\\n- `ActorEventTypes.PERSIST_STATE`:\\nEmitted in regular intervals (by default 60 seconds) to notify the actor that it should persist its state,\\nin order to avoid repeating all work when the actor restarts.\\nThis event is automatically emitted together with the migrating event,\\nin which case the `isMigrating` flag in the event data is set to True, otherwise the flag is False.\\nNote that this event is provided merely for your convenience,\\nyou can achieve the same effect using an interval and listening for the migrating event.\\n- `ActorEventTypes.ABORTING`:\\nWhen a user aborts an actor run on the Apify platform,\\nthey can choose to abort it gracefully, to allow the actor some time before getting terminated.\\nThis graceful abort emits the aborting event, which you can use to clean up the actor state.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Callable" + }, + "parameters": [ + { + "id": 105, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "ActorEventTypes" + } + }, + { + "id": 106, + "name": "listener", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Callable" + } + } + ] + } + ] + }, + { + "id": 107, + "name": "off", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"type\": \"ActorEventTypes\", \"desc\": \"The actor event for which to remove listeners.\"}, {\"param\": \"listener\", \"type\": \"Callable, optional\", \"desc\": \"The listener which is supposed to be removed. If not passed, all listeners of this event are removed.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 750, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L750" + } + ], + "signatures": [ + { + "id": 108, + "name": "off", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove a listener, or all listeners, from an actor event.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 109, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "ActorEventTypes" + } + }, + { + "id": 110, + "name": "listener", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Callable | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 111, + "name": "is_at_home", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 765, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L765" + } + ], + "signatures": [ + { + "id": 112, + "name": "is_at_home", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return `True` when the actor is running on the Apify platform, and `False` otherwise (for example when running locally).\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 113, + "name": "get_env", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 773, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L773" + } + ], + "signatures": [ + { + "id": 114, + "name": "get_env", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables,\\nsee the [Actor documentation](https://docs.apify.com/actors/development/environment-variables).\\nIf some variables are not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [] + } + ] + }, + { + "id": 115, + "name": "start", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"wait_for_finish\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish.\\nBy default, it is 0, the maximum value is 300.\"}, {\"param\": \"webhooks\", \"type\": \"list of dict, optional\", \"desc\": \"Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks)\\nassociated with the actor run which can be used to receive a notification,\\ne.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor or task, you do not have to add it again here.\\nEach webhook is represented by a dictionary containing these items:\\n* ``event_types``: list of ``WebhookEventType`` values which trigger the webhook\\n* ``request_url``: URL to which to send the webhook HTTP request\\n* ``payload_template`` (optional): Optional template for the request payload\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 788, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L788" + } + ], + "signatures": [ + { + "id": 116, + "name": "start", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run an actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [ + { + "id": 117, + "name": "actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 118, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 119, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 120, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 121, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 122, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 123, + "name": "timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 124, + "name": "wait_for_finish", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 125, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[dict] | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 126, + "name": "abort", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\", {\"Arguments\": [{\"param\": \"run_id\", \"type\": \"str\", \"desc\": \"The ID of the actor run to be aborted.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"gracefully\", \"type\": \"bool, optional\", \"desc\": \"If True, the actor run will abort gracefully.\\nIt will send ``aborting`` and ``persistStates`` events into the run and force-stop the run after 30 seconds.\\nIt is helpful in cases where you plan to resurrect the run later.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the aborted actor run\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 870, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L870" + } + ], + "signatures": [ + { + "id": 127, + "name": "abort", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Abort given actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [ + { + "id": 128, + "name": "run_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 129, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 130, + "name": "gracefully", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 131, + "name": "call", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"webhooks\", \"type\": \"list, optional\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor, you do not have to add it again here.\"}, {\"param\": \"wait_secs\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 913, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L913" + } + ], + "signatures": [ + { + "id": 132, + "name": "call", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 133, + "name": "actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 134, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 135, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 136, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 137, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 138, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 139, + "name": "timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 140, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[dict] | None" + }, + "defaultValue": "None" + }, + { + "id": 141, + "name": "wait_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 142, + "name": "call_task", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\", {\"Arguments\": [{\"param\": \"task_id\", \"type\": \"str\", \"desc\": \"The ID of the actor to be run.\"}, {\"param\": \"task_input\", \"type\": \"Any, optional\", \"desc\": \"Overrides the input to pass to the actor run.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"type\": \"str, optional\", \"desc\": \"Specifies the actor build to run. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"type\": \"int, optional\", \"desc\": \"Memory limit for the run, in megabytes.\\nBy default, the run uses a memory limit specified in the default run configuration for the actor.\"}, {\"param\": \"timeout_secs\", \"type\": \"int, optional\", \"desc\": \"Optional timeout for the run, in seconds.\\nBy default, the run uses timeout specified in the default run configuration for the actor.\"}, {\"param\": \"webhooks\", \"type\": \"list, optional\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the actor run,\\nwhich can be used to receive a notification, e.g. when the actor finished or failed.\\nIf you already have a webhook set up for the actor, you do not have to add it again here.\"}, {\"param\": \"wait_secs\", \"type\": \"int, optional\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits indefinitely.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Info about the started actor run\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 989, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L989" + } + ], + "signatures": [ + { + "id": 143, + "name": "call_task", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait_secs argument is provided.\\n\\nNote that an actor task is a saved input configuration and options for an actor.\\nIf you want to run an actor directly rather than an actor task, please use the `Actor.call`\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 144, + "name": "task_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 145, + "name": "task_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "dict | None" + }, + "defaultValue": "None" + }, + { + "id": 146, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 147, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 148, + "name": "timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 149, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[dict] | None" + }, + "defaultValue": "None" + }, + { + "id": 150, + "name": "wait_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 151, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 152, + "name": "metamorph", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\", {\"Arguments\": [{\"param\": \"target_actor_id\", \"type\": \"str\", \"desc\": \"ID of the target actor that the run should be transformed into\"}, {\"param\": \"run_input\", \"type\": \"Any, optional\", \"desc\": \"The input to pass to the new run.\"}, {\"param\": \"target_actor_build\", \"type\": \"str, optional\", \"desc\": \"The build of the target actor. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the target actor (typically the latest build).\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the input.\"}, {\"param\": \"custom_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long to sleep for after the metamorph, to wait for the container to be stopped.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The actor run data.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 1064, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1064" + } + ], + "signatures": [ + { + "id": 153, + "name": "metamorph", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Transform this actor run to an actor run of a different actor.\\n\\nThe platform stops the current actor container and starts a new container with the new actor instead.\\nAll the default storages are preserved,\\nand the new input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 154, + "name": "target_actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 155, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 156, + "name": "target_actor_build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 157, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 158, + "name": "custom_after_sleep_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 159, + "name": "reboot", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"Arguments\": [{\"param\": \"event_listeners_timeout_secs\", \"type\": \"int, optional\", \"desc\": \"How long should the actor wait for actor event listeners to finish before exiting\"}, {\"param\": \"custom_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long to sleep for after the reboot, to wait for the container to be stopped.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 1130, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1130" + } + ], + "signatures": [ + { + "id": 160, + "name": "reboot", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Internally reboot this actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 161, + "name": "event_listeners_timeout_secs", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT_SECS" + }, + { + "id": 162, + "name": "custom_after_sleep_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 163, + "name": "add_webhook", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"Arguments\": [{\"param\": \"event_types\", \"type\": \"list of WebhookEventType\", \"desc\": \"List of event types that should trigger the webhook. At least one is required.\"}, {\"param\": \"request_url\", \"type\": \"str\", \"desc\": \"URL that will be invoked once the webhook is triggered.\"}, {\"param\": \"payload_template\", \"type\": \"str, optional\", \"desc\": \"Specification of the payload that will be sent to request_url\"}, {\"param\": \"ignore_ssl_errors\", \"type\": \"bool, optional\", \"desc\": \"Whether the webhook should ignore SSL errors returned by request_url\"}, {\"param\": \"do_not_retry\", \"type\": \"bool, optional\", \"desc\": \"Whether the webhook should retry sending the payload to request_url upon\\nfailure.\"}, {\"param\": \"idempotency_key\", \"type\": \"str, optional\", \"desc\": \"A unique identifier of a webhook. You can use it to ensure that you won't\\ncreate the same webhook multiple times.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The created webhook\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 1178, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1178" + } + ], + "signatures": [ + { + "id": 164, + "name": "add_webhook", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an ad-hoc webhook for the current actor run.\\n\\nThis webhook lets you receive a notification when the actor run finished or failed.\\n\\nNote that webhooks are only supported for actors running on the Apify platform.\\nWhen running the actor locally, the function will print a warning and have no effect.\\n\\nFor more information about Apify actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [ + { + "id": 165, + "name": "event_types", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[WebhookEventType]" + } + }, + { + "id": 166, + "name": "request_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 167, + "name": "payload_template", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 168, + "name": "ignore_ssl_errors", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 169, + "name": "do_not_retry", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 170, + "name": "idempotency_key", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 171, + "name": "set_status_message", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set the status message for the current actor run.\\n\", {\"Arguments\": [{\"param\": \"status_message\", \"type\": \"str\", \"desc\": \"The status message to set to the run.\"}, {\"param\": \"is_terminal\", \"type\": \"bool, optional\", \"desc\": \"Set this flag to True if this is the final status message of the Actor run.\\n\"}]}, {\"Returns\": [{\"param\": \"dict\", \"desc\": \"The updated actor run object\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 1249, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1249" + } + ], + "signatures": [ + { + "id": 172, + "name": "set_status_message", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set the status message for the current actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 173, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 174, + "name": "is_terminal", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 175, + "name": "create_proxy_configuration", + "module": "actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"Arguments\": [{\"param\": \"actor_proxy_input\", \"type\": \"dict, optional\", \"desc\": \"Proxy configuration field from the actor input, if actor has such input field.\\nIf you pass this argument, all the other arguments will be inferred from it.\"}, {\"param\": \"password\", \"type\": \"str, optional\", \"desc\": \"Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'], if available.\"}, {\"param\": \"groups\", \"type\": \"list of str, optional\", \"desc\": \"Proxy groups which the Apify Proxy should use, if provided.\"}, {\"param\": \"country_code\", \"type\": \"str, optional\", \"desc\": \"Country which the Apify Proxy should use, if provided.\"}, {\"param\": \"proxy_urls\", \"type\": \"list of str, optional\", \"desc\": \"Custom proxy server URLs which should be rotated through.\"}, {\"param\": \"new_url_function\", \"type\": \"Callable, optional\", \"desc\": \"Function which returns a custom proxy URL to be used.\\n\"}]}, {\"Returns\": [\"ProxyConfiguration, optional: ProxyConfiguration object with the passed configuration,\", \"or None, if no proxy should be used based on the configuration.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 1285, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L1285" + } + ], + "signatures": [ + { + "id": 176, + "name": "create_proxy_configuration", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options.\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ProxyConfiguration | None" + }, + "parameters": [ + { + "id": 177, + "name": "actor_proxy_input", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "defaultValue": "None" + }, + { + "id": 178, + "name": "password", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 179, + "name": "groups", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 180, + "name": "country_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 181, + "name": "proxy_urls", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 182, + "name": "new_url_function", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Callable[[str | None], str] | Callable[[str | None], Awaitable[str]] | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 42 + ] + }, + { + "title": "Properties", + "children": [ + 45, + 46, + 47, + 48 + ] + }, + { + "title": "Methods", + "children": [ + 126, + 163, + 131, + 142, + 175, + 53, + 59, + 113, + 92, + 51, + 94, + 49, + 111, + 64, + 152, + 67, + 107, + 103, + 74, + 79, + 84, + 89, + 159, + 171, + 98, + 115 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/actor.py", + "line": 68, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/actor.py#L68" + } + ] + }, + { + "id": 183, + "name": "Configuration", + "module": "config", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A class for specifying the configuration of an actor.\\n\\nCan be used either globally via `Configuration.get_global_configuration()`,\\nor it can be specific to each `Actor` instance on the `actor.config` property.\"]}" + } + ] + }, + "children": [ + { + "id": 184, + "name": "__init__", + "module": "config", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\", {\"Arguments\": [{\"param\": \"api_base_url\", \"type\": \"str, optional\", \"desc\": \"The URL of the Apify API.\\nThis is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.\"}, {\"param\": \"api_public_base_url\", \"type\": \"str, optional\", \"desc\": \"The public URL of the Apify API.\\nThis will always contain the public URL of the API, even when running in a container on the platform.\\nUseful for generating shareable URLs to key-value store records or datasets.\"}, {\"param\": \"container_port\", \"type\": \"int, optional\", \"desc\": \"The port on which the container can listen for HTTP requests.\"}, {\"param\": \"container_url\", \"type\": \"str, optional\", \"desc\": \"The URL on which the container can listen for HTTP requests.\"}, {\"param\": \"default_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default dataset for the actor.\"}, {\"param\": \"default_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default key-value store for the actor.\"}, {\"param\": \"default_request_queue_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the default request queue for the actor.\"}, {\"param\": \"input_key\", \"type\": \"str, optional\", \"desc\": \"The key of the input record in the actor's default key-value store\"}, {\"param\": \"max_used_cpu_ratio\", \"type\": \"float, optional\", \"desc\": \"The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.\"}, {\"param\": \"metamorph_after_sleep_millis\", \"type\": \"int, optional\", \"desc\": \"How long should the actor sleep after calling metamorph.\"}, {\"param\": \"persist_state_interval_millis\", \"type\": \"int, optional\", \"desc\": \"How often should the actor emit the PERSIST_STATE event.\"}, {\"param\": \"persist_storage\", \"type\": \"bool, optional\", \"desc\": \"Whether the actor should persist its used storages to the filesystem when running locally.\"}, {\"param\": \"proxy_hostname\", \"type\": \"str, optional\", \"desc\": \"The hostname of Apify Proxy.\"}, {\"param\": \"proxy_password\", \"type\": \"str, optional\", \"desc\": \"The password for Apify Proxy.\"}, {\"param\": \"proxy_port\", \"type\": \"str, optional\", \"desc\": \"The port of Apify Proxy.\"}, {\"param\": \"proxy_status_url\", \"type\": \"str, optional\", \"desc\": \"The URL on which the Apify Proxy status page is available.\"}, {\"param\": \"purge_on_start\", \"type\": \"str, optional\", \"desc\": \"Whether the actor should purge its default storages on startup, when running locally.\"}, {\"param\": \"token\", \"type\": \"str, optional\", \"desc\": \"The API token for the Apify API this actor should use.\"}, {\"param\": \"system_info_interval_millis\", \"type\": \"str, optional\", \"desc\": \"How often should the actor emit the SYSTEM_INFO event when running locally.\"}, {\"param\": \"standby_port\", \"type\": \"int, optional\", \"desc\": \"The port on which the container can listen for Actor Standby HTTP requests.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/config.py", + "line": 17, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L17" + } + ], + "signatures": [ + { + "id": 185, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a `Configuration` instance.\\n\\nAll the parameters are loaded by default from environment variables when running on the Apify platform.\\nYou can override them here in the Configuration constructor, which might be useful for local testing of your actors.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 186, + "name": "api_base_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 187, + "name": "api_public_base_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 188, + "name": "container_port", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 189, + "name": "container_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 190, + "name": "default_dataset_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 191, + "name": "default_key_value_store_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 192, + "name": "default_request_queue_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 193, + "name": "input_key", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 194, + "name": "max_used_cpu_ratio", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "float | None" + }, + "defaultValue": "None" + }, + { + "id": 195, + "name": "metamorph_after_sleep_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 196, + "name": "persist_state_interval_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 197, + "name": "persist_storage", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 198, + "name": "proxy_hostname", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 199, + "name": "proxy_password", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 200, + "name": "proxy_port", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 201, + "name": "proxy_status_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 202, + "name": "purge_on_start", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 203, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 204, + "name": "standby_port", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 205, + "name": "system_info_interval_millis", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 206, + "name": "get_global_configuration", + "module": "config", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/config.py", + "line": 124, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L124" + } + ], + "signatures": [ + { + "id": 207, + "name": "get_global_configuration", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the global configuration.\\n\\nThe global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.\\nAlso accessible via `Actor.config`.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Configuration", + "id": 183 + }, + "parameters": [] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 184 + ] + }, + { + "title": "Methods", + "children": [ + 206 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/config.py", + "line": 8, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/config.py#L8" + } + ] + }, + { + "id": 208, + "name": "StorageTypes", + "module": "consts", + "kind": 8, + "kindString": "Enumeration", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Possible Apify storage types.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/consts.py", + "line": 46, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/consts.py#L46" + } + ] + }, + { + "id": 209, + "name": "ActorLogFormatter", + "module": "log", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.\\n\\nIt formats the log records so that they:\\n- start with the level (colorized, and padded to 5 chars so that it is nicely aligned)\\n- then have the actual log message, if it's multiline then it's nicely indented\\n- then have the stringified extra log fields\\n- then, if an exception is a part of the log record, prints the formatted exception.\"]}" + } + ] + }, + "children": [ + { + "id": 210, + "name": "__init__", + "module": "log", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of the ActorLogFormatter.\\n\", {\"Arguments\": [{\"param\": \"include_logger_name\", \"desc\": \"Include logger name at the beginning of the log line. Defaults to False.\"}, {\"param\": \"args\", \"desc\": \"Arguments passed to the parent class.\"}, {\"param\": \"kwargs\", \"desc\": \"Keyword arguments passed to the parent class.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/log.py", + "line": 59, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/log.py#L59" + } + ], + "signatures": [ + { + "id": 211, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of the ActorLogFormatter.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 212, + "name": "include_logger_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 213, + "name": "args", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + }, + { + "id": 214, + "name": "kwargs", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 210 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/log.py", + "line": 42, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/log.py#L42" + } + ] + }, + { + "id": 215, + "name": "is_url", + "module": "proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the given string is a valid URL.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 25, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L25" + } + ], + "signatures": [ + { + "id": 216, + "name": "is_url", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the given string is a valid URL.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [ + { + "id": 217, + "name": "url", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 218, + "name": "ProxyInfo", + "module": "proxy_configuration", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Provides information about a proxy connection that is used for requests.\"]}" + } + ] + }, + "children": [ + { + "id": 219, + "name": "url", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The URL of the proxy.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 68, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L68" + } + ] + }, + { + "id": 220, + "name": "hostname", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The hostname of the proxy.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 71, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L71" + } + ] + }, + { + "id": 221, + "name": "port", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The proxy port.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 74, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L74" + } + ] + }, + { + "id": 222, + "name": "username", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The username for the proxy.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "NotRequired[str]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 77, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L77" + } + ] + }, + { + "id": 223, + "name": "password", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The password for the proxy.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 80, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L80" + } + ] + }, + { + "id": 224, + "name": "groups", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).\\nIf not provided, the proxy will select the groups automatically.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "NotRequired[list[str]]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 83, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L83" + } + ] + }, + { + "id": 225, + "name": "country_code", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"If set and relevant proxies are available in your Apify account, all proxied requests will\\nuse IP addresses that are geolocated to the specified country. For example `GB` for IPs\\nfrom Great Britain. Note that online services often have their own rules for handling\\ngeolocation and thus the country selection is a best attempt at geolocation, rather than\\na guaranteed hit. This parameter is optional, by default, each proxied request is assigned\\nan IP address from a random country. The country code needs to be a two letter ISO country code.\\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "NotRequired[str]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 88, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L88" + } + ] + }, + { + "id": 226, + "name": "session_id", + "module": "proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The identifier of the used proxy session, if used. Using the same session ID guarantees getting the same proxy URL.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "NotRequired[str]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 99, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L99" + } + ] + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 225, + 224, + 220, + 223, + 221, + 226, + 219, + 222 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 65, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L65" + } + ] + }, + { + "id": 227, + "name": "ProxyConfiguration", + "module": "proxy_configuration", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Configures a connection to a proxy server with the provided options.\\n\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or blacklists.\\nThe default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide no configuration option,\\nthe proxies will be managed automatically using a smart algorithm.\\n\\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options.\\nYour list of proxy URLs will be rotated by the configuration, if this option is provided.\"]}" + } + ] + }, + "children": [ + { + "id": 228, + "name": "initialize", + "module": "proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 195, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L195" + } + ], + "signatures": [ + { + "id": 229, + "name": "initialize", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Load the Apify Proxy password if the API token is provided and check access to Apify Proxy and provided proxy groups.\\n\\nOnly called if Apify Proxy configuration is used.\\nAlso checks if country has access to Apify Proxy groups if the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function\\nto create a pre-initialized `ProxyConfiguration` instance instead of calling this manually.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 230, + "name": "new_url", + "module": "proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"type\": \"int or str, optional\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier\\nwill use the same target proxy server (i.e. the same IP address).\\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\"}]}, {\"Returns\": [{\"param\": \"str\", \"desc\": \"A string with a proxy URL, including authentication credentials and port number.\\nFor example, `http://bob:password123@proxy.example.com:8000`\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 208, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L208" + } + ], + "signatures": [ + { + "id": 231, + "name": "new_url", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new proxy URL based on provided configuration options and the `sessionId` parameter.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 232, + "name": "session_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 233, + "name": "new_proxy_info", + "module": "proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"type\": \"int or str, optional\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier\\nwill use the same target proxy server (i.e. the same IP address).\\nThe identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\\n\"}]}, {\"Returns\": [{\"param\": \"ProxyInfo\", \"desc\": \"Dictionary that represents information about the proxy and its configuration.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 251, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L251" + } + ], + "signatures": [ + { + "id": 234, + "name": "new_proxy_info", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL.\\nIf you need the URL string only, use `ProxyConfiguration.new_url`.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ProxyInfo", + "id": 218 + }, + "parameters": [ + { + "id": 235, + "name": "session_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | str | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 228, + 233, + 230 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/proxy_configuration.py", + "line": 103, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/proxy_configuration.py#L103" + } + ] + }, + { + "id": 236, + "name": "ResourceClientType", + "module": "_memory_storage.resource_clients.base_resource_collection_client", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"noqa: PLC0105\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py", + "line": 17, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py#L17" + } + ] + }, + { + "id": 237, + "name": "ApifyHttpProxyMiddleware", + "module": "scrapy.middlewares.apify_proxy", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Apify HTTP proxy middleware for Scrapy.\\n\\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\\nprovided by the Actor input. An example of the proxy settings:\\n\\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}\"]}" + } + ] + }, + "children": [ + { + "id": 238, + "name": "__init__", + "module": "scrapy.middlewares.apify_proxy", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\\n\", {\"Arguments\": [{\"param\": \"proxy_settings\", \"desc\": \"Dictionary containing proxy settings, provided by the Actor input.\"}, {\"param\": \"auth_encoding\", \"desc\": \"Encoding for basic authentication (default is 'latin-1').\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 31, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L31" + } + ], + "signatures": [ + { + "id": 239, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 240, + "name": "proxy_settings", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + } + ] + } + ] + }, + { + "id": 241, + "name": "from_crawler", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"Arguments\": [{\"param\": \"cls\", \"desc\": \"Class type.\"}, {\"param\": \"crawler\", \"desc\": \"Scrapy Crawler object.\\n\"}]}, {\"Returns\": [{\"param\": \"ApifyHttpProxyMiddleware\", \"desc\": \"Instance of the class.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 42, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L42" + } + ], + "signatures": [ + { + "id": 242, + "name": "from_crawler", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyHttpProxyMiddleware", + "id": 237 + }, + "parameters": [ + { + "id": 243, + "name": "crawler", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Crawler" + } + } + ] + } + ] + }, + { + "id": 244, + "name": "process_request", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If username and password are not provided in the proxy URL.\\n\"}]}, {\"Returns\": [{\"param\": \"None\", \"desc\": \"The request is processed and middleware pipeline can continue.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 68, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L68" + } + ], + "signatures": [ + { + "id": 245, + "name": "process_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 246, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 247, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 248, + "name": "process_exception", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"exception\", \"desc\": \"Exception object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Returns\": [\"If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.\", \"Return None otherwise to allow the continuation of request processing.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 93, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L93" + } + ], + "signatures": [ + { + "id": 249, + "name": "process_exception", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None | Request" + }, + "parameters": [ + { + "id": 250, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 251, + "name": "exception", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Exception" + } + }, + { + "id": 252, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 238 + ] + }, + { + "title": "Methods", + "children": [ + 241, + 248, + 244 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 20, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/middlewares/apify_proxy.py#L20" + } + ] + }, + { + "id": 253, + "name": "ActorDatasetPushPipeline", + "module": "scrapy.pipelines.actor_dataset_push", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A Scrapy pipeline for pushing items to an Actor's default dataset.\\n\\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor.\"]}" + } + ] + }, + "children": [ + { + "id": 254, + "name": "process_item", + "module": "scrapy.pipelines.actor_dataset_push", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py", + "line": 21, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py#L21" + } + ], + "signatures": [ + { + "id": 255, + "name": "process_item", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Item" + }, + "parameters": [ + { + "id": 256, + "name": "item", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Item" + } + }, + { + "id": 257, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 254 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py", + "line": 15, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/pipelines/actor_dataset_push.py#L15" + } + ] + }, + { + "id": 258, + "name": "to_apify_request", + "module": "scrapy.requests", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"Arguments\": [{\"param\": \"scrapy_request\", \"desc\": \"The Scrapy request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Returns\": [\"The converted Apify request if the conversion was successful, otherwise None.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/requests.py", + "line": 28, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/requests.py#L28" + } + ], + "signatures": [ + { + "id": 259, + "name": "to_apify_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 260, + "name": "scrapy_request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 261, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 262, + "name": "to_scrapy_request", + "module": "scrapy.requests", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"Arguments\": [{\"param\": \"apify_request\", \"desc\": \"The Apify request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Raises\": [{\"param\": \"TypeError\", \"desc\": \"If the apify_request is not a dictionary.\"}, {\"param\": \"ValueError\", \"desc\": \"If the apify_request does not contain the required keys.\\n\"}]}, {\"Returns\": [\"The converted Scrapy request.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/requests.py", + "line": 96, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/requests.py#L96" + } + ], + "signatures": [ + { + "id": 263, + "name": "to_scrapy_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Request" + }, + "parameters": [ + { + "id": 264, + "name": "apify_request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + }, + { + "id": 265, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 266, + "name": "ApifyScheduler", + "module": "scrapy.scheduler", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A Scrapy scheduler that uses the Apify Request Queue to manage requests.\\n\\nThis scheduler requires the asyncio Twisted reactor to be installed.\"]}" + } + ] + }, + "children": [ + { + "id": 267, + "name": "__init__", + "module": "scrapy.scheduler", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 28, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L28" + } + ], + "signatures": [ + { + "id": 268, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 269, + "name": "open", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the scheduler.\\n\", {\"Arguments\": [{\"param\": \"spider\", \"desc\": \"The spider that the scheduler is associated with.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 39, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L39" + } + ], + "signatures": [ + { + "id": 270, + "name": "open", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the scheduler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 271, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 272, + "name": "has_pending_requests", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"Returns\": [\"True if the scheduler has any pending requests, False otherwise.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 53, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L53" + } + ], + "signatures": [ + { + "id": 273, + "name": "has_pending_requests", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 274, + "name": "enqueue_request", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request to add to the scheduler.\\n\"}]}, {\"Returns\": [\"True if the request was successfully enqueued, False otherwise.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 70, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L70" + } + ], + "signatures": [ + { + "id": 275, + "name": "enqueue_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [ + { + "id": 276, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + } + ] + } + ] + }, + { + "id": 277, + "name": "next_request", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"Returns\": [\"The next request, or None if there are no more requests.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 111, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L111" + } + ], + "signatures": [ + { + "id": 278, + "name": "next_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Request | None" + }, + "parameters": [] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 267 + ] + }, + { + "title": "Methods", + "children": [ + 274, + 272, + 277, + 269 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/scheduler.py", + "line": 22, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/scheduler.py#L22" + } + ] + }, + { + "id": 279, + "name": "get_basic_auth_header", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", + "line": 22, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L22" + } + ], + "signatures": [ + { + "id": 280, + "name": "get_basic_auth_header", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "bytes" + }, + "parameters": [ + { + "id": 281, + "name": "username", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 282, + "name": "password", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 283, + "name": "auth_encoding", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + }, + "defaultValue": "'latin-1'" + } + ] + } + ] + }, + { + "id": 284, + "name": "get_running_event_loop_id", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the ID of the currently running event loop.\\n\\nIt could be useful mainly for debugging purposes.\\n\", {\"Returns\": [\"The ID of the event loop.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", + "line": 29, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L29" + } + ], + "signatures": [ + { + "id": 285, + "name": "get_running_event_loop_id", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the ID of the currently running event loop.\\n\\nIt could be useful mainly for debugging purposes.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "parameters": [] + } + ] + }, + { + "id": 286, + "name": "apply_apify_settings", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"Arguments\": [{\"param\": \"settings\", \"desc\": \"Scrapy project settings to be modified.\"}, {\"param\": \"proxy_config\", \"desc\": \"Proxy configuration to be stored in the settings.\\n\"}]}, {\"Returns\": [\"Scrapy project settings with custom configurations.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", + "line": 40, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L40" + } + ], + "signatures": [ + { + "id": 287, + "name": "apply_apify_settings", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Settings" + }, + "parameters": [ + { + "id": 288, + "name": "settings", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Settings | None" + }, + "defaultValue": "None" + }, + { + "id": 289, + "name": "proxy_config", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 290, + "name": "open_queue_with_custom_client", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a Request Queue with custom Apify Client.\\n\\nTODO: add support for custom client to Actor.open_request_queue(), so that\\nwe don't have to do this hacky workaround\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/scrapy/utils.py", + "line": 76, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/scrapy/utils.py#L76" + } + ], + "signatures": [ + { + "id": 291, + "name": "open_queue_with_custom_client", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a Request Queue with custom Apify Client.\\n\\nTODO: add support for custom client to Actor.open_request_queue(), so that\\nwe don't have to do this hacky workaround\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "RequestQueue", + "id": 376 + }, + "parameters": [] + } + ] + }, + { + "id": 292, + "name": "Dataset", + "module": "storages.dataset", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The `Dataset` class represents a store for structured data where each object stored has the same attributes.\\n\\nYou can imagine it as a table, where each object is a row and its attributes are columns.\\nDataset is an append-only storage - you can only add new records to it but you cannot modify or remove existing records.\\nTypically it is used to store crawling results.\\n\\nDo not instantiate this class directly, use the `Actor.open_dataset()` function instead.\\n\\n`Dataset` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/datasets/{DATASET_ID}/{INDEX}.json\\n```\", \"Note that `{DATASET_ID}` is the name or ID of the dataset. The default dataset has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_DATASET_ID` environment variable.\\nEach dataset item is stored as a separate JSON file, where `{INDEX}` is a zero-based index of the item in the dataset.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Dataset](https://docs.apify.com/storage/dataset) cloud storage.\"]}" + } + ] + }, + "children": [ + { + "id": 293, + "name": "push_data", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\", {\"Arguments\": [{\"param\": \"data\", \"type\": \"JSONSerializable\", \"desc\": \"dict or array of dicts containing data to be stored in the default dataset.\\nThe JSON representation of each item must be smaller than 9MB.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 153, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L153" + } + ], + "signatures": [ + { + "id": 294, + "name": "push_data", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or an array of objects to the dataset.\\n\\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 295, + "name": "data", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "JSONSerializable" + } + } + ] + } + ] + }, + { + "id": 296, + "name": "get_data", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get items from the dataset.\\n\", {\"Arguments\": [{\"param\": \"offset\", \"type\": \"int, optional\", \"desc\": \"Number of items that should be skipped at the start. The default value is 0\"}, {\"param\": \"limit\", \"type\": \"int, optional\", \"desc\": \"Maximum number of items to return. By default there is no limit.\"}, {\"param\": \"desc\", \"type\": \"bool, optional\", \"desc\": \"By default, results are returned in the same order as they were stored.\\nTo reverse the order, set this parameter to True.\"}, {\"param\": \"clean\", \"type\": \"bool, optional\", \"desc\": \"If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.\"}, {\"param\": \"fields\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be picked from the items,\\nonly these fields will remain in the resulting record objects.\\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\nYou can use this feature to effectively fix the output format.\"}, {\"param\": \"omit\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be omitted from the items.\"}, {\"param\": \"unwind\", \"type\": \"str, optional\", \"desc\": \"Name of a field which should be unwound.\\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\\nIf the unwound field is an object then it is merged with the parent object.\\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\"}, {\"param\": \"skip_empty\", \"type\": \"bool, optional\", \"desc\": \"If True, then empty items are skipped from the output.\\nNote that if used, the results might contain less items than the limit value.\"}, {\"param\": \"skip_hidden\", \"type\": \"bool, optional\", \"desc\": \"If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\"}, {\"param\": \"flatten\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields that should be flattened\"}, {\"param\": \"view\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset view to be used\\n\"}]}, {\"Returns\": [{\"param\": \"ListPage\", \"desc\": \"A page of the list of dataset items according to the specified filters.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 182, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L182" + } + ], + "signatures": [ + { + "id": 297, + "name": "get_data", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get items from the dataset.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ListPage" + }, + "parameters": [ + { + "id": 298, + "name": "offset", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 299, + "name": "limit", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 300, + "name": "clean", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 301, + "name": "desc", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 302, + "name": "fields", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 303, + "name": "omit", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 304, + "name": "unwind", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 305, + "name": "skip_empty", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 306, + "name": "skip_hidden", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 307, + "name": "flatten", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 308, + "name": "view", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 309, + "name": "export_to", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"Either 'text/csv' or 'application/json'. Defaults to JSON.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 272, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L272" + } + ], + "signatures": [ + { + "id": 310, + "name": "export_to", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one file within a key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 311, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 312, + "name": "to_key_value_store_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 313, + "name": "to_key_value_store_name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 314, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 315, + "name": "export_to_json", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"from_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\"}, {\"param\": \"from_dataset_name\", \"type\": \"str, optional\", \"desc\": \"The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\nIf you omit both, it uses the default dataset.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 317, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L317" + } + ], + "signatures": [ + { + "id": 316, + "name": "export_to_json", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one JSON file within a key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 317, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 318, + "name": "from_dataset_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 319, + "name": "from_dataset_name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 320, + "name": "to_key_value_store_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 321, + "name": "to_key_value_store_name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 322, + "name": "export_to_csv", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key to save the data under.\"}, {\"param\": \"from_dataset_id\", \"type\": \"str, optional\", \"desc\": \"The ID of the dataset in case of calling the class method. Uses default dataset if omitted.\"}, {\"param\": \"from_dataset_name\", \"type\": \"str, optional\", \"desc\": \"The name of the dataset in case of calling the class method. Uses default dataset if omitted.\\nYou must specify only one of `from_dataset_id` and `from_dataset_name` arguments.\\nIf you omit both, it uses the default dataset.\"}, {\"param\": \"to_key_value_store_id\", \"type\": \"str, optional\", \"desc\": \"The id of the key-value store in which the result will be saved.\"}, {\"param\": \"to_key_value_store_name\", \"type\": \"str, optional\", \"desc\": \"The name of the key-value store in which the result will be saved.\\nYou must specify only one of `to_key_value_store_id` and `to_key_value_store_name` arguments.\\nIf you omit both, it uses the default key-value store.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 359, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L359" + } + ], + "signatures": [ + { + "id": 323, + "name": "export_to_csv", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Save the entirety of the dataset's contents into one CSV file within a key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 324, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 325, + "name": "from_dataset_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 326, + "name": "from_dataset_name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 327, + "name": "to_key_value_store_id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 328, + "name": "to_key_value_store_name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 329, + "name": "get_info", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\", {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Object returned by calling the GET dataset API endpoint.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 400, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L400" + } + ], + "signatures": [ + { + "id": 330, + "name": "get_info", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get an object containing general information about the dataset.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [] + } + ] + }, + { + "id": 331, + "name": "iterate_items", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\", {\"Arguments\": [{\"param\": \"offset\", \"type\": \"int, optional\", \"desc\": \"Number of items that should be skipped at the start. The default value is 0\"}, {\"param\": \"limit\", \"type\": \"int, optional\", \"desc\": \"Maximum number of items to return. By default there is no limit.\"}, {\"param\": \"desc\", \"type\": \"bool, optional\", \"desc\": \"By default, results are returned in the same order as they were stored.\\nTo reverse the order, set this parameter to True.\"}, {\"param\": \"clean\", \"type\": \"bool, optional\", \"desc\": \"If True, returns only non-empty items and skips hidden fields (i.e. fields starting with the # character).\\nThe clean parameter is just a shortcut for skip_hidden=True and skip_empty=True parameters.\\nNote that since some objects might be skipped from the output, that the result might contain less items than the limit value.\"}, {\"param\": \"fields\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be picked from the items,\\nonly these fields will remain in the resulting record objects.\\nNote that the fields in the outputted items are sorted the same way as they are specified in the fields parameter.\\nYou can use this feature to effectively fix the output format.\"}, {\"param\": \"omit\", \"type\": \"list of str, optional\", \"desc\": \"A list of fields which should be omitted from the items.\"}, {\"param\": \"unwind\", \"type\": \"str, optional\", \"desc\": \"Name of a field which should be unwound.\\nIf the field is an array then every element of the array will become a separate record and merged with parent object.\\nIf the unwound field is an object then it is merged with the parent object.\\nIf the unwound field is missing or its value is neither an array nor an object and therefore cannot be merged with a parent object,\\nthen the item gets preserved as it is. Note that the unwound items ignore the desc parameter.\"}, {\"param\": \"skip_empty\", \"type\": \"bool, optional\", \"desc\": \"If True, then empty items are skipped from the output.\\nNote that if used, the results might contain less items than the limit value.\"}, {\"param\": \"skip_hidden\", \"type\": \"bool, optional\", \"desc\": \"If True, then hidden fields are skipped from the output, i.e. fields starting with the # character.\\n\"}]}, {\"Yields\": [{\"param\": \"dict\", \"desc\": \"An item from the dataset\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 408, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L408" + } + ], + "signatures": [ + { + "id": 332, + "name": "iterate_items", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Iterate over the items in the dataset.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "AsyncIterator[dict]" + }, + "parameters": [ + { + "id": 333, + "name": "offset", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "0" + }, + { + "id": 334, + "name": "limit", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 335, + "name": "clean", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 336, + "name": "desc", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 337, + "name": "fields", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 338, + "name": "omit", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 339, + "name": "unwind", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 340, + "name": "skip_empty", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 341, + "name": "skip_hidden", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 342, + "name": "drop", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 460, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L460" + } + ], + "signatures": [ + { + "id": 343, + "name": "drop", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the dataset either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 344, + "name": "open", + "module": "storages.dataset", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\nIf the dataset with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the dataset to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default dataset associated with the actor run.\\nIf the dataset with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a dataset on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"Dataset\", \"desc\": \"An instance of the `Dataset` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 466, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L466" + } + ], + "signatures": [ + { + "id": 345, + "name": "open", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes,\\nsuch as online store products or real estate offers.\\nThe actual data is stored either on the local filesystem or in the Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Dataset", + "id": 292 + }, + "parameters": [ + { + "id": 346, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 347, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 348, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 349, + "name": "config", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 342, + 309, + 322, + 315, + 296, + 329, + 331, + 344, + 293 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/dataset.py", + "line": 73, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/dataset.py#L73" + } + ] + }, + { + "id": 350, + "name": "IterateKeysInfo", + "module": "storages.key_value_store", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Contains information about a key-value store record.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 22, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L22" + } + ] + }, + { + "id": 351, + "name": "IterateKeysTuple", + "module": "storages.key_value_store", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A tuple representing a key-value store record.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 28, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L28" + } + ] + }, + { + "id": 352, + "name": "KeyValueStore", + "module": "storages.key_value_store", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The `KeyValueStore` class represents a key-value store.\\n\\nYou can imagine it as a simple data storage that is used\\nfor saving and reading data records or files. Each data record is\\nrepresented by a unique key and associated with a MIME content type.\\n\\nDo not instantiate this class directly, use the `Actor.open_key_value_store()` function instead.\\n\\nEach crawler run is associated with a default key-value store, which is created exclusively\\nfor the run. By convention, the crawler input and output are stored into the\\ndefault key-value store under the `INPUT` and `OUTPUT` key, respectively.\\nTypically, input and output are JSON files, although it can be any other format.\\nTo access the default key-value store directly, you can use the\\n`KeyValueStore.get_value` and `KeyValueStore.set_value` convenience functions.\\n\\n`KeyValueStore` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}\\n```\", \"Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.\\nThe `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Key-value store](https://docs.apify.com/storage/key-value-store) cloud storage.\"]}" + } + ] + }, + "children": [ + { + "id": 353, + "name": "get_value", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"Key of the record to retrieve.\"}, {\"param\": \"default_value\", \"type\": \"Any, optional\", \"desc\": \"Default value returned in case the record does not exist.\\n\"}]}, {\"Returns\": [{\"param\": \"Any\", \"desc\": \"The value associated with the given key. `default_value` is used in case the record does not exist.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 137, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L137" + } + ], + "signatures": [ + { + "id": 354, + "name": "get_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "T | None" + }, + "parameters": [ + { + "id": 355, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 356, + "name": "default_value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "T | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 357, + "name": "iterate_keys", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\", {\"Arguments\": [{\"param\": \"exclusive_start_key\", \"type\": \"str, optional\", \"desc\": \"All keys up to this one (including) are skipped from the result.\\n\"}]}, {\"Yields\": [{\"param\": \"IterateKeysTuple\", \"desc\": \"A tuple `(key, info)`,\\nwhere `key` is the record key, and `info` is an object that contains a single property `size`\\nindicating size of the record in bytes.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 154, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L154" + } + ], + "signatures": [ + { + "id": 358, + "name": "iterate_keys", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Iterate over the keys in the key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "AsyncIterator[IterateKeysTuple]" + }, + "parameters": [ + { + "id": 359, + "name": "exclusive_start_key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 360, + "name": "set_value", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key under which the value should be saved.\"}, {\"param\": \"value\", \"type\": \"Any\", \"desc\": \"The value to save. If the value is `None`, the corresponding key-value pair will be deleted.\"}, {\"param\": \"content_type\", \"type\": \"str, optional\", \"desc\": \"The content type of the saved value.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 178, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L178" + } + ], + "signatures": [ + { + "id": 361, + "name": "set_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 362, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 363, + "name": "value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + }, + { + "id": 364, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 365, + "name": "get_public_url", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"type\": \"str\", \"desc\": \"The key for which the URL should be generated.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 206, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L206" + } + ], + "signatures": [ + { + "id": 366, + "name": "get_public_url", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 367, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 368, + "name": "drop", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 223, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L223" + } + ], + "signatures": [ + { + "id": 369, + "name": "drop", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the key-value store either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 370, + "name": "open", + "module": "storages.key_value_store", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\nIf the key-value store with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the key-value store to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default key-value store associated with the actor run.\\nIf the key-value store with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a key-value store on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"KeyValueStore\", \"desc\": \"An instance of the `KeyValueStore` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 229, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L229" + } + ], + "signatures": [ + { + "id": 371, + "name": "open", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type.\\nThe records are stored and retrieved using a unique key.\\nThe actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "KeyValueStore", + "id": 352 + }, + "parameters": [ + { + "id": 372, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 373, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 374, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 375, + "name": "config", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 368, + 365, + 353, + 357, + 370, + 360 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/key_value_store.py", + "line": 35, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/key_value_store.py#L35" + } + ] + }, + { + "id": 376, + "name": "RequestQueue", + "module": "storages.request_queue", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Represents a queue of URLs to crawl.\\n\\nCan be used for deep crawling of websites where you start with several URLs and then recursively\\nfollow links to other pages. The data structure supports both breadth-first and depth-first crawling orders.\\n\\nEach URL is represented using an instance of the {@apilink Request} class.\\nThe queue can only contain unique URLs. More precisely, it can only contain request dictionaries\\nwith distinct `uniqueKey` properties. By default, `uniqueKey` is generated from the URL, but it can also be overridden.\\nTo add a single URL multiple times to the queue,\\ncorresponding request dictionary will need to have different `uniqueKey` properties.\\n\\nDo not instantiate this class directly, use the `Actor.open_request_queue()` function instead.\\n\\n`RequestQueue` stores its data either on local disk or in the Apify cloud,\\ndepending on whether the `APIFY_LOCAL_STORAGE_DIR` or `APIFY_TOKEN` environment variables are set.\\n\\nIf the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the data is stored in\\nthe local directory in the following files:\\n```\\n{APIFY_LOCAL_STORAGE_DIR}/request_queues/{QUEUE_ID}/{REQUEST_ID}.json\\n```\", \"Note that `{QUEUE_ID}` is the name or ID of the request queue. The default request queue has ID: `default`,\\nunless you override it by setting the `APIFY_DEFAULT_REQUEST_QUEUE_ID` environment variable.\\nThe `{REQUEST_ID}` is the id of the request.\\n\\nIf the `APIFY_TOKEN` environment variable is set but `APIFY_LOCAL_STORAGE_DIR` is not, the data is stored in the\\n[Apify Request Queue](https://docs.apify.com/storage/request-queue)\\ncloud storage.\"]}" + } + ] + }, + "children": [ + { + "id": 377, + "name": "add_request", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\\n\\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\\nand payload, respectively, in its computation.\\n\\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\\nparameter. Information about the request's addition to the queue, including whether it was already present or\\nhandled, is returned in an output dictionary.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request object to be added to the queue. Must include at least the `url` key.\\nOptionaly it can include the `method`, `payload` and `uniqueKey` keys.\\n\"}, {\"param\": \"forefront\", \"desc\": \"If True, adds the request to the forefront of the queue; otherwise, adds it to the end.\\n\"}, {\"param\": \"keep_url_fragment\", \"desc\": \"Determines whether the URL fragment (the part of the URL after '#') should be retained\\nin the unique key computation.\\n\"}, {\"param\": \"use_extended_unique_key\", \"desc\": \"Determines whether to use an extended unique key, incorporating the request's\\nmethod and payload into the unique key computation.\\n\"}]}, \"Returns: A dictionary containing information about the operation, including:\\n- `requestId` (str): The ID of the request.\\n- `uniqueKey` (str): The unique key associated with the request.\\n- `wasAlreadyPresent` (bool): Indicates whether the request was already in the queue.\\n- `wasAlreadyHandled` (bool): Indicates whether the request was already processed.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 143, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L143" + } + ], + "signatures": [ + { + "id": 378, + "name": "add_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Adds a request to the `RequestQueue` while managing deduplication and positioning within the queue.\\n\\nThe deduplication of requests relies on the `uniqueKey` field within the request dictionary. If `uniqueKey`\\nexists, it remains unchanged; if it does not, it is generated based on the request's `url`, `method`,\\nand `payload` fields. The generation of `uniqueKey` can be influenced by the `keep_url_fragment` and\\n`use_extended_unique_key` flags, which dictate whether to include the URL fragment and the request's method\\nand payload, respectively, in its computation.\\n\\nThe request can be added to the forefront (beginning) or the back of the queue based on the `forefront`\\nparameter. Information about the request's addition to the queue, including whether it was already present or\\nhandled, is returned in an output dictionary.\\n\", {\"" + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "\\n- `requestId` (str)" + } + ] + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [ + { + "id": 379, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + }, + { + "id": 380, + "name": "forefront", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 381, + "name": "keep_url_fragment", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 382, + "name": "use_extended_unique_key", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 383, + "name": "get_request", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve a request from the queue.\\n\", {\"Arguments\": [{\"param\": \"request_id\", \"type\": \"str\", \"desc\": \"ID of the request to retrieve.\\n\"}]}, {\"Returns\": [\"dict, optional: The retrieved request, or `None`, if it does not exist.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 226, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L226" + } + ], + "signatures": [ + { + "id": 384, + "name": "get_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve a request from the queue.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 385, + "name": "request_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 386, + "name": "fetch_next_request", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\", {\"Returns\": [\"dict, optional: The request or `None` if there are no more pending requests.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 238, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L238" + } + ], + "signatures": [ + { + "id": 387, + "name": "fetch_next_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return the next request in the queue to be processed.\\n\\nOnce you successfully finish processing of the request, you need to call\\n`RequestQueue.mark_request_as_handled` to mark the request as handled in the queue.\\nIf there was some error in processing the request, call `RequestQueue.reclaim_request` instead,\\nso that the queue will give the request to some other consumer in another call to the `fetch_next_request` method.\\n\\nNote that the `None` return value does not mean the queue processing finished, it means there are currently no pending requests.\\nTo check whether all requests in queue were finished, use `RequestQueue.is_finished` instead.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [] + } + ] + }, + { + "id": 388, + "name": "mark_request_as_handled", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\", {\"Arguments\": [{\"param\": \"request\", \"type\": \"dict\", \"desc\": \"The request to mark as handled.\\n\"}]}, {\"Returns\": [\"dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\", \"`None` if the given request was not in progress.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 306, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L306" + } + ], + "signatures": [ + { + "id": 389, + "name": "mark_request_as_handled", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Mark a request as handled after successful processing.\\n\\nHandled requests will never again be returned by the `RequestQueue.fetch_next_request` method.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 390, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + } + ] + } + ] + }, + { + "id": 391, + "name": "reclaim_request", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\", {\"Arguments\": [{\"param\": \"request\", \"type\": \"dict\", \"desc\": \"The request to return to the queue.\"}, {\"param\": \"forefront\", \"type\": \"bool, optional\", \"desc\": \"Whether to add the request to the head or the end of the queue\"}]}, {\"Returns\": [\"dict, optional: Information about the queue operation with keys `requestId`, `uniqueKey`, `wasAlreadyPresent`, `wasAlreadyHandled`.\", \"`None` if the given request was not in progress.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 345, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L345" + } + ], + "signatures": [ + { + "id": 392, + "name": "reclaim_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Reclaim a failed request back to the queue.\\n\\nThe request will be returned for processing later again\\nby another call to `RequestQueue.fetchNextRequest`.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [ + { + "id": 393, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + }, + { + "id": 394, + "name": "forefront", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 395, + "name": "is_empty", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check whether the queue is empty.\\n\", {\"Returns\": [{\"param\": \"bool\", \"desc\": \"`True` if the next call to `RequestQueue.fetchNextRequest` would return `None`, otherwise `False`.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 401, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L401" + } + ], + "signatures": [ + { + "id": 396, + "name": "is_empty", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check whether the queue is empty.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 397, + "name": "is_finished", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\", {\"Returns\": [{\"param\": \"bool\", \"desc\": \"`True` if all requests were already handled and there are no more left. `False` otherwise.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 410, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L410" + } + ], + "signatures": [ + { + "id": 398, + "name": "is_finished", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check whether the queue is finished.\\n\\nDue to the nature of distributed storage used by the queue,\\nthe function might occasionally return a false negative,\\nbut it will never return a false positive.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 399, + "name": "drop", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 557, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L557" + } + ], + "signatures": [ + { + "id": 400, + "name": "drop", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove the request queue either from the Apify cloud storage or from the local directory.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 401, + "name": "get_info", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\", {\"Returns\": [{\"param\": \"dict\", \"desc\": \"Object returned by calling the GET request queue API endpoint.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 562, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L562" + } + ], + "signatures": [ + { + "id": 402, + "name": "get_info", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get an object containing general information about the request queue.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "parameters": [] + } + ] + }, + { + "id": 403, + "name": "open", + "module": "storages.request_queue", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"type\": \"str, optional\", \"desc\": \"ID of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\nIf the request queue with the given ID does not exist, it raises an error.\"}, {\"param\": \"name\", \"type\": \"str, optional\", \"desc\": \"Name of the request queue to be opened.\\nIf neither `id` nor `name` are provided, the method returns the default request queue associated with the actor run.\\nIf the request queue with the given name does not exist, it is created.\"}, {\"param\": \"force_cloud\", \"type\": \"bool, optional\", \"desc\": \"If set to True, it will open a request queue on the Apify Platform even when running the actor locally.\\nDefaults to False.\"}, {\"param\": \"config\", \"type\": \"Configuration, optional\", \"desc\": \"A `Configuration` instance, uses global configuration if omitted.\\n\"}]}, {\"Returns\": [{\"param\": \"RequestQueue\", \"desc\": \"An instance of the `RequestQueue` class for the given ID or name.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 571, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L571" + } + ], + "signatures": [ + { + "id": 404, + "name": "open", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in the Apify cloud.\\nThe queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first\\nand depth-first crawling orders.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "RequestQueue", + "id": 376 + }, + "parameters": [ + { + "id": 405, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 406, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 407, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + }, + { + "id": 408, + "name": "config", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 377, + 399, + 386, + 401, + 383, + 395, + 397, + 388, + 403, + 391 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v17/src/apify/storages/request_queue.py", + "line": 49, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v1.7.2/tmp-old-source-v17/src/apify/storages/request_queue.py#L49" + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 286, + 21, + 26, + 34, + 1, + 4, + 8, + 17, + 279, + 284, + 11, + 215, + 30, + 290, + 258, + 262, + 14 + ] + }, + { + "title": "Other", + "children": [ + 41, + 253, + 209, + 237, + 266, + 183, + 292, + 350, + 351, + 352, + 227, + 218, + 376 + ] + }, + { + "title": "Constants", + "children": [ + 208 + ] + }, + { + "title": "Properties", + "children": [ + 236 + ] + } + ], + "sources": [ + { + "fileName": "src/index.ts", + "line": 1, + "character": 0, + "url": "http://example.com/blob/123456/src/dummy.py" + } + ] +} \ No newline at end of file diff --git a/website/versioned_docs/version-2.7/.gitignore b/website/versioned_docs/version-2.7/.gitignore new file mode 100644 index 00000000..1a13c363 --- /dev/null +++ b/website/versioned_docs/version-2.7/.gitignore @@ -0,0 +1 @@ +changelog.md diff --git a/website/versioned_docs/version-2.7/01_introduction/code/01_introduction.py b/website/versioned_docs/version-2.7/01_introduction/code/01_introduction.py new file mode 100644 index 00000000..a3eaba25 --- /dev/null +++ b/website/versioned_docs/version-2.7/01_introduction/code/01_introduction.py @@ -0,0 +1,17 @@ +import httpx +from bs4 import BeautifulSoup + +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() + async with httpx.AsyncClient() as client: + response = await client.get(actor_input['url']) + soup = BeautifulSoup(response.content, 'html.parser') + data = { + 'url': actor_input['url'], + 'title': soup.title.string if soup.title else None, + } + await Actor.push_data(data) diff --git a/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/__init__.py b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/__main__.py b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/__main__.py new file mode 100644 index 00000000..8c4ab0b8 --- /dev/null +++ b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/__main__.py @@ -0,0 +1,6 @@ +import asyncio + +from .main import main + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/main.py b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/main.py new file mode 100644 index 00000000..1fb4b610 --- /dev/null +++ b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/main.py @@ -0,0 +1,8 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + await Actor.set_value('OUTPUT', 'Hello, world!') diff --git a/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/py.typed b/website/versioned_docs/version-2.7/01_introduction/code/actor_structure/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-2.7/01_introduction/index.mdx b/website/versioned_docs/version-2.7/01_introduction/index.mdx new file mode 100644 index 00000000..168370ab --- /dev/null +++ b/website/versioned_docs/version-2.7/01_introduction/index.mdx @@ -0,0 +1,45 @@ +--- +id: introduction +title: Overview +sidebar_label: Overview +slug: /overview +description: 'The official library for creating Apify Actors in Python, providing tools for web scraping, automation, and data storage integration.' +--- + +import CodeBlock from '@theme/CodeBlock'; + +import IntroductionExample from '!!raw-loader!./code/01_introduction.py'; + +The Apify SDK for Python is the official library for creating [Apify Actors](https://docs.apify.com/platform/actors) using Python. + + + {IntroductionExample} + + +## What are Actors? + +Actors are serverless cloud programs capable of performing tasks in a web browser, similar to what a human can do. These tasks can range from simple operations, such as filling out forms or unsubscribing from services, to complex jobs like scraping and processing large numbers of web pages. + +Actors can be executed locally or on the [Apify platform](https://docs.apify.com/platform/), which provides features for running them at scale, monitoring, scheduling, and even publishing and monetizing them. + +If you're new to Apify, refer to the Apify platform documentation to learn [what Apify is](https://docs.apify.com/platform/about). + +## Quick start + +To create and run Actors using Apify Console, check out [Apify Console documentation](https://docs.apify.com/platform/console). For creating and running Python Actors locally, refer to the [quick start guide](./quick-start). + +Explore the Guides section in the sidebar for a deeper understanding of the SDK's features and best practices. + +## Installation + +The Apify SDK for Python requires Python version 3.9 or above. It is typically installed when you create a new Actor project using the [Apify CLI](https://docs.apify.com/cli). To install it manually in an existing project, use: + +```bash +pip install apify +``` + +:::note API client alternative + +If you need to interact with the Apify API programmatically without creating Actors, use the [Apify API client for Python](https://docs.apify.com/api/client/python) instead. + +::: diff --git a/website/versioned_docs/version-2.7/01_introduction/quick-start.mdx b/website/versioned_docs/version-2.7/01_introduction/quick-start.mdx new file mode 100644 index 00000000..3ea5a5d0 --- /dev/null +++ b/website/versioned_docs/version-2.7/01_introduction/quick-start.mdx @@ -0,0 +1,124 @@ +--- +id: quick-start +title: Quick start +sidebar_label: Quick start +description: 'Get started with the Apify SDK for Python by creating your first Actor and learning the basics.' +--- + +Learn how to create and run Actors using the Apify SDK for Python. + +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import UnderscoreMainExample from '!!raw-loader!./code/actor_structure/main.py'; +import MainExample from '!!raw-loader!./code/actor_structure/__main__.py'; + +## Step 1: Create Actors + +To create and run Actors in Apify Console, refer to the [Console documentation](https://docs.apify.com/platform/console). + +To create a new Apify Actor on your computer, you can use the [Apify CLI](https://docs.apify.com/cli), and select one of the [Python Actor templates](https://apify.com/templates?category=python). + +For example, to create an Actor from the Python SDK template, you can use the [`apify create` command](https://docs.apify.com/cli/docs/reference#apify-create-actorname). + +```bash +apify create my-first-actor --template python-start +``` + +This will create a new folder called `my-first-actor`, download and extract the "Getting started with Python" Actor template there, create a virtual environment in `my-first-actor/.venv`, and install the Actor dependencies in it. + +## Step 2: Run Actors + +To run the Actor, you can use the [`apify run` command](https://docs.apify.com/cli/docs/reference#apify-run): + +```bash +cd my-first-actor +apify run +``` + +This command: + +- Activates the virtual environment in `.venv` (if no other virtual environment is activated yet) +- Starts the Actor with the appropriate environment variables for local running +- Configures it to use local storages from the `storage` folder + +The Actor input, for example, will be in `storage/key_value_stores/default/INPUT.json`. + +## Step 3: Understand Actor structure + +All Python Actor templates follow the same structure. + +The `.actor/` directory contains the [Actor configuration](https://docs.apify.com/platform/actors/development/actor-config), such as the Actor's definition and input schema, and the Dockerfile necessary to run the Actor on the Apify platform. + +The Actor's runtime dependencies are specified in the `requirements.txt` file, which follows the [standard requirements file format](https://pip.pypa.io/en/stable/reference/requirements-file-format/). + +The Actor's source code is in the `src/` folder. This folder contains two important files: + +- `main.py` - which contains the main function of the Actor +- `__main__.py` - which is the entrypoint of the Actor package, setting up the Actor [logger](../concepts/logging) and executing the Actor's main function via [`asyncio.run`](https://docs.python.org/3/library/asyncio-runner.html#asyncio.run). + + + + + {MainExample} + + + + + {UnderscoreMainExample} + + + + +If you want to modify the Actor structure, you need to make sure that your Actor is executable as a module, via `python -m src`, as that is the command started by `apify run` in the Apify CLI. We recommend keeping the entrypoint for the Actor in the `src/__main__.py` file. + +## Step 4: Add dependencies {#adding-dependencies} + +Adding dependencies into the Actor is simple. + +First, add them in the [`requirements.txt`](https://pip.pypa.io/en/stable/reference/requirements-file-format/) file in the Actor source folder. + +Then activate the virtual environment in `.venv`: + + + + { +`source .venv/bin/activate` + } + + + { +`.venv\\Scripts\\activate` + } + + + +Then install the dependencies: + +```bash +python -m pip install -r requirements.txt +``` + +## Next steps + +### Guides + +Integrate the Apify SDK with popular web scraping libraries by following these guides: + +- [BeautifulSoup with HTTPX](../guides/beautifulsoup-httpx) +- [Crawlee](../guides/crawlee) +- [Playwright](../guides/playwright) +- [Selenium](../guides/selenium) +- [Scrapy](../guides/scrapy) + +### Concepts + +For a deeper understanding of the Apify SDK's features, refer to the Concepts section in the sidebar. Key topics include: + +- [Actor lifecycle](../concepts/actor-lifecycle) +- [Working with storages](../concepts/storages) +- [Handling Actor events](../concepts/actor-events) +- [Using proxies](../concepts/proxy-management) diff --git a/website/versioned_docs/version-2.7/02_guides/01_beautifulsoup_httpx.mdx b/website/versioned_docs/version-2.7/02_guides/01_beautifulsoup_httpx.mdx new file mode 100644 index 00000000..4ecabd6e --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/01_beautifulsoup_httpx.mdx @@ -0,0 +1,30 @@ +--- +id: beautifulsoup-httpx +title: Using BeautifulSoup with HTTPX +--- + +import CodeBlock from '@theme/CodeBlock'; + +import BeautifulSoupHttpxExample from '!!raw-loader!./code/01_beautifulsoup_httpx.py'; + +In this guide, you'll learn how to use the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) library with the [HTTPX](https://www.python-httpx.org/) library in your Apify Actors. + +## Introduction + +`BeautifulSoup` is a Python library for extracting data from HTML and XML files. It provides simple methods and Pythonic idioms for navigating, searching, and modifying a website's element tree, enabling efficient data extraction. + +`HTTPX` is a modern, high-level HTTP client library for Python. It provides a simple interface for making HTTP requests and supports both synchronous and asynchronous requests. + +To create an `Actor` which uses those libraries, start from the [BeautifulSoup & Python](https://apify.com/templates/categories/python) Actor template. This template includes the `BeautifulSoup` and `HTTPX` libraries preinstalled, allowing you to begin development immediately. + +## Example Actor + +Below is a simple Actor that recursively scrapes titles from all linked websites, up to a specified maximum depth, starting from URLs provided in the Actor input. It uses `HTTPX` for fetching pages and `BeautifulSoup` for parsing their content to extract titles and links to other pages. + + + {BeautifulSoupHttpxExample} + + +## Conclusion + +In this guide, you learned how to use the `BeautifulSoup` with the `HTTPX` in your Apify Actors. By combining these libraries, you can efficiently extract data from HTML or XML files, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-2.7/02_guides/02_crawlee.mdx b/website/versioned_docs/version-2.7/02_guides/02_crawlee.mdx new file mode 100644 index 00000000..b040cad2 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/02_crawlee.mdx @@ -0,0 +1,37 @@ +--- +id: crawlee +title: Using Crawlee +--- + +import CodeBlock from '@theme/CodeBlock'; + +import CrawleeBeautifulSoupExample from '!!raw-loader!./code/02_crawlee_beautifulsoup.py'; +import CrawleePlaywrightExample from '!!raw-loader!./code/02_crawlee_playwright.py'; + +In this guide you'll learn how to use the [Crawlee](https://crawlee.dev/python) library in your Apify Actors. + +## Introduction + +`Crawlee` is a Python library for web scraping and browser automation that provides a robust and flexible framework for building web scraping tasks. It seamlessly integrates with the Apify platform and supports a variety of scraping techniques, from static HTML parsing to dynamic JavaScript-rendered content handling. Crawlee offers a range of crawlers, including HTTP-based crawlers like [`HttpCrawler`](https://crawlee.dev/python/api/class/HttpCrawler), [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) and [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler), and browser-based crawlers like [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler), to suit different scraping needs. + +In this guide, you'll learn how to use Crawlee with `BeautifulSoupCrawler` and `PlaywrightCrawler` to build Apify Actors for web scraping. + +## Actor with BeautifulSoupCrawler + +The `BeautifulSoupCrawler` is ideal for extracting data from static HTML pages. It uses `BeautifulSoup` for parsing and [`HttpxHttpClient`](https://crawlee.dev/python/api/class/HttpxHttpClient) for HTTP communication, ensuring efficient and lightweight scraping. If you do not need to execute JavaScript on the page, `BeautifulSoupCrawler` is a great choice for your scraping tasks. Below is an example of how to use `BeautifulSoupCrawler` in an Apify Actor. + + + {CrawleeBeautifulSoupExample} + + +## Actor with PlaywrightCrawler + +The `PlaywrightCrawler` is built for handling dynamic web pages that rely on JavaScript for content generation. Using the [Playwright](https://playwright.dev/) library, it provides a browser-based automation environment to interact with complex websites. Below is an example of how to use `PlaywrightCrawler` in an Apify Actor. + + + {CrawleePlaywrightExample} + + +## Conclusion + +In this guide, you learned how to use the `Crawlee` library in your Apify Actors. By using the `BeautifulSoupCrawler` and `PlaywrightCrawler` crawlers, you can efficiently scrape static or dynamic web pages, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-2.7/02_guides/03_playwright.mdx b/website/versioned_docs/version-2.7/02_guides/03_playwright.mdx new file mode 100644 index 00000000..8cada682 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/03_playwright.mdx @@ -0,0 +1,56 @@ +--- +id: playwright +title: Using Playwright +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import PlaywrightExample from '!!raw-loader!./code/03_playwright.py'; + +[Playwright](https://playwright.dev) is a tool for web automation and testing that can also be used for web scraping. It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Playwright for web scraping include: + +- **Cross-browser support** - Playwright supports the latest versions of major browsers like Chrome, Firefox, and Safari, so you can choose the one that suits your needs the best. +- **Headless mode** - Playwright can run in headless mode, meaning that the browser window is not visible on your screen while it is scraping, which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Playwright provides a variety of powerful selectors that allow you to target specific elements on a web page, including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Playwright allows you to emulate user interactions like clicking, scrolling, filling out forms, and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Playwright in Actors + +To create Actors which use Playwright, start from the [Playwright & Python](https://apify.com/templates/categories/python) Actor template. + +On the Apify platform, the Actor will already have Playwright and the necessary browsers preinstalled in its Docker image, including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to finish the Playwright setup yourself before you can run the Actor. + + + + { +`source .venv/bin/activate +playwright install --with-deps` + } + + + { +`.venv\\Scripts\\activate +playwright install --with-deps` + } + + + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, up to a maximum depth, starting from URLs in the Actor input. + +It uses Playwright to open the pages in an automated Chrome browser, and to extract the title and anchor elements after the pages load. + + + {PlaywrightExample} + + +## Conclusion + +In this guide you learned how to create Actors that use Playwright to scrape websites. Playwright is a powerful tool that can be used to manage browser instances and scrape websites that require JavaScript execution. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-2.7/02_guides/04_selenium.mdx b/website/versioned_docs/version-2.7/02_guides/04_selenium.mdx new file mode 100644 index 00000000..834dc33c --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/04_selenium.mdx @@ -0,0 +1,46 @@ +--- +id: selenium +title: Using Selenium +--- + +import CodeBlock from '@theme/CodeBlock'; + +import SeleniumExample from '!!raw-loader!./code/04_selenium.py'; + +[Selenium](https://www.selenium.dev/) is a tool for web automation and testing that can also be used for web scraping. It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Selenium for web scraping include: + +- **Cross-browser support** - Selenium supports the latest versions of major browsers like Chrome, Firefox, and Safari, +so you can choose the one that suits your needs the best. +- **Headless mode** - Selenium can run in headless mode, +meaning that the browser window is not visible on your screen while it is scraping, +which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Selenium provides a variety of powerful selectors that allow you to target specific elements on a web page, +including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Selenium allows you to emulate user interactions like clicking, scrolling, filling out forms, +and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Selenium in Actors + +To create Actors which use Selenium, start from the [Selenium & Python](https://apify.com/templates/categories/python) Actor template. + +On the Apify platform, the Actor will already have Selenium and the necessary browsers preinstalled in its Docker image, +including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to install the Selenium browser drivers yourself. +Refer to the [Selenium documentation](https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/) for installation instructions. + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, up to a maximum depth, starting from URLs in the Actor input. + +It uses Selenium ChromeDriver to open the pages in an automated Chrome browser, and to extract the title and anchor elements after the pages load. + + + {SeleniumExample} + + +## Conclusion + +In this guide you learned how to use Selenium for web scraping in Apify Actors. You can now create your own Actors that use Selenium to scrape dynamic websites and interact with web pages just like a human would. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-2.7/02_guides/05_scrapy.mdx b/website/versioned_docs/version-2.7/02_guides/05_scrapy.mdx new file mode 100644 index 00000000..95f34fae --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/05_scrapy.mdx @@ -0,0 +1,114 @@ +--- +id: scrapy +title: Using Scrapy +--- + +import CodeBlock from '@theme/CodeBlock'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import UnderscoreMainExample from '!!raw-loader!./code/scrapy_project/src/__main__.py'; +import MainExample from '!!raw-loader!./code/scrapy_project/src/main.py'; +import ItemsExample from '!!raw-loader!./code/scrapy_project/src/items.py'; +import SpidersExample from '!!raw-loader!./code/scrapy_project/src/spiders/title.py'; +import SettingsExample from '!!raw-loader!./code/scrapy_project/src/settings.py'; + +[Scrapy](https://scrapy.org/) is an open-source web scraping framework for Python. It provides tools for defining scrapers, extracting data from web pages, following links, and handling pagination. With the Apify SDK, Scrapy projects can be converted into Apify [Actors](https://docs.apify.com/platform/actors), integrated with Apify [storages](https://docs.apify.com/platform/storage), and executed on the Apify [platform](https://docs.apify.com/platform). + +## Integrating Scrapy with the Apify platform + +The Apify SDK provides an Apify-Scrapy integration. The main challenge of this is to combine two asynchronous frameworks that use different event loop implementations. Scrapy uses [Twisted](https://twisted.org/) for asynchronous execution, while the Apify SDK is based on [asyncio](https://docs.python.org/3/library/asyncio.html). The key thing is to install the Twisted's `asyncioreactor` to run Twisted's asyncio compatible event loop. This allows both Twisted and asyncio to run on a single event loop, enabling a Scrapy spider to run as an Apify Actor with minimal modifications. + + + {UnderscoreMainExample} + + +In this setup, `apify.scrapy.initialize_logging` configures an Apify log formatter and reconfigures loggers to ensure consistent logging across Scrapy, the Apify SDK, and other libraries. The `apify.scrapy.run_scrapy_actor` bridges asyncio coroutines with Twisted's reactor, enabling the Actor's main coroutine, which contains the Scrapy spider, to be executed. + +Make sure the `SCRAPY_SETTINGS_MODULE` environment variable is set to the path of the Scrapy settings module. This variable is also used by the `Actor` class to detect that the project is a Scrapy project, triggering additional actions. + + + {MainExample} + + +Within the Actor's main coroutine, the Actor's input is processed as usual. The function `apify.scrapy.apply_apify_settings` is then used to configure Scrapy settings with Apify-specific components before the spider is executed. The key components and other helper functions are described in the next section. + +## Key integration components + +The Apify SDK provides several custom components to support integration with the Apify platform: + +- [`apify.scrapy.ApifyScheduler`](https://docs.apify.com/sdk/python/reference/class/ApifyScheduler) - Replaces Scrapy's default [scheduler](https://docs.scrapy.org/en/latest/topics/scheduler.html) with one that uses Apify's [request queue](https://docs.apify.com/platform/storage/request-queue) for storing requests. It manages enqueuing, dequeuing, and maintaining the state and priority of requests. +- [`apify.scrapy.ActorDatasetPushPipeline`](https://docs.apify.com/sdk/python/reference/class/ActorDatasetPushPipeline) - A Scrapy [item pipeline](https://docs.scrapy.org/en/latest/topics/item-pipeline.html) that pushes scraped items to Apify's [dataset](https://docs.apify.com/platform/storage/dataset). When enabled, every item produced by the spider is sent to the dataset. +- [`apify.scrapy.ApifyHttpProxyMiddleware`](https://docs.apify.com/sdk/python/reference/class/ApifyHttpProxyMiddleware) - A Scrapy [middleware](https://docs.scrapy.org/en/latest/topics/downloader-middleware.html) that manages proxy configurations. This middleware replaces Scrapy's default `HttpProxyMiddleware` to facilitate the use of Apify's proxy service. +- [`apify.scrapy.extensions.ApifyCacheStorage`](https://docs.apify.com/sdk/python/reference/class/ApifyCacheStorage) - A storage backend for Scrapy's built-in [HTTP cache middleware](https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpcache). This backend uses Apify's [key-value store](https://docs.apify.com/platform/storage/key-value-store). Make sure to set `HTTPCACHE_ENABLED` and `HTTPCACHE_EXPIRATION_SECS` in your settings, or caching won't work. + +Additional helper functions in the [`apify.scrapy`](https://github.com/apify/apify-sdk-python/tree/master/src/apify/scrapy) subpackage include: + +- `apply_apify_settings` - Applies Apify-specific components to Scrapy settings. +- `to_apify_request` and `to_scrapy_request` - Convert between Apify and Scrapy request objects. +- `initialize_logging` - Configures logging for the Actor environment. +- `run_scrapy_actor` - Bridges asyncio and Twisted event loops. + +## Create a new Apify-Scrapy project + +The simplest way to start using Scrapy in Apify Actors is to use the [Scrapy Actor template](https://apify.com/templates/python-scrapy). The template provides a pre-configured project structure and setup that includes all necessary components to run Scrapy spiders as Actors and store their output in Apify datasets. If you prefer manual setup, refer to the example Actor section below for configuration details. + +## Wrapping an existing Scrapy project + +The Apify CLI supports converting an existing Scrapy project into an Apify Actor with a single command. The CLI expects the project to follow the standard Scrapy layout (including a `scrapy.cfg` file in the project root). During the wrapping process, the CLI: + +- Creates the necessary files and directories for an Apify Actor. +- Installs the Apify SDK and required dependencies. +- Updates Scrapy settings to include Apify-specific components. + +For further details, see the [Scrapy migration guide](https://docs.apify.com/cli/docs/integrating-scrapy). + +## Example Actor + +The following example demonstrates a Scrapy Actor that scrapes page titles and enqueues links found on each page. This example aligns with the structure provided in the Apify Actor templates. + + + + + {UnderscoreMainExample} + + + + + {MainExample} + + + + + {SettingsExample} + + + + + {ItemsExample} + + + + + {SpidersExample} + + + + +## Dealing with ‘imminent migration to another host’ + +Under some circumstances, the platform may decide to [migrate your Actor](https://docs.apify.com/academy/expert-scraping-with-apify/migrations-maintaining-state) from one piece of infrastructure to another while it's in progress. While [Crawlee](https://crawlee.dev/python)-based projects can pause and resume their work after a restart, achieving the same with a Scrapy-based project can be challenging. + +As a workaround for this issue (tracked as [apify/actor-templates#303](https://github.com/apify/actor-templates/issues/303)), turn on caching with `HTTPCACHE_ENABLED` and set `HTTPCACHE_EXPIRATION_SECS` to at least a few minutes—the exact value depends on your use case. If your Actor gets migrated and restarted, the subsequent run will hit the cache, making it fast and avoiding unnecessary resource consumption. + +## Conclusion + +In this guide you learned how to use Scrapy in Apify Actors. You can now start building your own web scraping projects using Scrapy, the Apify SDK and host them on the Apify platform. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! + +## Additional resources + +- [Apify CLI: Integrating Scrapy projects](https://docs.apify.com/cli/docs/integrating-scrapy) +- [Apify: Run Scrapy spiders on Apify](https://apify.com/run-scrapy-in-cloud) +- [Apify templates: Python Actor Scrapy template](https://apify.com/templates/python-scrapy) +- [Apify store: Scrapy Books Example Actor](https://apify.com/vdusek/scrapy-books-example) +- [Scrapy: Official documentation](https://docs.scrapy.org/) diff --git a/website/versioned_docs/version-2.7/02_guides/code/01_beautifulsoup_httpx.py b/website/versioned_docs/version-2.7/02_guides/code/01_beautifulsoup_httpx.py new file mode 100644 index 00000000..8cfaa606 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/01_beautifulsoup_httpx.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from urllib.parse import urljoin + +from bs4 import BeautifulSoup +from httpx import AsyncClient + +from apify import Actor, Request + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + # Create an HTTPX client to fetch the HTML content of the URLs. + async with AsyncClient() as client: + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Fetch the HTTP response from the specified URL using HTTPX. + response = await client.get(url, follow_redirects=True) + + # Parse the HTML content using Beautiful Soup. + soup = BeautifulSoup(response.content, 'html.parser') + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in soup.find_all('a'): + link_href = link.get('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': soup.title.string if soup.title else None, + 'h1s': [h1.text for h1 in soup.find_all('h1')], + 'h2s': [h2.text for h2 in soup.find_all('h2')], + 'h3s': [h3.text for h3 in soup.find_all('h3')], + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(new_request) diff --git a/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_beautifulsoup.py b/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_beautifulsoup.py new file mode 100644 index 00000000..e2dba8a1 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_beautifulsoup.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext + +from apify import Actor + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [ + url.get('url') + for url in actor_input.get( + 'start_urls', + [{'url': 'https://apify.com'}], + ) + ] + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Create a crawler. + crawler = BeautifulSoupCrawler( + # Limit the crawl to max requests. + # Remove or increase it for crawling all links. + max_requests_per_crawl=50, + ) + + # Define a request handler, which will be called for every request. + @crawler.router.default_handler + async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + url = context.request.url + Actor.log.info(f'Scraping {url}...') + + # Extract the desired data. + data = { + 'url': context.request.url, + 'title': context.soup.title.string if context.soup.title else None, + 'h1s': [h1.text for h1 in context.soup.find_all('h1')], + 'h2s': [h2.text for h2 in context.soup.find_all('h2')], + 'h3s': [h3.text for h3 in context.soup.find_all('h3')], + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional links found on the current page. + await context.enqueue_links() + + # Run the crawler with the starting requests. + await crawler.run(start_urls) diff --git a/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_playwright.py b/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_playwright.py new file mode 100644 index 00000000..2f0f110f --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/02_crawlee_playwright.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + +from apify import Actor + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [ + url.get('url') + for url in actor_input.get( + 'start_urls', + [{'url': 'https://apify.com'}], + ) + ] + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Create a crawler. + crawler = PlaywrightCrawler( + # Limit the crawl to max requests. + # Remove or increase it for crawling all links. + max_requests_per_crawl=50, + headless=True, + browser_launch_options={ + 'args': ['--disable-gpu'], + }, + ) + + # Define a request handler, which will be called for every request. + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + url = context.request.url + Actor.log.info(f'Scraping {url}...') + + # Extract the desired data. + data = { + 'url': context.request.url, + 'title': await context.page.title(), + 'h1s': [ + await h1.text_content() + for h1 in await context.page.locator('h1').all() + ], + 'h2s': [ + await h2.text_content() + for h2 in await context.page.locator('h2').all() + ], + 'h3s': [ + await h3.text_content() + for h3 in await context.page.locator('h3').all() + ], + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional links found on the current page. + await context.enqueue_links() + + # Run the crawler with the starting requests. + await crawler.run(start_urls) diff --git a/website/versioned_docs/version-2.7/02_guides/code/03_playwright.py b/website/versioned_docs/version-2.7/02_guides/code/03_playwright.py new file mode 100644 index 00000000..08a0cd97 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/03_playwright.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from urllib.parse import urljoin + +from playwright.async_api import async_playwright + +from apify import Actor, Request + +# Note: To run this Actor locally, ensure that Playwright browsers are installed. +# Run `playwright install --with-deps` in the Actor's virtual environment to install them. +# When running on the Apify platform, these dependencies are already included +# in the Actor's Docker image. + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + Actor.log.info('Launching Playwright...') + + # Launch Playwright and open a new browser context. + async with async_playwright() as playwright: + # Configure the browser to launch in headless mode as per Actor configuration. + browser = await playwright.chromium.launch( + headless=Actor.config.headless, + args=['--disable-gpu'], + ) + context = await browser.new_context() + + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Open a new page in the browser context and navigate to the URL. + page = await context.new_page() + await page.goto(url) + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in await page.locator('a').all(): + link_href = await link.get_attribute('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': await page.title(), + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + await page.close() + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(request) diff --git a/website/versioned_docs/version-2.7/02_guides/code/04_selenium.py b/website/versioned_docs/version-2.7/02_guides/code/04_selenium.py new file mode 100644 index 00000000..a043e4a1 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/04_selenium.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import asyncio +from urllib.parse import urljoin + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.common.by import By + +from apify import Actor, Request + +# To run this Actor locally, you need to have the Selenium Chromedriver installed. +# Follow the installation guide at: +# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/ +# When running on the Apify platform, the Chromedriver is already included +# in the Actor's Docker image. + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + # Launch a new Selenium Chrome WebDriver and configure it. + Actor.log.info('Launching Chrome WebDriver...') + chrome_options = ChromeOptions() + + if Actor.config.headless: + chrome_options.add_argument('--headless') + + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + driver = webdriver.Chrome(options=chrome_options) + + # Test WebDriver setup by navigating to an example page. + driver.get('http://www.example.com') + if driver.title != 'Example Domain': + raise ValueError('Failed to open example page.') + + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Navigate to the URL using Selenium WebDriver. Use asyncio.to_thread + # for non-blocking execution. + await asyncio.to_thread(driver.get, url) + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in driver.find_elements(By.TAG_NAME, 'a'): + link_href = link.get_attribute('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': driver.title, + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(request) + + driver.quit() diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/__init__.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/__main__.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/__main__.py new file mode 100644 index 00000000..bc2cf9ba --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/__main__.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from scrapy.utils.reactor import install_reactor + +# Install Twisted's asyncio reactor before importing any other Twisted or +# Scrapy components. +install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +import os + +from apify.scrapy import initialize_logging, run_scrapy_actor + +# Import your main Actor coroutine here. +from .main import main + +# Ensure the location to the Scrapy settings module is defined. +os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings' + + +if __name__ == '__main__': + initialize_logging() + run_scrapy_actor(main()) diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/items.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/items.py new file mode 100644 index 00000000..6579083f --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/items.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from scrapy import Field, Item + + +class TitleItem(Item): + """Represents a title item scraped from a web page.""" + + url = Field() + title = Field() diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/main.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/main.py new file mode 100644 index 00000000..a5586a25 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/main.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from scrapy.crawler import CrawlerRunner +from scrapy.utils.defer import deferred_to_future + +from apify import Actor +from apify.scrapy import apply_apify_settings + +# Import your Scrapy spider here. +from .spiders import TitleSpider as Spider + + +async def main() -> None: + """Apify Actor main coroutine for executing the Scrapy spider.""" + async with Actor: + # Retrieve and process Actor input. + actor_input = await Actor.get_input() or {} + start_urls = [url['url'] for url in actor_input.get('startUrls', [])] + allowed_domains = actor_input.get('allowedDomains') + proxy_config = actor_input.get('proxyConfiguration') + + # Apply Apify settings, which will override the Scrapy project settings. + settings = apply_apify_settings(proxy_config=proxy_config) + + # Create CrawlerRunner and execute the Scrapy spider. + crawler_runner = CrawlerRunner(settings) + crawl_deferred = crawler_runner.crawl( + Spider, + start_urls=start_urls, + allowed_domains=allowed_domains, + ) + await deferred_to_future(crawl_deferred) diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/py.typed b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/settings.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/settings.py new file mode 100644 index 00000000..5c0e56e3 --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/settings.py @@ -0,0 +1,11 @@ +BOT_NAME = 'titlebot' +DEPTH_LIMIT = 1 +LOG_LEVEL = 'INFO' +NEWSPIDER_MODULE = 'src.spiders' +ROBOTSTXT_OBEY = True +SPIDER_MODULES = ['src.spiders'] +TELNETCONSOLE_ENABLED = False +# Do not change the Twisted reactor unless you really know what you are doing. +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' +HTTPCACHE_ENABLED = True +HTTPCACHE_EXPIRATION_SECS = 7200 diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/__init__.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/__init__.py new file mode 100644 index 00000000..3745a12c --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/__init__.py @@ -0,0 +1,3 @@ +from .title import TitleSpider + +__all__ = ['TitleSpider'] diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/py.typed b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/title.py b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/title.py new file mode 100644 index 00000000..7223a53d --- /dev/null +++ b/website/versioned_docs/version-2.7/02_guides/code/scrapy_project/src/spiders/title.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +from scrapy import Request, Spider + +from ..items import TitleItem + +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy.http.response import Response + + +class TitleSpider(Spider): + """A spider that scrapes web pages to extract titles and discover new links. + + This spider retrieves the content of the element from each page and queues + any valid hyperlinks for further crawling. + """ + + name = 'title_spider' + + # Limit the number of pages to scrape. + custom_settings = {'CLOSESPIDER_PAGECOUNT': 10} + + def __init__( + self, + start_urls: list[str], + allowed_domains: list[str], + *args: Any, + **kwargs: Any, + ) -> None: + """A default constructor. + + Args: + start_urls: URLs to start the scraping from. + allowed_domains: Domains that the scraper is allowed to crawl. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + super().__init__(*args, **kwargs) + self.start_urls = start_urls + self.allowed_domains = allowed_domains + + def parse(self, response: Response) -> Generator[TitleItem | Request, None, None]: + """Parse the web page response. + + Args: + response: The web page response. + + Yields: + Yields scraped `TitleItem` and new `Request` objects for links. + """ + self.logger.info('TitleSpider is parsing %s...', response) + + # Extract and yield the TitleItem + url = response.url + title = response.css('title::text').extract_first() + yield TitleItem(url=url, title=title) + + # Extract all links from the page, create `Request` objects out of them, + # and yield them. + for link_href in response.css('a::attr("href")'): + link_url = urljoin(response.url, link_href.get()) + if link_url.startswith(('http://', 'https://')): + yield Request(link_url) diff --git a/website/versioned_docs/version-2.7/03_concepts/01_actor_lifecycle.mdx b/website/versioned_docs/version-2.7/03_concepts/01_actor_lifecycle.mdx new file mode 100644 index 00000000..38281046 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/01_actor_lifecycle.mdx @@ -0,0 +1,55 @@ +--- +id: actor-lifecycle +title: Actor lifecycle +--- + +import CodeBlock from '@theme/CodeBlock'; + +import InitExitExample from '!!raw-loader!./code/01_init_exit.py'; +import ContextManagerExample from '!!raw-loader!./code/01_context_manager.py'; +import RebootExample from '!!raw-loader!./code/01_reboot.py'; +import StatusMessageExample from '!!raw-loader!./code/01_status_message.py'; + +In this guide, we will show you how to manage the lifecycle of an Apify Actor. + +## Initialization and cleanup + +At the start of its runtime, the Actor needs to initialize itself, its event manager and its storages, and at the end of the runtime it needs to close these cleanly. The Apify SDK provides several options on how to manage this. + +The [`Actor.init`](../../reference/class/Actor#init) method initializes the Actor, the event manager which processes the Actor events from the platform event websocket, and the storage client used in the execution environment. It should be called before performing any other Actor operations. + +The [`Actor.exit`](../../reference/class/Actor#exit) method then exits the Actor cleanly, tearing down the event manager and the storage client. There is also the [`Actor.fail`](../../reference/class/Actor#fail) method, which exits the Actor while marking it as failed. + +<CodeBlock className="language-python"> + {InitExitExample} +</CodeBlock> + +### Context manager + +So that you don't have to call the lifecycle methods manually, the [`Actor`](../../reference/class/Actor) class provides a context manager, which calls the [`Actor.init`](../../reference/class/Actor#init) method on enter, the [`Actor.exit`](../../reference/class/Actor#exit) method on a clean exit, and the [`Actor.fail`](../../reference/class/Actor#fail) method when there is an exception during the run of the Actor. + +This is the recommended way to work with the `Actor` class. + +<CodeBlock className="language-python"> + {ContextManagerExample} +</CodeBlock> + +## Rebooting an Actor + +Sometimes, you want to restart your Actor to make it run from the beginning again. To do that, you can use the [`Actor.reboot`](../../reference/class/Actor#reboot) method. When you call it, the Apify platform stops the container of the run, and starts a new container of the same Actor with the same run ID and storages. + +Don't do it unconditionally, or you might get the Actor in a reboot loop. + +<CodeBlock className="language-python"> + {RebootExample} +</CodeBlock> + +## Actor status message + +To inform you or the users running your Actors about the progress of their runs, you can set the status message for the run, which will then be visible in the run detail in Apify Console, or accessible through the Apify API. + +To set the status message for the Actor run, you can use the [`Actor.set_status_message`](../../reference/class/Actor#set_status_message) method. + +<CodeBlock className="language-python"> + {StatusMessageExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/02_actor_input.mdx b/website/versioned_docs/version-2.7/03_concepts/02_actor_input.mdx new file mode 100644 index 00000000..ec68b849 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/02_actor_input.mdx @@ -0,0 +1,18 @@ +--- +id: actor-input +title: Actor input +--- + +import CodeBlock from '@theme/CodeBlock'; + +import InputExample from '!!raw-loader!./code/02_input.py'; + +The Actor gets its [input](https://docs.apify.com/platform/actors/running/input) from the input record in its default [key-value store](https://docs.apify.com/platform/storage/key-value-store). + +To access it, instead of reading the record manually, you can use the [`Actor.get_input`](../../reference/class/Actor#get_input) convenience method. It will get the input record key from the Actor configuration, read the record from the default key-value store,and decrypt any [secret input fields](https://docs.apify.com/platform/actors/development/secret-input). + +For example, if an Actor received a JSON input with two fields, `{ "firstNumber": 1, "secondNumber": 2 }`, this is how you might process it: + +<CodeBlock className="language-python"> + {InputExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/03_storages.mdx b/website/versioned_docs/version-2.7/03_concepts/03_storages.mdx new file mode 100644 index 00000000..a56a54c1 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/03_storages.mdx @@ -0,0 +1,173 @@ +--- +id: storages +title: Working with storages +--- + +import CodeBlock from '@theme/CodeBlock'; + +import OpeningStoragesExample from '!!raw-loader!./code/03_opening_storages.py'; +import DeletingStoragesExample from '!!raw-loader!./code/03_deleting_storages.py'; +import DatasetReadWriteExample from '!!raw-loader!./code/03_dataset_read_write.py'; +import DatasetExportsExample from '!!raw-loader!./code/03_dataset_exports.py'; +import KvsReadWriteExample from '!!raw-loader!./code/03_kvs_read_write.py'; +import KvsIteratingExample from '!!raw-loader!./code/03_kvs_iterating.py'; +import KvsPublicRecordExample from '!!raw-loader!./code/03_kvs_public_url.py'; +import RqExample from '!!raw-loader!./code/03_rq.py'; + +The `Actor` class provides methods to work either with the default storages of the Actor, or with any other storage, named or unnamed. + +## Types of storages + +There are three types of storages available to Actors. + +First are [datasets](https://docs.apify.com/platform/storage/dataset), which are append-only tables for storing the results of your Actors. You can open a dataset through the [`Actor.open_dataset`](../../reference/class/Actor#open_dataset) method, and work with it through the resulting [`Dataset`](../../reference/class/Dataset) class instance. + +Next there are [key-value stores](https://docs.apify.com/platform/storage/key-value-store), which function as a read/write storage for storing file-like objects, typically the Actor state or binary results. You can open a key-value store through the [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store) method, and work with it through the resulting [`KeyValueStore`](../../reference/class/KeyValueStore) class instance. + +Finally, there are [request queues](https://docs.apify.com/platform/storage/request-queue). These are queues into which you can put the URLs you want to scrape, and from which the Actor can dequeue them and process them. You can open a request queue through the [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) method, and work with it through the resulting [`RequestQueue`](../../reference/class/RequestQueue) class instance. + +Each Actor run has its default dataset, default key-value store and default request queue. + +## Local storage emulation + +To be able to develop Actors locally, the storages that the Apify platform provides are emulated on the local filesystem. + +The storage contents are loaded from and saved to the `storage` folder in the Actor's main folder. Each storage type is stored in its own subfolder, so for example datasets are stored in the `storage/datasets` folder. + +Each storage is then stored in its own folder, named after the storage, or called `default` if it's the default storage. For example, a request queue with the name `my-queue` would be stored in `storage/request_queues/my-queue`. + +Each dataset item, key-value store record, or request in a request queue is then stored in its own file in the storage folder. Dataset items and request queue requests are always JSON files, and key-value store records can be any file type, based on its content type. For example, the Actor input is typically stored in `storage/key_value_stores/default/INPUT.json`. + +## Local Actor run with remote storage + +When developing locally, opening any storage will by default use local storage. To change this behavior and to use remote storage you have to use `force_cloud=True` argument in [`Actor.open_dataset`](../../reference/class/Actor#open_dataset), [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) or [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store). Proper use of this argument allows you to work with both local and remote storages. + +Calling another remote Actor and accessing its default storage is typical use-case for using `force-cloud=True` argument to open remote Actor's storages. + +### Local storage persistence + +By default, the storage contents are persisted across multiple Actor runs. To clean up the Actor storages before the running the Actor, use the `--purge` flag of the [`apify run`](https://docs.apify.com/cli/docs/reference#apify-run) command of the Apify CLI. + +```bash +apify run --purge +``` + +## Convenience methods for working with default storages + +There are several methods for directly working with the default key-value store or default dataset of the Actor. + +- [`Actor.get_value('my-record')`](../../reference/class/Actor#get_value) reads a record from the default key-value store of the Actor. +- [`Actor.set_value('my-record', 'my-value')`](../../reference/class/Actor#set_value) saves a new value to the record in the default key-value store. +- [`Actor.get_input`](../../reference/class/Actor#get_input) reads the Actor input from the default key-value store of the Actor. +- [`Actor.push_data([{'result': 'Hello, world!'}, ...])`](../../reference/class/Actor#push_data) saves results to the default dataset of the Actor. + +## Opening named and unnamed storages + +The [`Actor.open_dataset`](../../reference/class/Actor#open_dataset), [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store) and [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) methods can be used to open any storage for reading and writing. You can either use them without arguments to open the default storages, or you can pass a storage ID or name to open another storage. + +<CodeBlock className="language-python"> + {OpeningStoragesExample} +</CodeBlock> + +## Deleting storages + +To delete a storage, you can use the [`Dataset.drop`](../../reference/class/Dataset#drop), +[`KeyValueStore.drop`](../../reference/class/KeyValueStore#drop) or [`RequestQueue.drop`](../../reference/class/RequestQueue#drop) methods. + +<CodeBlock className="language-python"> + {DeletingStoragesExample} +</CodeBlock> + +## Working with datasets + +In this section we will show you how to work with [datasets](https://docs.apify.com/platform/storage/dataset). + +### Reading & writing items + +To write data into a dataset, you can use the [`Dataset.push_data`](../../reference/class/Dataset#push_data) method. + +To read data from a dataset, you can use the [`Dataset.get_data`](../../reference/class/Dataset#get_data) method. + +To get an iterator of the data, you can use the [`Dataset.iterate_items`](../../reference/class/Dataset#iterate_items) method. + +<CodeBlock className="language-python"> + {DatasetReadWriteExample} +</CodeBlock> + +### Exporting items + +You can also export the dataset items into a key-value store, as either a CSV or a JSON record, +using the [`Dataset.export_to_csv`](../../reference/class/Dataset#export_to_csv) +or [`Dataset.export_to_json`](../../reference/class/Dataset#export_to_json) method. + +<CodeBlock className="language-python"> + {DatasetExportsExample} +</CodeBlock> + +## Working with key-value stores + +In this section we will show you how to work with [key-value stores](https://docs.apify.com/platform/storage/key-value-store). + +### Reading and writing records + +To read records from a key-value store, you can use the [`KeyValueStore.get_value`](../../reference/class/KeyValueStore#get_value) method. + +To write records into a key-value store, you can use the [`KeyValueStore.set_value`](../../reference/class/KeyValueStore#set_value) method. +You can set the content type of a record with the `content_type` argument. +To delete a record, set its value to `None`. + +<CodeBlock className="language-python"> + {KvsReadWriteExample} +</CodeBlock> + +### Iterating keys + +To get an iterator of the key-value store record keys, +you can use the [`KeyValueStore.iterate_keys`](../../reference/class/KeyValueStore#iterate_keys) method. + +<CodeBlock className="language-python"> + {KvsIteratingExample} +</CodeBlock> + +### Public URLs of records + +To get a publicly accessible URL of a key-value store record, +you can use the [`KeyValueStore.get_public_url`](../../reference/class/KeyValueStore#get_public_url) method. + +<CodeBlock className="language-python"> + {KvsPublicRecordExample} +</CodeBlock> + +## Working with request queues + +In this section we will show you how to work with [request queues](https://docs.apify.com/platform/storage/request-queue). + +### Adding requests to a queue + +To add a request into the queue, you can use the [`RequestQueue.add_request`](../../reference/class/RequestQueue#add_request) method. + +You can use the `forefront` boolean argument to specify whether the request should go to the beginning of the queue, or to the end. + +You can use the `unique_key` of the request to uniquely identify a request. If you try to add more requests with the same unique key, only the first one will be added. + +Check out the [`Request`](../../reference/class/Request) for more information on how to create requests and what properties they have. + +### Reading requests + +To fetch the next request from the queue for processing, you can use the [`RequestQueue.fetch_next_request`](../../reference/class/RequestQueue#fetch_next_request) method. + +To get info about a specific request from the queue, you can use the [`RequestQueue.get_request`](../../reference/class/RequestQueue#get_request) method. + +### Handling requests + +To mark a request as handled, you can use the [`RequestQueue.mark_request_as_handled`](../../reference/class/RequestQueue#mark_request_as_handled) method. + +To mark a request as not handled, so that it gets retried, you can use the [`RequestQueue.reclaim_request`](../../reference/class/RequestQueue#reclaim_request) method. + +To check if all the requests in the queue are handled, you can use the [`RequestQueue.is_finished`](../../reference/class/RequestQueue#is_finished) method. + +### Full example + +<CodeBlock className="language-python"> + {RqExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/04_actor_events.mdx b/website/versioned_docs/version-2.7/03_concepts/04_actor_events.mdx new file mode 100644 index 00000000..1be6cc63 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/04_actor_events.mdx @@ -0,0 +1,81 @@ +--- +id: actor-events +title: Actor events & state persistence +--- + +import CodeBlock from '@theme/CodeBlock'; + +import ActorEventsExample from '!!raw-loader!./code/04_actor_events.py'; + +During its runtime, the Actor receives Actor events sent by the Apify platform or generated by the Apify SDK itself. + +## Event types + +<table> + <thead> + <tr> + <th>Event</th> + <th>Data</th> + <th>Description</th> + </tr> + </thead> + <tbody> + <tr> + <td><code>SYSTEM_INFO</code></td> + <td><pre>{`{ + "created_at": datetime, + "cpu_current_usage": float, + "mem_current_bytes": int, + "is_cpu_overloaded": bool +}`} + </pre></td> + <td> + <p>This event is emitted regularly and it indicates the current resource usage of the Actor.</p> + The <code>is_cpu_overloaded</code> argument indicates whether the current CPU usage is higher than <code>Config.max_used_cpu_ratio</code> + </td> + </tr> + <tr> + <td><code>MIGRATING</code></td> + <td><code>None</code></td> + <td> + <p>Emitted when the Actor running on the Apify platform + is going to be <a href="https://docs.apify.com/platform/actors/development/state-persistence#what-is-a-migration">migrated</a> + {' '}to another worker server soon.</p> + You can use it to persist the state of the Actor so that once it is executed again on the new server, + it doesn't have to start over from the beginning. + Once you have persisted the state of your Actor, you can call <a href="../../reference/class/Actor#reboot"><code>Actor.reboot</code></a> + to reboot the Actor and trigger the migration immediately, to speed up the process. + </td> + </tr> + <tr> + <td><code>ABORTING</code></td> + <td><code>None</code></td> + <td> + When a user aborts an Actor run on the Apify platform, + they can choose to abort gracefully to allow the Actor some time before getting killed. + This graceful abort emits the <code>ABORTING</code> event which you can use to finish all running tasks and do cleanup. + </td> + </tr> + <tr> + <td><code>PERSIST_STATE</code></td> + <td><pre>{`{ "is_migrating": bool }`}</pre></td> + <td> + <p>Emitted in regular intervals (by default 60 seconds) to notify the Actor that it should persist its state, + in order to avoid repeating all work when the Actor restarts.</p> + <p>This event is also emitted automatically when the <code>MIGRATING</code> event happens, + in which case the <code>is_migrating</code> flag is set to <code>True</code>.</p> + Note that the <code>PERSIST_STATE</code> event is provided merely for user convenience, + you can achieve the same effect by persisting the state regularly in an interval and listening for the migrating event. + </td> + </tr> + </tbody> +</table> + +## Adding handlers to events + +To add handlers to these events, you use the [`Actor.on`](../../reference/class/Actor#on) method, +and to remove them, you use the [`Actor.off`](../../reference/class/Actor#off) method. + +<CodeBlock className="language-python"> + {ActorEventsExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/05_proxy_management.mdx b/website/versioned_docs/version-2.7/03_concepts/05_proxy_management.mdx new file mode 100644 index 00000000..1f15cfae --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/05_proxy_management.mdx @@ -0,0 +1,108 @@ +--- +id: proxy-management +title: Proxy management +--- + +import CodeBlock from '@theme/CodeBlock'; + +import ApifyProxyExample from '!!raw-loader!./code/05_apify_proxy.py'; +import CustomProxyExample from '!!raw-loader!./code/05_custom_proxy.py'; +import ProxyRotationExample from '!!raw-loader!./code/05_proxy_rotation.py'; +import ApifyProxyConfig from '!!raw-loader!./code/05_apify_proxy_config.py'; +import CustomProxyFunctionExample from '!!raw-loader!./code/05_custom_proxy_function.py'; +import ProxyActorInputExample from '!!raw-loader!./code/05_proxy_actor_input.py'; +import ProxyHttpxExample from '!!raw-loader!./code/05_proxy_httpx.py'; + +[IP address blocking](https://en.wikipedia.org/wiki/IP_address_blocking) is one of the oldest and most effective ways of preventing access to a website. It is therefore paramount for a good web scraping library to provide easy to use but powerful tools which can work around IP blocking. The most powerful weapon in your anti IP blocking arsenal is a [proxy server](https://en.wikipedia.org/wiki/Proxy_server). + +With the Apify SDK, you can use your own proxy servers, proxy servers acquired from third-party providers, or you can rely on [Apify Proxy](https://apify.com/proxy) for your scraping needs. + +## Quick start + +If you want to use Apify Proxy locally, make sure that you run your Actors via the Apify CLI and that you are [logged in](https://docs.apify.com/cli/docs/installation#login-with-your-apify-account) with your Apify account in the CLI. + +### Using Apify proxy + +<CodeBlock className="language-python"> + {ApifyProxyExample} +</CodeBlock> + +### Using your own proxies + +<CodeBlock className="language-python"> + {CustomProxyExample} +</CodeBlock> + +## Proxy configuration + +All your proxy needs are managed by the [`ProxyConfiguration`](../../reference/class/ProxyConfiguration) class. You create an instance using the [`Actor.create_proxy_configuration()`](../../reference/class/Actor#create_proxy_configuration) method. Then you generate proxy URLs using the [`ProxyConfiguration.new_url()`](../../reference/class/ProxyConfiguration#new_url) method. + +### Apify proxy vs. your own proxies + +The `ProxyConfiguration` class covers both Apify Proxy and custom proxy URLs, so that you can easily switch between proxy providers. However, some features of the class are available only to Apify Proxy users, mainly because Apify Proxy is what one would call a super-proxy. It's not a single proxy server, but an API endpoint that allows connectionthrough millions of different IP addresses. So the class essentially has two modes: Apify Proxy or Your proxy. + +The difference is easy to remember. Using the `proxy_url` or `new_url_function` arguments enables use of your custom proxy URLs, whereas all the other options are there to configure Apify Proxy. Visit the [Apify Proxy docs](https://docs.apify.com/proxy) for more info on how these parameters work. + +### IP rotation and session management + +`ProxyConfiguration.new_url` allows you to pass a `session_id` parameter. It will then be used to create a `session_id`-`proxy_url` pair, and subsequent `new_url()` calls with the same `session_id` will always return the same `proxy_url`. This is extremely useful in scraping, because you want to create the impression of a real user. + +When no `session_id` is provided, your custom proxy URLs are rotated round-robin, whereas Apify Proxy manages their rotation using black magic to get the best performance. + +<CodeBlock className="language-python"> + {ProxyRotationExample} +</CodeBlock> + +### Apify proxy configuration + +With Apify Proxy, you can select specific proxy groups to use, or countries to connect from. This allows you to get better proxy performance after some initial research. + +<CodeBlock className="language-python"> + {ApifyProxyConfig} +</CodeBlock> + +Now your connections using proxy_url will use only Residential proxies from the US. Note that you must first get access to a proxy group before you are able to use it. You can find your available proxy groups in the [proxy dashboard](https://console.apify.com/proxy). + +If you don't specify any proxy groups, automatic proxy selection will be used. + +### Your own proxy configuration + +There are two options how to make `ProxyConfiguration` work with your own proxies. + +Either you can pass it a list of your own proxy servers: + +<CodeBlock className="language-python"> + {CustomProxyExample} +</CodeBlock> + +Or you can pass it a method (accepting one optional argument, the session ID), to generate proxy URLs automatically: + +<CodeBlock className="language-python"> + {CustomProxyFunctionExample} +</CodeBlock> + +### Configuring proxy based on Actor input + +To make selecting the proxies that the Actor uses easier, you can use an input field with the editor [`proxy` in your input schema](https://docs.apify.com/platform/actors/development/input-schema#object). This input will then be filled with a dictionary containing the proxy settings you or the users of your Actor selected for the Actor run. + +You can then use that input to create the proxy configuration: + +<CodeBlock className="language-python"> + {ProxyActorInputExample} +</CodeBlock> + +## Using the generated proxy URLs + +### HTTPX + +To use the generated proxy URLs with the `httpx` library, use the [`proxies`](https://www.python-httpx.org/advanced/#http-proxying) argument: + +<CodeBlock className="language-python"> + {ProxyHttpxExample} +</CodeBlock> + +Make sure you have the `httpx` library installed: + +```bash +pip install httpx +``` diff --git a/website/versioned_docs/version-2.7/03_concepts/06_interacting_with_other_actors.mdx b/website/versioned_docs/version-2.7/03_concepts/06_interacting_with_other_actors.mdx new file mode 100644 index 00000000..d9b0b3d0 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/06_interacting_with_other_actors.mdx @@ -0,0 +1,51 @@ +--- +id: interacting-with-other-actors +title: Interacting with other Actors +--- + +import CodeBlock from '@theme/CodeBlock'; + +import InteractingStartExample from '!!raw-loader!./code/06_interacting_start.py'; +import InteractingCallExample from '!!raw-loader!./code/06_interacting_call.py'; +import InteractingCallTaskExample from '!!raw-loader!./code/06_interacting_call_task.py'; +import InteractingMetamorphExample from '!!raw-loader!./code/06_interacting_metamorph.py'; + +There are several methods that interact with other Actors and Actor tasks on the Apify platform. + +## Actor start + +The [`Actor.start`](../../reference/class/Actor#start) method starts another Actor on the Apify platform, and immediately returns the details of the started Actor run. + +<CodeBlock className="language-python"> + {InteractingStartExample} +</CodeBlock> + +## Actor call + +The [`Actor.call`](../../reference/class/Actor#call) method starts another Actor on the Apify platform, and waits for the started Actor run to finish. + +<CodeBlock className="language-python"> + {InteractingCallExample} +</CodeBlock> + +## Actor call task + +The [`Actor.call_task`](../../reference/class/Actor#call_task) method starts an [Actor task](https://docs.apify.com/platform/actors/tasks) on the Apify platform, and waits for the started Actor run to finish. + +<CodeBlock className="language-python"> + {InteractingCallTaskExample} +</CodeBlock> + +## Actor metamorph + +The [`Actor.metamorph`](../../reference/class/Actor#metamorph) operation transforms an Actor run into a run of another Actor with a new input. This feature is useful if you want to use another Actor to finish the work of your current Actor, instead of internally starting a new Actor run and waiting for its finish. With metamorph, you can easily create new Actors on top of existing ones, and give your users nicer input structure and user interface for the final Actor. For the users of your Actors, the metamorph operation is completely transparent; they will just see your Actor got the work done. + +Internally, the system stops the container corresponding to the original Actor run and starts a new container using a different container image. All the default storages are preserved,and the new Actor input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store. + +To make you Actor compatible with the metamorph operation, use [`Actor.get_input`](../../reference/class/Actor#get_input) instead of [`Actor.get_value('INPUT')`](../../reference/class/Actor#get_value) to read your Actor input. This method will fetch the input using the right key in a case of metamorphed run. + +For example, imagine you have an Actor that accepts a hotel URL on input, and then internally uses the [`apify/web-scraper`](https://apify.com/apify/web-scraper) public Actor to scrape all the hotel reviews. The metamorphing code would look as follows: + +<CodeBlock className="language-python"> + {InteractingMetamorphExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/07_webhooks.mdx b/website/versioned_docs/version-2.7/03_concepts/07_webhooks.mdx new file mode 100644 index 00000000..9dd11531 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/07_webhooks.mdx @@ -0,0 +1,31 @@ +--- +id: webhooks +title: Creating webhooks +--- + +import CodeBlock from '@theme/CodeBlock'; + +import WebhookExample from '!!raw-loader!./code/07_webhook.py'; +import WebhookPreventingExample from '!!raw-loader!./code/07_webhook_preventing.py'; + +Webhooks allow you to configure the Apify platform to perform an action when a certain event occurs. For example, you can use them to start another Actor when the current run finishes or fails. + +You can learn more in the [documentation for webhooks](https://docs.apify.com/platform/integrations/webhooks). + +## Creating an ad-hoc webhook dynamically + +Besides creating webhooks manually in Apify Console, or through the Apify API,you can also create [ad-hoc webhooks](https://docs.apify.com/platform/integrations/webhooks/ad-hoc-webhooks) dynamically from the code of your Actor using the [`Actor.add_webhook`](../../reference/class/Actor#add_webhook) method: + +<CodeBlock className="language-python"> + {WebhookExample} +</CodeBlock> + +Note that webhooks are only supported when running on the Apify platform. When running the Actor locally, the method will print a warning and have no effect. + +## Preventing duplicate webhooks + +To ensure that duplicate ad-hoc webhooks won't get created in a case of Actor restart, you can use the `idempotency_key` parameter. The idempotency key must be unique across all the webhooks of a user so that only one webhook gets created for a given value. You can use, for example, the Actor run ID as the idempotency key: + +<CodeBlock className="language-python"> + {WebhookPreventingExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/08_access_apify_api.mdx b/website/versioned_docs/version-2.7/03_concepts/08_access_apify_api.mdx new file mode 100644 index 00000000..d3fc05bf --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/08_access_apify_api.mdx @@ -0,0 +1,31 @@ +--- +id: access-apify-api +title: Accessing Apify API +--- + +import CodeBlock from '@theme/CodeBlock'; + +import ActorClientExample from '!!raw-loader!./code/08_actor_client.py'; +import ActorNewClientExample from '!!raw-loader!./code/08_actor_new_client.py'; + +The Apify SDK contains many useful features for making Actor development easier. However, it does not cover all the features the Apify API offers. + +For working with the Apify API directly, you can use the provided instance of the [Apify API Client](https://docs.apify.com/api/client/python) library. + +## Actor client + +To access the provided instance of [`ApifyClientAsync`](https://docs.apify.com/api/client/python/reference/class/ApifyClientAsync), you can use the [`Actor.apify_client`](../../reference/class/Actor#apify_client) property. + +For example, to get the details of your user, you can use this snippet: + +<CodeBlock className="language-python"> + {ActorClientExample} +</CodeBlock> + +## Actor new client + +If you want to create a completely new instance of the client, for example, to get a client for a different user or change the configuration of the client,you can use the [`Actor.new_client`](../../reference/class/Actor#new_client) method: + +<CodeBlock className="language-python"> + {ActorNewClientExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/09_running_webserver.mdx b/website/versioned_docs/version-2.7/03_concepts/09_running_webserver.mdx new file mode 100644 index 00000000..7d13a504 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/09_running_webserver.mdx @@ -0,0 +1,26 @@ +--- +id: running-webserver +title: Running webserver in your Actor +--- + +import CodeBlock from '@theme/CodeBlock'; + +import WebserverExample from '!!raw-loader!./code/09_webserver.py'; + +Each Actor run on the Apify platform is assigned a unique hard-to-guess URL (for example `https://8segt5i81sokzm.runs.apify.net`), which enables HTTP access to an optional web server running inside the Actor run's container. + +The URL is available in the following places: + +- In Apify Console, on the Actor run details page as the **Container URL** field. +- In the API as the `container_url` property of the [Run object](https://docs.apify.com/api/v2#/reference/actors/run-object/get-run). +- In the Actor as the `Actor.config.container_url` property. + +The web server running inside the container must listen at the port defined by the `Actor.config.container_port` property. When running Actors locally, the port defaults to `4321`, so the web server will be accessible at `http://localhost:4321`. + +## Example + +The following example demonstrates how to start a simple web server in your Actor,which will respond to every GET request with the number of items that the Actor has processed so far: + +<CodeBlock className="language-python"> + {WebserverExample} +</CodeBlock> diff --git a/website/versioned_docs/version-2.7/03_concepts/10_logging.mdx b/website/versioned_docs/version-2.7/03_concepts/10_logging.mdx new file mode 100644 index 00000000..69dea7dd --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/10_logging.mdx @@ -0,0 +1,84 @@ +--- +id: logging +title: Logging +--- + +import CodeBlock from '@theme/CodeBlock'; + +import LogConfigExample from '!!raw-loader!./code/10_log_config.py'; +import LoggerUsageExample from '!!raw-loader!./code/10_logger_usage.py'; + +The Apify SDK is logging useful information through the [`logging`](https://docs.python.org/3/library/logging.html) module from Python's standard library, into the logger with the name `apify`. + +## Automatic configuration + +When you create an Actor from an Apify-provided template, either in Apify Console or through the Apify CLI, you do not have to configure the logger yourself. The template already contains initialization code for the logger,which sets the logger level to `DEBUG` and the log formatter to [`ActorLogFormatter`](../../reference/class/ActorLogFormatter). + +## Manual configuration + +### Configuring the log level + +In Python's default behavior, if you don't configure the logger otherwise, only logs with level `WARNING` or higher are printed out to the standard output, without any formatting. To also have logs with `DEBUG` and `INFO` level printed out, you need to call the [`Logger.setLevel`](https://docs.python.org/3/library/logging.html#logging.Logger.setLevel) method on the logger, with the desired minimum level as an argument. + +### Configuring the log formatting + +By default, only the log message is printed out to the output, without any formatting. To have a nicer output, with the log level printed in color, the messages nicely aligned, and extra log fields printed out,you can use the [`ActorLogFormatter`](../../reference/class/ActorLogFormatter) class from the `apify.log` module. + +### Example log configuration + +To configure and test the logger, you can use this snippet: + +<CodeBlock className="language-python"> + {LogConfigExample} +</CodeBlock> + +This configuration will cause all levels of messages to be printed to the standard output, with some pretty formatting. + +## Logger usage + +Here you can see how all the log levels would look like. + +You can use the `extra` argument for all log levels, it's not specific to the warning level. When you use `Logger.exception`, there is no need to pass the Exception object to the log manually, it will automatiacally infer it from the current execution context and print the exception details. + +<CodeBlock className="language-python"> + {LoggerUsageExample} +</CodeBlock> + +Result: + +<!-- TODO: This is an ugly ugly hack, we should make a component for this in the docs theme --> +<!-- markdownlint-disable no-inline-html --> +<style>{` +.actor-log-block .ansi-blue-fg { + color: rgb(0, 160, 228); +} +.actor-log-block .ansi-green-fg { + color: rgb(0, 162, 82); +} +.actor-log-block .ansi-red-fg { + color: rgb(219, 45, 32); +} +.actor-log-block .ansi-yellow-fg { + color: rgb(255, 201, 0); +} +.actor-log-block .ansi-bright-black-fg { + color: rgb(128, 128, 128); +} +[data-theme='dark'] .actor-log-block .ansi-yellow-fg { + color: rgb(253, 237, 0); +} + +`}</style> +<pre className="actor-log-block"> + <div><span class="ansi-blue-fg">DEBUG</span> This is a debug message</div> + <div><span class="ansi-green-fg">INFO </span> This is an info message</div> + <div><span class="ansi-yellow-fg">WARN </span> This is a warning message <span class="ansi-bright-black-fg">{`({"reason": "Bad Actor!"})`}</span></div> + <div><span class="ansi-red-fg">ERROR</span> This is an error message</div> + <div><span class="ansi-red-fg">ERROR</span> This is an exceptional message</div> + <div> Traceback (most recent call last):</div> + <div> File "main.py", line 6, in <module></div> + <div> raise RuntimeError('Ouch!')</div> + <div> RuntimeError: Ouch!</div> +</pre> + +<!-- markdownlint-enable no-inline-html --> diff --git a/website/versioned_docs/version-2.7/03_concepts/11_configuration.mdx b/website/versioned_docs/version-2.7/03_concepts/11_configuration.mdx new file mode 100644 index 00000000..36ea66b5 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/11_configuration.mdx @@ -0,0 +1,34 @@ +--- +id: actor-configuration +title: Actor configuration +--- + +import CodeBlock from '@theme/CodeBlock'; + +import ConfigExample from '!!raw-loader!./code/11_config.py'; + +The [`Actor`](../../reference/class/Actor) class gets configured using the [`Configuration`](../../reference/class/Configuration) class, which initializes itself based on the provided environment variables. + +If you're using the Apify SDK in your Actors on the Apify platform, or Actors running locally through the Apify CLI, you don't need to configure the `Actor` class manually,unless you have some specific requirements, everything will get configured automatically. + +If you need some special configuration, you can adjust it either through the `Configuration` class directly,or by setting environment variables when running the Actor locally. + +To see the full list of configuration options, check the `Configuration` class or the list of environment variables that the Actor understands. + +## Configuring from code + +This will cause the Actor to persist its state every 10 seconds: + +<CodeBlock className="language-python"> + {ConfigExample} +</CodeBlock> + +## Configuring via environment variables + +All the configuration options can be set via environment variables. The environment variables are prefixed with `APIFY_`, and the configuration options are in uppercase, with underscores as separators. See the [`Configuration`](../../reference/class/Configuration) API reference for the full list of configuration options. + +This Actor run will not persist its local storages to the filesystem: + +```bash +APIFY_PERSIST_STORAGE=0 apify run +``` diff --git a/website/versioned_docs/version-2.7/03_concepts/12_pay_per_event.mdx b/website/versioned_docs/version-2.7/03_concepts/12_pay_per_event.mdx new file mode 100644 index 00000000..0d8edbbc --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/12_pay_per_event.mdx @@ -0,0 +1,46 @@ +--- +id: pay-per-event +title: Pay-per-event monetization +description: Monetize your Actors using the pay-per-event pricing model +--- + +import ActorChargeSource from '!!raw-loader!./code/actor_charge.py'; +import ConditionalActorChargeSource from '!!raw-loader!./code/conditional_actor_charge.py'; +import ApiLink from '@site/src/components/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; + +Apify provides several [pricing models](https://docs.apify.com/platform/actors/publishing/monetize) for monetizing your Actors. The most recent and most flexible one is [pay-per-event](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event), which lets you charge your users programmatically directly from your Actor. As the name suggests, you may charge the users each time a specific event occurs, for example a call to an external API or when you return a result. + +To use the pay-per-event pricing model, you first need to [set it up](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event) for your Actor in the Apify console. After that, you're free to start charging for events. + +## Charging for events + +After monetization is set in the Apify console, you can add <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> calls to your code and start monetizing! + +<CodeBlock language="python"> +{ActorChargeSource} +</CodeBlock> + +Then you just push your code to Apify and that's it! The SDK will even keep track of the max total charge setting for you, so you will not provide more value than what the user chose to pay for. + +If you need finer control over charging, you can access call <ApiLink to="class/Actor#get_charging_manager">`Actor.get_charging_manager()`</ApiLink> to access the <ApiLink to="class/ChargingManager">`ChargingManager`</ApiLink>, which can provide more detailed information - for example how many events of each type can be charged before reaching the configured limit. + +## Transitioning from a different pricing model + +When you plan to start using the pay-per-event pricing model for an Actor that is already monetized with a different pricing model, your source code will need support both pricing models during the transition period enforced by the Apify platform. Arguably the most frequent case is the transition from the pay-per-result model which utilizes the `ACTOR_MAX_PAID_DATASET_ITEMS` environment variable to prevent returning unpaid dataset items. The following is an example how to handle such scenarios. The key part is the <ApiLink to="class/ChargingManager#get_pricing_info">`ChargingManager.get_pricing_info()`</ApiLink> method which returns information about the current pricing model. + +<CodeBlock language="python"> +{ConditionalActorChargeSource} +</CodeBlock> + +## Local development + +It is encouraged to test your monetization code on your machine before releasing it to the public. To tell your Actor that it should work in pay-per-event mode, pass it the `ACTOR_TEST_PAY_PER_EVENT` environment variable: + +```shell +ACTOR_TEST_PAY_PER_EVENT=true python -m youractor +``` + +If you also wish to see a log of all the events charged throughout the run, the Apify SDK keeps a log of charged events in a so called charging dataset. Your charging dataset can be found under the `charging_log` name (unless you change your storage settings, this dataset is stored in `storage/datasets/charging_log/`). Please note that this log is not available when running the Actor in production on the Apify platform. + +Because pricing configuration is stored by the Apify platform, all events will have a default price of $1. diff --git a/website/versioned_docs/version-2.7/03_concepts/code/01_context_manager.py b/website/versioned_docs/version-2.7/03_concepts/code/01_context_manager.py new file mode 100644 index 00000000..8a3e8654 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/01_context_manager.py @@ -0,0 +1,9 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/01_init_exit.py b/website/versioned_docs/version-2.7/03_concepts/code/01_init_exit.py new file mode 100644 index 00000000..674c9285 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/01_init_exit.py @@ -0,0 +1,16 @@ +from apify import Actor + + +async def main() -> None: + await Actor.init() + + try: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') + + except Exception as exc: + Actor.log.exception('Error while running Actor') + await Actor.fail(exit_code=91, exception=exc) + + await Actor.exit() diff --git a/website/versioned_docs/version-2.7/03_concepts/code/01_reboot.py b/website/versioned_docs/version-2.7/03_concepts/code/01_reboot.py new file mode 100644 index 00000000..e398c5f4 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/01_reboot.py @@ -0,0 +1,7 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # ... your code here ... + await Actor.reboot() diff --git a/website/versioned_docs/version-2.7/03_concepts/code/01_status_message.py b/website/versioned_docs/version-2.7/03_concepts/code/01_status_message.py new file mode 100644 index 00000000..13bf2b34 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/01_status_message.py @@ -0,0 +1,14 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + await Actor.set_status_message('Here we go!') + # Do some work... + await Actor.set_status_message('So far so good...') + # Do some more work... + await Actor.set_status_message('Steady as she goes...') + # Do even more work... + await Actor.set_status_message('Almost there...') + # Finish the job + await Actor.set_status_message('Phew! That was not that hard!') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/02_input.py b/website/versioned_docs/version-2.7/03_concepts/code/02_input.py new file mode 100644 index 00000000..b3bd3034 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/02_input.py @@ -0,0 +1,9 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() or {} + first_number = actor_input.get('firstNumber', 0) + second_number = actor_input.get('secondNumber', 0) + Actor.log.info('Sum: %s', first_number + second_number) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_exports.py b/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_exports.py new file mode 100644 index 00000000..78f0f5b9 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_exports.py @@ -0,0 +1,31 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a dataset and write some data in it + dataset = await Actor.open_dataset(name='my-cool-dataset') + await dataset.push_data([{'itemNo': i} for i in range(1000)]) + + # Export the data as CSV + await dataset.export_to( + content_type='csv', + key='data.csv', + to_key_value_store_name='my-cool-key-value-store', + ) + + # Export the data as JSON + await dataset.export_to( + content_type='json', + key='data.json', + to_key_value_store_name='my-cool-key-value-store', + ) + + # Print the exported records + store = await Actor.open_key_value_store(name='my-cool-key-value-store') + + csv_data = await store.get_value('data.csv') + Actor.log.info(f'CSV data: {csv_data}') + + json_data = await store.get_value('data.json') + Actor.log.info(f'JSON data: {json_data}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_read_write.py b/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_read_write.py new file mode 100644 index 00000000..6d8ac7f0 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_dataset_read_write.py @@ -0,0 +1,16 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a dataset and write some data in it + dataset = await Actor.open_dataset(name='my-cool-dataset') + await dataset.push_data([{'itemNo': i} for i in range(1000)]) + + # Read back the first half of the data + first_half = await dataset.get_data(limit=500) + Actor.log.info(f'The first half of items = {first_half.items}') + + # Iterate over the second half + second_half = [item async for item in dataset.iterate_items(offset=500)] + Actor.log.info(f'The second half of items = {second_half}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_deleting_storages.py b/website/versioned_docs/version-2.7/03_concepts/code/03_deleting_storages.py new file mode 100644 index 00000000..68925bd9 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_deleting_storages.py @@ -0,0 +1,13 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a key-value store with the name 'my-cool-store' + key_value_store = await Actor.open_key_value_store(name='my-cool-store') + await key_value_store.set_value('record', 'Hello, world!') + + # Do something ... + + # Now we don't want it anymore + await key_value_store.drop() diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_iterating.py b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_iterating.py new file mode 100644 index 00000000..f5944095 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_iterating.py @@ -0,0 +1,18 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + kvs = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Write some data to it + await kvs.set_value('automatic_text', 'abcd') + await kvs.set_value('automatic_json', {'ab': 'cd'}) + await kvs.set_value('explicit_csv', 'a,b\nc,d', content_type='text/csv') + + # Print the info for each record + Actor.log.info('Records in store:') + + async for key, info in kvs.iterate_keys(): + Actor.log.info(f'key={key}, info={info}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_public_url.py b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_public_url.py new file mode 100644 index 00000000..fe8ae07a --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_public_url.py @@ -0,0 +1,11 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + store = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Get the public URL of a record + my_record_url = await store.get_public_url('my_record') + Actor.log.info(f'URL of "my_record": {my_record_url}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_read_write.py b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_read_write.py new file mode 100644 index 00000000..239aa2e2 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_kvs_read_write.py @@ -0,0 +1,25 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + kvs = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Write some data to it + await kvs.set_value('automatic_text', 'abcd') + await kvs.set_value('automatic_json', {'ab': 'cd'}) + await kvs.set_value('explicit_csv', 'a,b\nc,d', content_type='text/csv') + + # Get the values and log them + automatic_text = await kvs.get_value('automatic_text') + Actor.log.info(f'Automatic text: {automatic_text}') + + automatic_json = await kvs.get_value('automatic_json') + Actor.log.info(f'Automatic JSON: {automatic_json}') + + explicit_csv = await kvs.get_value('explicit_csv') + Actor.log.info(f'Explicit CSV: {explicit_csv}') + + # Delete the `automatic_text` value + await kvs.set_value('automatic_text', None) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_opening_storages.py b/website/versioned_docs/version-2.7/03_concepts/code/03_opening_storages.py new file mode 100644 index 00000000..b4ccbd09 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_opening_storages.py @@ -0,0 +1,16 @@ +from apify import Actor, Request + + +async def main() -> None: + async with Actor: + # Work with the default dataset of the Actor + dataset = await Actor.open_dataset() + await dataset.push_data({'result': 'Hello, world!'}) + + # Work with the key-value store with ID 'mIJVZsRQrDQf4rUAf' + key_value_store = await Actor.open_key_value_store(id='mIJVZsRQrDQf4rUAf') + await key_value_store.set_value('record', 'Hello, world!') + + # Work with the request queue with the name 'my-queue' + request_queue = await Actor.open_request_queue(name='my-queue') + await request_queue.add_request(Request.from_url('https://apify.com')) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/03_rq.py b/website/versioned_docs/version-2.7/03_concepts/code/03_rq.py new file mode 100644 index 00000000..fe1ea605 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/03_rq.py @@ -0,0 +1,53 @@ +import asyncio +import random + +from apify import Actor, Request + +FAILURE_RATE = 0.3 + + +async def main() -> None: + async with Actor: + # Open the queue + queue = await Actor.open_request_queue() + + # Add some requests to the queue + for i in range(1, 10): + await queue.add_request(Request.from_url(f'http://example.com/{i}')) + + # Add a request to the start of the queue, for priority processing + await queue.add_request(Request.from_url('http://example.com/0'), forefront=True) + + # If you try to add an existing request again, it will not do anything + add_request_info = await queue.add_request( + Request.from_url('http://different-example.com/5') + ) + Actor.log.info(f'Add request info: {add_request_info}') + + processed_request = await queue.get_request(add_request_info.id) + Actor.log.info(f'Processed request: {processed_request}') + + # Finally, process the queue until all requests are handled + while not await queue.is_finished(): + # Fetch the next unhandled request in the queue + request = await queue.fetch_next_request() + # This can happen due to the eventual consistency of the underlying request + # queue storage, best solution is just to sleep a bit. + if request is None: + await asyncio.sleep(1) + continue + + Actor.log.info(f'Processing request {request.unique_key}...') + Actor.log.info(f'Scraping URL {request.url}...') + + # Do some fake work, which fails 30% of the time + await asyncio.sleep(1) + if random.random() > FAILURE_RATE: + # If processing the request was successful, mark it as handled + Actor.log.info('Request successful.') + await queue.mark_request_as_handled(request) + else: + # If processing the request was unsuccessful, reclaim it so it can be + # processed again. + Actor.log.warning('Request failed, will retry!') + await queue.reclaim_request(request) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/04_actor_events.py b/website/versioned_docs/version-2.7/03_concepts/code/04_actor_events.py new file mode 100644 index 00000000..1c8c785d --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/04_actor_events.py @@ -0,0 +1,38 @@ +import asyncio +from typing import Any + +from apify import Actor, Event + + +async def main() -> None: + async with Actor: + total_items = 1000 + + # Load the state if it's saved from some previous execution + processed_items = 0 + actor_state = await Actor.get_value('STATE') + if actor_state is not None: + processed_items = actor_state + + # Save the state when the `PERSIST_STATE` event happens + async def save_state(event_data: Any) -> None: + nonlocal processed_items + Actor.log.info('Saving Actor state', extra=event_data) + await Actor.set_value('STATE', processed_items) + + Actor.on(Event.PERSIST_STATE, save_state) + + # Do some fake work + for i in range(processed_items, total_items): + Actor.log.info(f'Processing item {i}...') + processed_items = i + await asyncio.sleep(0.1) + + # Suppose we can stop saving the state now + Actor.off(Event.PERSIST_STATE, save_state) + + # Do some more fake work, this time something that can't be restarted, + # so no point persisting the state + for j in range(10): + Actor.log.info(f'Processing item {j} of another kind...') + await asyncio.sleep(1) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy.py b/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy.py new file mode 100644 index 00000000..96e7104c --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy.py @@ -0,0 +1,12 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration() + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy_config.py b/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy_config.py new file mode 100644 index 00000000..ba078b35 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_apify_proxy_config.py @@ -0,0 +1,15 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration( + groups=['RESIDENTIAL'], + country_code='US', + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() + Actor.log.info(f'Proxy URL: {proxy_url}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy.py b/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy.py new file mode 100644 index 00000000..d4c8a24a --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy.py @@ -0,0 +1,17 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy_function.py b/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy_function.py new file mode 100644 index 00000000..71aced2a --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_custom_proxy_function.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from apify import Actor, Request + + +async def custom_new_url_function( + session_id: str | None = None, + _: Request | None = None, +) -> str | None: + if session_id is not None: + return f'http://my-custom-proxy-supporting-sessions.com?session-id={session_id}' + return 'http://my-custom-proxy-not-supporting-sessions.com' + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration( + new_url_function=custom_new_url_function, # type: ignore[arg-type] + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url_with_session = await proxy_configuration.new_url('a') + Actor.log.info(f'Using proxy URL: {proxy_url_with_session}') + + proxy_url_without_session = await proxy_configuration.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url_without_session}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_actor_input.py b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_actor_input.py new file mode 100644 index 00000000..3ca0344d --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_actor_input.py @@ -0,0 +1,16 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() or {} + proxy_settings = actor_input.get('proxySettings') + proxy_configuration = await Actor.create_proxy_configuration( + actor_proxy_input=proxy_settings + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_httpx.py b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_httpx.py new file mode 100644 index 00000000..a124d1a5 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_httpx.py @@ -0,0 +1,22 @@ +import httpx + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() + + async with httpx.AsyncClient(proxy=proxy_url) as httpx_client: + response = await httpx_client.get('http://example.com') + Actor.log.info(f'Response: {response}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_rotation.py b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_rotation.py new file mode 100644 index 00000000..8e6a5de0 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/05_proxy_rotation.py @@ -0,0 +1,31 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_configuration: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_configuration.new_url() # http://proxy-1.com + proxy_url = await proxy_configuration.new_url() # http://proxy-2.com + proxy_url = await proxy_configuration.new_url() # http://proxy-1.com + proxy_url = await proxy_configuration.new_url() # http://proxy-2.com + proxy_url = await proxy_configuration.new_url( + session_id='a' + ) # http://proxy-1.com + proxy_url = await proxy_configuration.new_url( + session_id='b' + ) # http://proxy-2.com + proxy_url = await proxy_configuration.new_url( + session_id='b' + ) # http://proxy-2.com + proxy_url = await proxy_configuration.new_url( + session_id='a' + ) # http://proxy-1.com diff --git a/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call.py b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call.py new file mode 100644 index 00000000..46a0a90a --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call.py @@ -0,0 +1,22 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start the apify/screenshot-url Actor. + actor_run = await Actor.call( + actor_id='apify/screenshot-url', + run_input={'url': 'http://example.com', 'delay': 10000}, + ) + + if actor_run is None: + raise RuntimeError('Actor task failed to start.') + + # Wait for the Actor run to finish. + run_client = Actor.apify_client.run(actor_run.id) + await run_client.wait_for_finish() + + # Get the Actor output from the key-value store. + kvs_client = run_client.key_value_store() + output = await kvs_client.get_record('OUTPUT') + Actor.log.info(f'Actor output: {output}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call_task.py b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call_task.py new file mode 100644 index 00000000..75335d69 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_call_task.py @@ -0,0 +1,19 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start the Actor task by its ID. + actor_run = await Actor.call_task(task_id='Z3m6FPSj0GYZ25rQc') + + if actor_run is None: + raise RuntimeError('Actor task failed to start.') + + # Wait for the task run to finish. + run_client = Actor.apify_client.run(actor_run.id) + await run_client.wait_for_finish() + + # Get the task run dataset items + dataset_client = run_client.dataset() + items = await dataset_client.list_items() + Actor.log.info(f'Task run dataset items: {items}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_metamorph.py b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_metamorph.py new file mode 100644 index 00000000..53d48882 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_metamorph.py @@ -0,0 +1,24 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Get the original Actor input. + actor_input = await Actor.get_input() or {} + hotel_url = actor_input.get('hotel_url') + + # Create new input for apify/web-scraper Actor. + web_scraper_input = { + 'startUrls': [{'url': hotel_url}], + 'pageFunction': """async function pageFunction(context) { + // Here you pass the JavaScript page function + // that scrapes all the reviews from the hotel's URL + }""", + } + + # Metamorph the Actor run to `apify/web-scraper` with the new input. + await Actor.metamorph('apify/web-scraper', web_scraper_input) + + # This code will not be called, since the `metamorph` action terminates + # the current Actor run container. + Actor.log.info('You will not see this!') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_start.py b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_start.py new file mode 100644 index 00000000..075347c2 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/06_interacting_start.py @@ -0,0 +1,13 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start your own Actor named 'my-fancy-actor'. + actor_run = await Actor.start( + actor_id='~my-fancy-actor', + run_input={'foo': 'bar'}, + ) + + # Log the Actor run ID. + Actor.log.info(f'Actor run ID: {actor_run.id}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/07_webhook.py b/website/versioned_docs/version-2.7/03_concepts/code/07_webhook.py new file mode 100644 index 00000000..c2e382cf --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/07_webhook.py @@ -0,0 +1,16 @@ +from apify import Actor, Webhook + + +async def main() -> None: + async with Actor: + # Create a webhook that will be triggered when the Actor run fails. + webhook = Webhook( + event_types=['ACTOR.RUN.FAILED'], + request_url='https://example.com/run-failed', + ) + + # Add the webhook to the Actor. + await Actor.add_webhook(webhook) + + # Raise an error to simulate a failed run. + raise RuntimeError('I am an error and I know it!') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/07_webhook_preventing.py b/website/versioned_docs/version-2.7/03_concepts/code/07_webhook_preventing.py new file mode 100644 index 00000000..988c531c --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/07_webhook_preventing.py @@ -0,0 +1,17 @@ +from apify import Actor, Webhook + + +async def main() -> None: + async with Actor: + # Create a webhook that will be triggered when the Actor run fails. + webhook = Webhook( + event_types=['ACTOR.RUN.FAILED'], + request_url='https://example.com/run-failed', + idempotency_key=Actor.config.actor_run_id, + ) + + # Add the webhook to the Actor. + await Actor.add_webhook(webhook) + + # Raise an error to simulate a failed run. + raise RuntimeError('I am an error and I know it!') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/08_actor_client.py b/website/versioned_docs/version-2.7/03_concepts/code/08_actor_client.py new file mode 100644 index 00000000..68f5c2d7 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/08_actor_client.py @@ -0,0 +1,11 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Create a new user client. + user_client = Actor.apify_client.user('me') + + # Get information about the current user. + me = await user_client.get() + Actor.log.info(f'User: {me}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/08_actor_new_client.py b/website/versioned_docs/version-2.7/03_concepts/code/08_actor_new_client.py new file mode 100644 index 00000000..da59e6fc --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/08_actor_new_client.py @@ -0,0 +1,14 @@ +from apify import Actor + +TOKEN = 'ANOTHER_USERS_TOKEN' + + +async def main() -> None: + async with Actor: + # Create a new user client with a custom token. + apify_client = Actor.new_client(token=TOKEN, max_retries=2) + user_client = apify_client.user('me') + + # Get information about the another user. + them = await user_client.get() + Actor.log.info(f'Another user: {them}') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/09_webserver.py b/website/versioned_docs/version-2.7/03_concepts/code/09_webserver.py new file mode 100644 index 00000000..48a5c10d --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/09_webserver.py @@ -0,0 +1,49 @@ +import asyncio +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from apify import Actor + +processed_items = 0 +http_server = None + + +# Just a simple handler that will print the number of processed items so far +# on every GET request. +class RequestHandler(BaseHTTPRequestHandler): + def do_get(self) -> None: + self.log_request() + self.send_response(200) + self.end_headers() + self.wfile.write(bytes(f'Processed items: {processed_items}', encoding='utf-8')) + + +def run_server() -> None: + # Start the HTTP server on the provided port, + # and save a reference to the server. + global http_server + with ThreadingHTTPServer( + ('', Actor.config.web_server_port), RequestHandler + ) as server: + Actor.log.info(f'Server running on {Actor.config.web_server_port}') + http_server = server + server.serve_forever() + + +async def main() -> None: + global processed_items + async with Actor: + # Start the HTTP server in a separate thread. + run_server_task = asyncio.get_running_loop().run_in_executor(None, run_server) + + # Simulate doing some work. + for _ in range(100): + await asyncio.sleep(1) + processed_items += 1 + Actor.log.info(f'Processed items: {processed_items}') + + if http_server is None: + raise RuntimeError('HTTP server not started') + + # Signal the HTTP server to shut down, and wait for it to finish. + http_server.shutdown() + await run_server_task diff --git a/website/versioned_docs/version-2.7/03_concepts/code/10_log_config.py b/website/versioned_docs/version-2.7/03_concepts/code/10_log_config.py new file mode 100644 index 00000000..520df753 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/10_log_config.py @@ -0,0 +1,12 @@ +import logging + +from apify.log import ActorLogFormatter + + +async def main() -> None: + handler = logging.StreamHandler() + handler.setFormatter(ActorLogFormatter()) + + apify_logger = logging.getLogger('apify') + apify_logger.setLevel(logging.DEBUG) + apify_logger.addHandler(handler) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/10_logger_usage.py b/website/versioned_docs/version-2.7/03_concepts/code/10_logger_usage.py new file mode 100644 index 00000000..a707ab5c --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/10_logger_usage.py @@ -0,0 +1,23 @@ +import logging + +from apify import Actor +from apify.log import ActorLogFormatter + + +async def main() -> None: + handler = logging.StreamHandler() + handler.setFormatter(ActorLogFormatter()) + + apify_logger = logging.getLogger('apify') + apify_logger.setLevel(logging.DEBUG) + apify_logger.addHandler(handler) + + async with Actor: + Actor.log.debug('This is a debug message') + Actor.log.info('This is an info message') + Actor.log.warning('This is a warning message', extra={'reason': 'Bad Actor!'}) + Actor.log.error('This is an error message') + try: + raise RuntimeError('Ouch!') + except RuntimeError: + Actor.log.exception('This is an exceptional message') diff --git a/website/versioned_docs/version-2.7/03_concepts/code/11_config.py b/website/versioned_docs/version-2.7/03_concepts/code/11_config.py new file mode 100644 index 00000000..10b07079 --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/11_config.py @@ -0,0 +1,16 @@ +from datetime import timedelta + +from apify import Actor, Configuration, Event + + +async def main() -> None: + global_config = Configuration.get_global_configuration() + global_config.persist_state_interval = timedelta(seconds=10) + + async with Actor: + # Define a handler that will be called for every persist state event. + async def save_state() -> None: + await Actor.set_value('STATE', 'Hello, world!') + + # The save_state handler will be called every 10 seconds now. + Actor.on(Event.PERSIST_STATE, save_state) diff --git a/website/versioned_docs/version-2.7/03_concepts/code/actor_charge.py b/website/versioned_docs/version-2.7/03_concepts/code/actor_charge.py new file mode 100644 index 00000000..3478f60f --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/actor_charge.py @@ -0,0 +1,30 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # highlight-start + # Charge for a single occurrence of an event + await Actor.charge(event_name='init') + # highlight-end + + # Prepare some mock results + result = [ + {'word': 'Lorem'}, + {'word': 'Ipsum'}, + {'word': 'Dolor'}, + {'word': 'Sit'}, + {'word': 'Amet'}, + ] + # highlight-start + # Shortcut for charging for each pushed dataset item + await Actor.push_data(result, 'result-item') + # highlight-end + + # highlight-start + # Or you can charge for a given number of events manually + await Actor.charge( + event_name='result-item', + count=len(result), + ) + # highlight-end diff --git a/website/versioned_docs/version-2.7/03_concepts/code/conditional_actor_charge.py b/website/versioned_docs/version-2.7/03_concepts/code/conditional_actor_charge.py new file mode 100644 index 00000000..926c591d --- /dev/null +++ b/website/versioned_docs/version-2.7/03_concepts/code/conditional_actor_charge.py @@ -0,0 +1,18 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + # Check the dataset because there might already be items + # if the run migrated or was restarted + default_dataset = await Actor.open_dataset() + dataset_info = await default_dataset.get_info() + charged_items = dataset_info.item_count if dataset_info else 0 + + # highlight-start + if Actor.get_charging_manager().get_pricing_info().is_pay_per_event: + # highlight-end + await Actor.push_data({'hello': 'world'}, 'dataset-item') + elif charged_items < (Actor.config.max_paid_dataset_items or 0): + await Actor.push_data({'hello': 'world'}) + charged_items += 1 diff --git a/website/versioned_docs/version-2.7/04_upgrading/upgrading_to_v2.md b/website/versioned_docs/version-2.7/04_upgrading/upgrading_to_v2.md new file mode 100644 index 00000000..90062305 --- /dev/null +++ b/website/versioned_docs/version-2.7/04_upgrading/upgrading_to_v2.md @@ -0,0 +1,48 @@ +--- +id: upgrading-to-v2 +title: Upgrading to v2 +--- + +This page summarizes most of the breaking changes between Apify Python SDK v1.x and v2.0. + +## Python version support + +Support for Python 3.8 has been dropped. The Apify Python SDK v2.x now requires Python 3.9 or later. Make sure your environment is running a compatible version before upgrading. + +## Storages + +- The SDK now uses [crawlee](https://github.com/apify/crawlee-python) for local storage emulation. This change should not affect intended usage (working with `Dataset`, `KeyValueStore` and `RequestQueue` classes from the `apify.storages` module or using the shortcuts exposed by the `Actor` class) in any way. +- There is a difference in the `RequestQueue.add_request` method: it accepts an `apify.Request` object instead of a free-form dictionary. + - A quick way to migrate from dict-based arguments is to wrap it with a `Request.model_validate()` call. + - The preferred way is using the `Request.from_url` helper which prefills the `unique_key` and `id` attributes, or instantiating it directly, e.g., `Request(url='https://example.tld', ...)`. + - For simple use cases, `add_request` also accepts plain strings that contain an URL, e.g. `queue.add_request('https://example.tld')`. +- Removing the `StorageClientManager` class is a significant change. If you need to change the storage client, use `crawlee.service_container` instead. + +## Configuration + +The `apify.Configuration` class now uses `pydantic_settings` to load configuration from environment variables. This eliminates the need for the helper functions which handled environment variables in `apify._utils`. + +Attributes suffixed with `_millis` were renamed to remove said suffix and have the `datetime.timedelta` type now. + +## Actor + +- The `Actor.main` method has been removed as it brings no benefits compared to using `async with Actor`. +- The `Actor.add_webhook`, `Actor.start`, `Actor.call` and `Actor.start_task` methods now accept instances of the `apify.Webhook` model instead of an untyped `dict`. +- `Actor.start`, `Actor.call`, `Actor.start_task`, `Actor.set_status_message` and `Actor.abort` return instances of the `ActorRun` model instead of an untyped `dict`. +- Upon entering the context manager (`async with Actor`), the `Actor` puts the default logging configuration in place. This can be disabled using the `configure_logging` parameter. +- The `config` parameter of `Actor` has been renamed to `configuration`. +- Event handlers registered via `Actor.on` will now receive Pydantic objects instead of untyped dicts. For example, where you would do `event['isMigrating']`, you should now use `event.is_migrating` + +## Scrapy integration + +The `apify.scrapy.utils.open_queue_with_custom_client` function is not necessary anymore and has been removed. + +## Subpackage visibility + +The following modules were made private: + +- `apify.proxy_configuration` (`ProxyConfiguration` is still exported from `apify`) +- `apify.config` (`Configuration` is still exported from `apify`) +- `apify.actor` (`Actor` is still exported from `apify`) +- `apify.event_manager` +- `apify.consts` diff --git a/website/versioned_docs/version-2.7/api-packages.json b/website/versioned_docs/version-2.7/api-packages.json new file mode 100644 index 00000000..e2e41f0d --- /dev/null +++ b/website/versioned_docs/version-2.7/api-packages.json @@ -0,0 +1 @@ +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] diff --git a/website/versioned_docs/version-2.7/api-typedoc.json b/website/versioned_docs/version-2.7/api-typedoc.json new file mode 100644 index 00000000..7561b6f2 --- /dev/null +++ b/website/versioned_docs/version-2.7/api-typedoc.json @@ -0,0 +1,6951 @@ +{ + "id": 0, + "name": "apify", + "kind": 1, + "kindString": "Project", + "flags": {}, + "originalName": "", + "children": [ + { + "id": 1, + "name": "_ActorType", + "module": "_actor", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The class of `Actor`. Only make a new instance if you're absolutely sure you need to.\"]}" + } + ] + }, + "children": [ + { + "id": 2, + "name": "__init__", + "module": "_actor", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the functionality is accessible using the default instance\\n(e.g. `Actor.open_dataset()`).\\n\", {\"Arguments\": [{\"param\": \"configuration\", \"desc\": \"The Actor configuration to be used. If not passed, a new Configuration instance will\\nbe created.\"}, {\"param\": \"configure_logging\", \"desc\": \"Should the default logging configuration be configured?\"}, {\"param\": \"exit_process\", \"desc\": \"Whether the Actor should call `sys.exit` when the context manager exits. The default is\\nTrue except for the IPython, Pytest and Scrapy environments.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 63, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L63" + } + ], + "signatures": [ + { + "id": 3, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an Actor instance.\\n\\nNote that you don't have to do this, all the functionality is accessible using the default instance\\n(e.g. `Actor.open_dataset()`).\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 4, + "name": "configuration", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + }, + { + "id": 5, + "name": "configure_logging", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "True" + }, + { + "id": 6, + "name": "exit_process", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 7, + "name": "__call__", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Make a new Actor instance with a non-default configuration.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 149, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L149" + } + ], + "signatures": [ + { + "id": 8, + "name": "__call__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Make a new Actor instance with a non-default configuration.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Self" + }, + "parameters": [ + { + "id": 9, + "name": "configuration", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Configuration | None" + }, + "defaultValue": "None" + }, + { + "id": 10, + "name": "configure_logging", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "True" + }, + { + "id": 11, + "name": "exit_process", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 12, + "name": "apify_client", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 164, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L164" + } + ], + "signatures": [ + { + "id": 13, + "name": "apify_client", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The ApifyClientAsync instance the Actor instance uses.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyClientAsync" + }, + "parameters": [] + } + ] + }, + { + "id": 14, + "name": "configuration", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 169, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L169" + } + ], + "signatures": [ + { + "id": 15, + "name": "configuration", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Configuration", + "id": 180 + }, + "parameters": [] + } + ] + }, + { + "id": 16, + "name": "config", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 174, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L174" + } + ], + "signatures": [ + { + "id": 17, + "name": "config", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The Configuration instance the Actor instance uses.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Configuration", + "id": 180 + }, + "parameters": [] + } + ] + }, + { + "id": 18, + "name": "event_manager", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 179, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L179" + } + ], + "signatures": [ + { + "id": 19, + "name": "event_manager", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The EventManager instance the Actor instance uses.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "EventManager" + }, + "parameters": [] + } + ] + }, + { + "id": 20, + "name": "log", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 184, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L184" + } + ], + "signatures": [ + { + "id": 21, + "name": "log", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The logging.Logger instance the Actor uses.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "logging.Logger" + }, + "parameters": [] + } + ] + }, + { + "id": 22, + "name": "init", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the Actor instance.\\n\\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\\nits state in response to these events.\\n\\nThis method should be called immediately before performing any additional Actor actions, and it should be\\ncalled only once.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 207, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L207" + } + ], + "signatures": [ + { + "id": 23, + "name": "init", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize the Actor instance.\\n\\nThis initializes the Actor instance. It configures the right storage client based on whether the Actor is\\nrunning locally or on the Apify platform, it initializes the event manager for processing Actor events,\\nand starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist\\nits state in response to these events.\\n\\nThis method should be called immediately before performing any additional Actor actions, and it should be\\ncalled only once.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 24, + "name": "exit", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Exit the Actor instance.\\n\\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"desc\": \"The exit code with which the Actor should fail (defaults to `0`).\"}, {\"param\": \"event_listeners_timeout\", \"desc\": \"How long should the Actor wait for Actor event listeners to finish before exiting.\"}, {\"param\": \"status_message\", \"desc\": \"The final status message that the Actor should display.\"}, {\"param\": \"cleanup_timeout\", \"desc\": \"How long we should wait for event listeners.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 256, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L256" + } + ], + "signatures": [ + { + "id": 25, + "name": "exit", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Exit the Actor instance.\\n\\nThis stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,\\nsends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 26, + "name": "exit_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "0" + }, + { + "id": 27, + "name": "event_listeners_timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT" + }, + { + "id": 28, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 29, + "name": "cleanup_timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta" + }, + "defaultValue": "timedelta(seconds=30)" + } + ] + } + ] + }, + { + "id": 30, + "name": "fail", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fail the Actor instance.\\n\\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\\n\", {\"Arguments\": [{\"param\": \"exit_code\", \"desc\": \"The exit code with which the Actor should fail (defaults to `1`).\"}, {\"param\": \"exception\", \"desc\": \"The exception with which the Actor failed.\"}, {\"param\": \"status_message\", \"desc\": \"The final status message that the Actor should display.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 302, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L302" + } + ], + "signatures": [ + { + "id": 31, + "name": "fail", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fail the Actor instance.\\n\\nThis performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 32, + "name": "exit_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "1" + }, + { + "id": 33, + "name": "exception", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "BaseException | None" + }, + "defaultValue": "None" + }, + { + "id": 34, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 35, + "name": "new_client", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\\nvariables.\\n\\nYou can override the token via the available options. That's useful if you want to use the client\\nas a different Apify user than the SDK internals are using.\\n\", {\"Arguments\": [{\"param\": \"token\", \"desc\": \"The Apify API token.\"}, {\"param\": \"api_url\", \"desc\": \"The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.\"}, {\"param\": \"max_retries\", \"desc\": \"How many times to retry a failed request at most.\"}, {\"param\": \"min_delay_between_retries\", \"desc\": \"How long will the client wait between retrying requests\\n(increases exponentially from this value).\"}, {\"param\": \"timeout\", \"desc\": \"The socket timeout of the HTTP requests sent to the Apify API.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 327, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L327" + } + ], + "signatures": [ + { + "id": 36, + "name": "new_client", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a new instance of the Apify API client.\\n\\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\\nvariables.\\n\\nYou can override the token via the available options. That's useful if you want to use the client\\nas a different Apify user than the SDK internals are using.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyClientAsync" + }, + "parameters": [ + { + "id": 37, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 38, + "name": "api_url", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 39, + "name": "max_retries", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 40, + "name": "min_delay_between_retries", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + }, + { + "id": 41, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 42, + "name": "open_dataset", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes, such as online\\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\\nthe Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default dataset associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the dataset to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default dataset associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `Dataset` class for the given ID or name.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 365, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L365" + } + ], + "signatures": [ + { + "id": 43, + "name": "open_dataset", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a dataset.\\n\\nDatasets are used to store structured data where each object stored has the same attributes, such as online\\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\\nthe Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Dataset" + }, + "parameters": [ + { + "id": 44, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 45, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 46, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 47, + "name": "open_key_value_store", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the key-value store to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default key-value store associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the key-value store to be opened. If neither `id` nor `name` are provided, the method\\nreturns the default key-value store associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `KeyValueStore` class for the given ID or name.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 401, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L401" + } + ], + "signatures": [ + { + "id": 48, + "name": "open_key_value_store", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a key-value store.\\n\\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "KeyValueStore" + }, + "parameters": [ + { + "id": 49, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 50, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 51, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 52, + "name": "open_request_queue", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\\ncrawling orders.\\n\", {\"Arguments\": [{\"param\": \"id\", \"desc\": \"ID of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default request queue associated with the Actor run.\"}, {\"param\": \"name\", \"desc\": \"Name of the request queue to be opened. If neither `id` nor `name` are provided, the method returns\\nthe default request queue associated with the Actor run.\"}, {\"param\": \"force_cloud\", \"desc\": \"If set to `True` then the Apify cloud storage is always used. This way it is possible\\nto combine local and cloud storage.\\n\"}]}, {\"Returns\": [\"An instance of the `RequestQueue` class for the given ID or name.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 435, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L435" + } + ], + "signatures": [ + { + "id": 53, + "name": "open_request_queue", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open a request queue.\\n\\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\\ncrawling orders.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "RequestQueue" + }, + "parameters": [ + { + "id": 54, + "name": "id", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 55, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 56, + "name": "force_cloud", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "defaultValue": "False" + } + ] + } + ] + }, + { + "id": 57, + "name": "push_data", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"data\", \"desc\": \"The data to push to the default dataset.\"}, {\"param\": \"charged_event_name\", \"desc\": \"If provided and if the Actor uses the pay-per-event pricing model,\\nthe method will attempt to charge for the event for each pushed item.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 476, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L476" + } + ], + "signatures": [ + { + "id": 58, + "name": "push_data", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store an object or a list of objects to the default dataset of the current Actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ChargeResult | None" + }, + "parameters": [ + { + "id": 59, + "name": "data", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "dict | list[dict]" + } + }, + { + "id": 60, + "name": "charged_event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 61, + "name": "get_input", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the Actor input value from the default key-value store associated with the current Actor run.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 513, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L513" + } + ], + "signatures": [ + { + "id": 62, + "name": "get_input", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the Actor input value from the default key-value store associated with the current Actor run.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [] + } + ] + }, + { + "id": 63, + "name": "get_value", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the default key-value store associated with the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key of the record which to retrieve.\"}, {\"param\": \"default_value\", \"desc\": \"Default value returned in case the record does not exist.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 529, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L529" + } + ], + "signatures": [ + { + "id": 64, + "name": "get_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a value from the default key-value store associated with the current Actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [ + { + "id": 65, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 66, + "name": "default_value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 67, + "name": "set_value", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key of the record which to set.\"}, {\"param\": \"value\", \"desc\": \"The value of the record which to set, or None, if the record should be deleted.\"}, {\"param\": \"content_type\", \"desc\": \"The content type which should be set to the value.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 541, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L541" + } + ], + "signatures": [ + { + "id": 68, + "name": "set_value", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set or delete a value in the default key-value store associated with the current Actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 69, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 70, + "name": "value", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + } + }, + { + "id": 71, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 72, + "name": "get_charging_manager", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the charging manager to access granular pricing information.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 560, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L560" + } + ], + "signatures": [ + { + "id": 73, + "name": "get_charging_manager", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the charging manager to access granular pricing information.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "ChargingManager", + "id": 159 + }, + "parameters": [] + } + ] + }, + { + "id": 74, + "name": "charge", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the event to be charged for.\"}, {\"param\": \"count\", \"desc\": \"Number of events to charge for.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 565, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L565" + } + ], + "signatures": [ + { + "id": 75, + "name": "charge", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ChargeResult", + "id": 171 + }, + "parameters": [ + { + "id": 76, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 77, + "name": "count", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "1" + } + ] + } + ] + }, + { + "id": 78, + "name": "on", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add an event listener to the Actor's event manager.\\n\\nThe following events can be emitted:\\n\\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\\nusage.\\n\\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\\ndisruption.\\n\\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\\n\\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\\nto clean up its state if the abort is graceful.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"The Actor event to listen for.\"}, {\"param\": \"listener\", \"desc\": \"The function to be called when the event is emitted (can be async).\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 600, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L600" + } + ], + "signatures": [ + { + "id": 79, + "name": "on", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add an event listener to the Actor's event manager.\\n\\nThe following events can be emitted:\\n\\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\\nusage.\\n\\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\\ndisruption.\\n\\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\\n\\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\\nto clean up its state if the abort is graceful.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "EventListener[Any]" + }, + "parameters": [ + { + "id": 80, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Event" + } + }, + { + "id": 81, + "name": "listener", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "EventListener[Any]" + } + } + ] + } + ] + }, + { + "id": 82, + "name": "off", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove a listener, or all listeners, from an Actor event.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"The Actor event for which to remove listeners.\"}, {\"param\": \"listener\", \"desc\": \"The listener which is supposed to be removed. If not passed, all listeners of this event\\nare removed.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 642, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L642" + } + ], + "signatures": [ + { + "id": 83, + "name": "off", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Remove a listener, or all listeners, from an Actor event.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 84, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Event" + } + }, + { + "id": 85, + "name": "listener", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Callable | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 86, + "name": "is_at_home", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run).\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 654, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L654" + } + ], + "signatures": [ + { + "id": 87, + "name": "is_at_home", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run).\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 88, + "name": "get_env", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables, see the\\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 658, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L658" + } + ], + "signatures": [ + { + "id": 89, + "name": "get_env", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\\n\\nFor a list of all the environment variables, see the\\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [] + } + ] + }, + { + "id": 90, + "name": "start", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run an Actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\\nto the time remaining from this Actor timeout.\"}, {\"param\": \"wait_for_finish\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. By default,\\nit is 0, the maximum value is 300.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 687, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L687" + } + ], + "signatures": [ + { + "id": 91, + "name": "start", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run an Actor on the Apify platform.\\n\\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorRun", + "id": 193 + }, + "parameters": [ + { + "id": 92, + "name": "actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 93, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 94, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 95, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 96, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 97, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 98, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None | Literal['RemainingTime']" + }, + "defaultValue": "None" + }, + { + "id": 99, + "name": "wait_for_finish", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 100, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[Webhook] | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 101, + "name": "abort", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Abort given Actor run on the Apify platform using the current user account.\\n\\nThe user account is determined by the `APIFY_TOKEN` environment variable.\\n\", {\"Arguments\": [{\"param\": \"run_id\", \"desc\": \"The ID of the Actor run to be aborted.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"status_message\", \"desc\": \"Status message of the Actor to be set on the platform.\"}, {\"param\": \"gracefully\", \"desc\": \"If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\\nto resurrect the run later.\\n\"}]}, {\"Returns\": [\"Info about the aborted Actor run.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 769, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L769" + } + ], + "signatures": [ + { + "id": 102, + "name": "abort", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Abort given Actor run on the Apify platform using the current user account.\\n\\nThe user account is determined by the `APIFY_TOKEN` environment variable.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorRun", + "id": 193 + }, + "parameters": [ + { + "id": 103, + "name": "run_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 104, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 105, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 106, + "name": "gracefully", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 107, + "name": "call", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an Actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\", {\"Arguments\": [{\"param\": \"actor_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor. Using `RemainingTime` will set timeout of the other Actor\\nto the time remaining from this Actor timeout.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\\na webhook set up for the Actor, you do not have to add it again here.\"}, {\"param\": \"wait\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided,\\nwaits indefinitely.\"}, {\"param\": \"logger\", \"desc\": \"Logger used to redirect logs from the Actor run. Using \\\"default\\\" literal means that a predefined\\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\\nwill redirect logs to the provided logger.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 803, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L803" + } + ], + "signatures": [ + { + "id": 108, + "name": "call", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an Actor on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorRun | None" + }, + "parameters": [ + { + "id": 109, + "name": "actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 110, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 111, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 112, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 113, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 114, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 115, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None | Literal['RemainingTime']" + }, + "defaultValue": "None" + }, + { + "id": 116, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[Webhook] | None" + }, + "defaultValue": "None" + }, + { + "id": 117, + "name": "wait", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + }, + { + "id": 118, + "name": "logger", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "logging.Logger | None | Literal['default']" + }, + "defaultValue": "'default'" + } + ] + } + ] + }, + { + "id": 119, + "name": "call_task", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an Actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\\ndirectly rather than an Actor task, please use the `Actor.call`\\n\", {\"Arguments\": [{\"param\": \"task_id\", \"desc\": \"The ID of the Actor to be run.\"}, {\"param\": \"task_input\", \"desc\": \"Overrides the input to pass to the Actor run.\"}, {\"param\": \"token\", \"desc\": \"The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"build\", \"desc\": \"Specifies the Actor build to run. It can be either a build tag or build number. By default,\\nthe run uses the build specified in the default run configuration for the Actor (typically latest).\"}, {\"param\": \"memory_mbytes\", \"desc\": \"Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\\nin the default run configuration for the Actor.\"}, {\"param\": \"timeout\", \"desc\": \"Optional timeout for the run, in seconds. By default, the run uses timeout specified in\\nthe default run configuration for the Actor.\"}, {\"param\": \"webhooks\", \"desc\": \"Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\\na webhook set up for the Actor, you do not have to add it again here.\"}, {\"param\": \"wait\", \"desc\": \"The maximum number of seconds the server waits for the run to finish. If not provided, waits\\nindefinitely.\\n\"}]}, {\"Returns\": [\"Info about the started Actor run.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 878, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L878" + } + ], + "signatures": [ + { + "id": 120, + "name": "call_task", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start an Actor task on the Apify Platform and wait for it to finish before returning.\\n\\nIt waits indefinitely, unless the wait argument is provided.\\n\\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\\ndirectly rather than an Actor task, please use the `Actor.call`\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorRun | None" + }, + "parameters": [ + { + "id": 121, + "name": "task_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 122, + "name": "task_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "dict | None" + }, + "defaultValue": "None" + }, + { + "id": 123, + "name": "build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 124, + "name": "memory_mbytes", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + }, + { + "id": 125, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + }, + { + "id": 126, + "name": "webhooks", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[Webhook] | None" + }, + "defaultValue": "None" + }, + { + "id": 127, + "name": "wait", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + }, + { + "id": 128, + "name": "token", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 129, + "name": "metamorph", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Transform this Actor run to an Actor run of a different Actor.\\n\\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\\ndefault key-value store.\\n\", {\"Arguments\": [{\"param\": \"target_actor_id\", \"desc\": \"ID of the target Actor that the run should be transformed into\"}, {\"param\": \"run_input\", \"desc\": \"The input to pass to the new run.\"}, {\"param\": \"target_actor_build\", \"desc\": \"The build of the target Actor. It can be either a build tag or build number.\\nBy default, the run uses the build specified in the default run configuration for the target Actor\\n(typically the latest build).\"}, {\"param\": \"content_type\", \"desc\": \"The content type of the input.\"}, {\"param\": \"custom_after_sleep\", \"desc\": \"How long to sleep for after the metamorph, to wait for the container to be stopped.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 939, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L939" + } + ], + "signatures": [ + { + "id": 130, + "name": "metamorph", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Transform this Actor run to an Actor run of a different Actor.\\n\\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\\ndefault key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 131, + "name": "target_actor_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 132, + "name": "run_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Any" + }, + "defaultValue": "None" + }, + { + "id": 133, + "name": "target_actor_build", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 134, + "name": "content_type", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 135, + "name": "custom_after_sleep", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 136, + "name": "reboot", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Internally reboot this Actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"Arguments\": [{\"param\": \"event_listeners_timeout\", \"desc\": \"How long should the Actor wait for Actor event listeners to finish before exiting.\"}, {\"param\": \"custom_after_sleep\", \"desc\": \"How long to sleep for after the reboot, to wait for the container to be stopped.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 986, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L986" + } + ], + "signatures": [ + { + "id": 137, + "name": "reboot", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Internally reboot this Actor.\\n\\nThe system stops the current container and starts a new one, with the same run ID and default storages.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 138, + "name": "event_listeners_timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT" + }, + { + "id": 139, + "name": "custom_after_sleep", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "timedelta | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 140, + "name": "add_webhook", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an ad-hoc webhook for the current Actor run.\\n\\nThis webhook lets you receive a notification when the Actor run finished or failed.\\n\\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\\nthe function will print a warning and have no effect.\\n\\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"Arguments\": [{\"param\": \"webhook\", \"desc\": \"The webhook to be added\"}, {\"param\": \"ignore_ssl_errors\", \"desc\": \"Whether the webhook should ignore SSL errors returned by request_url\"}, {\"param\": \"do_not_retry\", \"desc\": \"Whether the webhook should retry sending the payload to request_url upon failure.\"}, {\"param\": \"idempotency_key\", \"desc\": \"A unique identifier of a webhook. You can use it to ensure that you won't create\\nthe same webhook multiple times.\\n\"}]}, {\"Returns\": [\"The created webhook.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 1041, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1041" + } + ], + "signatures": [ + { + "id": 141, + "name": "add_webhook", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an ad-hoc webhook for the current Actor run.\\n\\nThis webhook lets you receive a notification when the Actor run finished or failed.\\n\\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\\nthe function will print a warning and have no effect.\\n\\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 142, + "name": "webhook", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Webhook", + "id": 188 + } + }, + { + "id": 143, + "name": "ignore_ssl_errors", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 144, + "name": "do_not_retry", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + }, + { + "id": 145, + "name": "idempotency_key", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 146, + "name": "set_status_message", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set the status message for the current Actor run.\\n\", {\"Arguments\": [{\"param\": \"status_message\", \"desc\": \"The status message to set to the run.\"}, {\"param\": \"is_terminal\", \"desc\": \"Set this flag to True if this is the final status message of the Actor run.\\n\"}]}, {\"Returns\": [\"The updated Actor run object.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 1088, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1088" + } + ], + "signatures": [ + { + "id": 147, + "name": "set_status_message", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Set the status message for the current Actor run.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorRun | None" + }, + "parameters": [ + { + "id": 148, + "name": "status_message", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 149, + "name": "is_terminal", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "bool | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 150, + "name": "create_proxy_configuration", + "module": "_actor", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"Arguments\": [{\"param\": \"actor_proxy_input\", \"desc\": \"Proxy configuration field from the Actor input, if input has such input field. If you\\npass this argument, all the other arguments will be inferred from it.\"}, {\"param\": \"password\", \"desc\": \"Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\\nif available.\"}, {\"param\": \"groups\", \"desc\": \"Proxy groups which the Apify Proxy should use, if provided.\"}, {\"param\": \"country_code\", \"desc\": \"Country which the Apify Proxy should use, if provided.\"}, {\"param\": \"proxy_urls\", \"desc\": \"Custom proxy server URLs which should be rotated through.\"}, {\"param\": \"new_url_function\", \"desc\": \"Function which returns a custom proxy URL to be used.\\n\"}]}, {\"Returns\": [\"ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\", \"on the configuration.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 1120, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1120" + } + ], + "signatures": [ + { + "id": 151, + "name": "create_proxy_configuration", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a ProxyConfiguration object with the passed proxy configuration.\\n\\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\\n\\nFor more details and code examples, see the `ProxyConfiguration` class.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ProxyConfiguration | None" + }, + "parameters": [ + { + "id": 152, + "name": "actor_proxy_input", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "dict\n | None" + }, + "defaultValue": "None" + }, + { + "id": 153, + "name": "password", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 154, + "name": "groups", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str] | None" + }, + "defaultValue": "None" + }, + { + "id": 155, + "name": "country_code", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 156, + "name": "proxy_urls", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "list[str | None] | None" + }, + "defaultValue": "None" + }, + { + "id": 157, + "name": "new_url_function", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "_NewUrlFunction | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 2 + ] + }, + { + "title": "Methods", + "children": [ + 7, + 101, + 140, + 12, + 107, + 119, + 74, + 16, + 14, + 150, + 18, + 24, + 30, + 72, + 88, + 61, + 63, + 22, + 86, + 20, + 129, + 35, + 82, + 78, + 42, + 47, + 52, + 57, + 136, + 146, + 67, + 90 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 57, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L57" + } + ] + }, + { + "id": 158, + "name": "Actor", + "module": "_actor", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The entry point of the SDK, through which all the Actor operations should be done.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_actor.py", + "line": 1198, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_actor.py#L1198" + } + ] + }, + { + "id": 159, + "name": "ChargingManager", + "module": "_charging", + "docsGroup": "Interfaces", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Provides fine-grained access to pay-per-event functionality.\"]}" + } + ] + }, + "children": [ + { + "id": 160, + "name": "charge", + "module": "_charging", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the event to be charged for.\"}, {\"param\": \"count\", \"desc\": \"Number of events to charge for.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 34, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L34" + } + ], + "signatures": [ + { + "id": 161, + "name": "charge", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Charge for a specified number of events - sub-operations of the Actor.\\n\\nThis is relevant only for the pay-per-event pricing model.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ChargeResult", + "id": 171 + }, + "parameters": [ + { + "id": 162, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 163, + "name": "count", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "1" + } + ] + } + ] + }, + { + "id": 164, + "name": "calculate_total_charged_amount", + "module": "_charging", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Calculate the total amount of money charged for pay-per-event events so far.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 44, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L44" + } + ], + "signatures": [ + { + "id": 165, + "name": "calculate_total_charged_amount", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Calculate the total amount of money charged for pay-per-event events so far.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Decimal" + }, + "parameters": [] + } + ] + }, + { + "id": 166, + "name": "calculate_max_event_charge_count_within_limit", + "module": "_charging", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Calculate how many instances of an event can be charged before we reach the configured limit.\\n\", {\"Arguments\": [{\"param\": \"event_name\", \"desc\": \"Name of the inspected event.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 47, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L47" + } + ], + "signatures": [ + { + "id": 167, + "name": "calculate_max_event_charge_count_within_limit", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Calculate how many instances of an event can be charged before we reach the configured limit.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "int | None" + }, + "parameters": [ + { + "id": 168, + "name": "event_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 169, + "name": "get_pricing_info", + "module": "_charging", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve detailed information about the effective pricing of the current Actor run.\\n\\nThis can be used for instance when your code needs to support multiple pricing models in transition periods.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 54, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L54" + } + ], + "signatures": [ + { + "id": 170, + "name": "get_pricing_info", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve detailed information about the effective pricing of the current Actor run.\\n\\nThis can be used for instance when your code needs to support multiple pricing models in transition periods.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "ActorPricingInfo", + "id": 175 + }, + "parameters": [] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 166, + 164, + 160, + 169 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 31, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L31" + } + ] + }, + { + "id": 171, + "name": "ChargeResult", + "module": "_charging", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Result of the `ChargingManager.charge` method.\"]}" + } + ] + }, + "children": [ + { + "id": 172, + "name": "event_charge_limit_reached", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"If true, no more events of this type can be charged within the limit.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 66, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L66" + } + ] + }, + { + "id": 173, + "name": "charged_count", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Total amount of charged events - may be lower than the requested amount.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 69, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L69" + } + ] + }, + { + "id": 174, + "name": "chargeable_within_limit", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"How many events of each known type can still be charged within the limit.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "dict[str, int | None]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 72, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L72" + } + ] + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 174, + 173, + 172 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 63, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L63" + } + ] + }, + { + "id": 175, + "name": "ActorPricingInfo", + "module": "_charging", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Result of the `ChargingManager.get_pricing_info` method.\"]}" + } + ] + }, + "children": [ + { + "id": 176, + "name": "pricing_model", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"The currently effective pricing model.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "PricingModel | None" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 81, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L81" + } + ] + }, + { + "id": 177, + "name": "max_total_charge_usd", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A configured limit for the total charged amount - if you exceed it, you won't receive more money than this.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Decimal" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 84, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L84" + } + ] + }, + { + "id": 178, + "name": "is_pay_per_event", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A shortcut - true if the Actor runs with the pay-per-event pricing model.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 87, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L87" + } + ] + }, + { + "id": 179, + "name": "per_event_prices", + "module": "_charging", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Price of every known event type.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "dict[str, Decimal]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 90, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L90" + } + ] + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 178, + 177, + 179, + 176 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_charging.py", + "line": 78, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_charging.py#L78" + } + ] + }, + { + "id": 180, + "name": "Configuration", + "module": "_configuration", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A class for specifying the configuration of an Actor.\\n\\nCan be used either globally via `Configuration.get_global_configuration()`,\\nor it can be specific to each `Actor` instance on the `actor.config` property.\"]}" + } + ] + }, + "children": [ + { + "id": 181, + "name": "disable_browser_sandbox_on_platform", + "module": "_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Disable the browser sandbox mode when running on the Apify platform.\\n\\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\\nthis will crash chromium that was started with browser sandbox mode.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_configuration.py", + "line": 378, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L378" + } + ], + "signatures": [ + { + "id": 182, + "name": "disable_browser_sandbox_on_platform", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Disable the browser sandbox mode when running on the Apify platform.\\n\\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\\nthis will crash chromium that was started with browser sandbox mode.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Self" + }, + "parameters": [] + } + ] + }, + { + "id": 183, + "name": "get_global_configuration", + "module": "_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the global instance of the configuration.\\n\\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\\ninstead.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_configuration.py", + "line": 391, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L391" + } + ], + "signatures": [ + { + "id": 184, + "name": "get_global_configuration", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve the global instance of the configuration.\\n\\nMostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`\\ninstead.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Configuration", + "id": 180 + }, + "parameters": [] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 181, + 183 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_configuration.py", + "line": 29, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_configuration.py#L29" + } + ] + }, + { + "id": 185, + "name": "encode_base62", + "module": "_crypto", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Encode the given number to base62.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_crypto.py", + "line": 171, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_crypto.py#L171" + } + ], + "signatures": [ + { + "id": 186, + "name": "encode_base62", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Encode the given number to base62.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 187, + "name": "num", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "int" + } + } + ] + } + ] + }, + { + "id": 188, + "name": "Webhook", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 20, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L20" + } + ] + }, + { + "id": 189, + "name": "ActorRunMeta", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 39, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L39" + } + ] + }, + { + "id": 190, + "name": "ActorRunStats", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 46, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L46" + } + ] + }, + { + "id": 191, + "name": "ActorRunOptions", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 67, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L67" + } + ] + }, + { + "id": 192, + "name": "ActorRunUsage", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 78, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L78" + } + ] + }, + { + "id": 193, + "name": "ActorRun", + "module": "_models", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_models.py", + "line": 96, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_models.py#L96" + } + ] + }, + { + "id": 194, + "name": "PersistStateEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 35, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L35" + } + ] + }, + { + "id": 195, + "name": "SystemInfoEventData", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 41, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L41" + } + ] + }, + { + "id": 196, + "name": "SystemInfoEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 68, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L68" + } + ] + }, + { + "id": 197, + "name": "MigratingEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 74, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L74" + } + ] + }, + { + "id": 198, + "name": "AbortingEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 80, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L80" + } + ] + }, + { + "id": 199, + "name": "ExitEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 86, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L86" + } + ] + }, + { + "id": 200, + "name": "EventWithoutData", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 92, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L92" + } + ] + }, + { + "id": 201, + "name": "DeprecatedEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 105, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L105" + } + ] + }, + { + "id": 202, + "name": "UnknownEvent", + "module": "_platform_event_manager", + "docsGroup": "Data structures", + "kind": 128, + "kindString": "Class", + "flags": {}, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 111, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L111" + } + ] + }, + { + "id": 203, + "name": "PlatformEventManager", + "module": "_platform_event_manager", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A class for managing Actor events.\\n\\nYou shouldn't use this class directly,\\nbut instead use it via the `Actor.on()` and `Actor.off()` methods.\"]}" + } + ] + }, + "children": [ + { + "id": 204, + "name": "__init__", + "module": "_platform_event_manager", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of the EventManager.\\n\", {\"Arguments\": [{\"param\": \"config\", \"desc\": \"The Actor configuration to be used in this event manager.\"}, {\"param\": \"kwargs\", \"desc\": \"Event manager options - forwarded to the base class\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 151, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L151" + } + ], + "signatures": [ + { + "id": 205, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of the EventManager.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 206, + "name": "config", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Configuration", + "id": 180 + } + }, + { + "id": 207, + "name": "kwargs", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Unpack[EventManagerOptions]" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 204 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_platform_event_manager.py", + "line": 139, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_platform_event_manager.py#L139" + } + ] + }, + { + "id": 208, + "name": "ProxyInfo", + "module": "_proxy_configuration", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Provides information about a proxy connection that is used for requests.\"]}" + } + ] + }, + "children": [ + { + "id": 209, + "name": "groups", + "module": "_proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\\nthe proxy will select the groups automatically.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "list[str]" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 77, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L77" + } + ] + }, + { + "id": 210, + "name": "country_code", + "module": "_proxy_configuration", + "kind": 1024, + "kindString": "Property", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "str | None" + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 81, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L81" + } + ] + } + ], + "groups": [ + { + "title": "Properties", + "children": [ + 210, + 209 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 74, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L74" + } + ] + }, + { + "id": 211, + "name": "ProxyConfiguration", + "module": "_proxy_configuration", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Configures a connection to a proxy server with the provided options.\\n\\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\\nno configuration option, the proxies will be managed automatically using a smart algorithm.\\n\\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\\nof proxy URLs will be rotated by the configuration, if this option is provided.\"]}" + } + ] + }, + "children": [ + { + "id": 212, + "name": "initialize", + "module": "_proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if using proxy, if so, check the access.\\n\\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\\n\\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\\nif the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\\n`ProxyConfiguration` instance instead of calling this manually.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 180, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L180" + } + ], + "signatures": [ + { + "id": 213, + "name": "initialize", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if using proxy, if so, check the access.\\n\\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\\n\\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\\nif the country code is provided.\\n\\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\\n`ProxyConfiguration` instance instead of calling this manually.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 214, + "name": "new_proxy_info", + "module": "_proxy_configuration", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\\nuse `ProxyConfiguration.new_url`.\\n\", {\"Arguments\": [{\"param\": \"session_id\", \"desc\": \"Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\\nAll the HTTP requests going through the proxy with the same session identifier will use the same\\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\\nand include only the following: `0-9`, `a-z`, `A-Z`, `\\\".\\\"`, `\\\"_\\\"` and `\\\"~\\\"`.\"}, {\"param\": \"request\", \"desc\": \"request for which the proxy info is being issued, used in proxy tier handling.\"}, {\"param\": \"proxy_tier\", \"desc\": \"allows forcing the proxy tier to be used.\\n\"}]}, {\"Returns\": [\"Dictionary that represents information about the proxy and its configuration.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 205, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L205" + } + ], + "signatures": [ + { + "id": 215, + "name": "new_proxy_info", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new ProxyInfo object.\\n\\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\\nuse `ProxyConfiguration.new_url`.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ProxyInfo | None" + }, + "parameters": [ + { + "id": 216, + "name": "session_id", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 217, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request | None" + }, + "defaultValue": "None" + }, + { + "id": 218, + "name": "proxy_tier", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 212, + 214 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_proxy_configuration.py", + "line": 93, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_proxy_configuration.py#L93" + } + ] + }, + { + "id": 219, + "name": "docs_group", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Mark a symbol for rendering and grouping in documentation.\\n\\nThis decorator is used solely for documentation purposes and does not modify the behavior\\nof the decorated callable.\\n\", {\"Arguments\": [{\"param\": \"group_name\", \"desc\": \"The documentation group to which the symbol belongs.\\n\"}]}, {\"Returns\": [\"The original callable without modification.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_utils.py", + "line": 33, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_utils.py#L33" + } + ], + "signatures": [ + { + "id": 220, + "name": "docs_group", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Mark a symbol for rendering and grouping in documentation.\\n\\nThis decorator is used solely for documentation purposes and does not modify the behavior\\nof the decorated callable.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Callable" + }, + "parameters": [ + { + "id": 221, + "name": "group_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "GroupName" + } + } + ] + } + ] + }, + { + "id": 222, + "name": "docs_name", + "module": "_utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Rename a symbol for documentation rendering.\\n\\nThis decorator modifies only the displayed name of the symbol in the generated documentation\\nand does not affect its runtime behavior.\\n\", {\"Arguments\": [{\"param\": \"symbol_name\", \"desc\": \"The name to be used in the documentation.\\n\"}]}, {\"Returns\": [\"The original callable without modification.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/_utils.py", + "line": 52, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/_utils.py#L52" + } + ], + "signatures": [ + { + "id": 223, + "name": "docs_name", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Rename a symbol for documentation rendering.\\n\\nThis decorator modifies only the displayed name of the symbol in the generated documentation\\nand does not affect its runtime behavior.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Callable" + }, + "parameters": [ + { + "id": 224, + "name": "symbol_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 225, + "name": "ApifyStorageClient", + "module": "apify_storage_client._apify_storage_client", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A storage client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_apify_storage_client.py", + "line": 24, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_apify_storage_client.py#L24" + } + ] + }, + { + "id": 226, + "name": "DatasetClient", + "module": "apify_storage_client._dataset_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Dataset resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_client.py", + "line": 20, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_client.py#L20" + } + ] + }, + { + "id": 227, + "name": "DatasetCollectionClient", + "module": "apify_storage_client._dataset_collection_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Dataset collection resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_collection_client.py", + "line": 14, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_dataset_collection_client.py#L14" + } + ] + }, + { + "id": 228, + "name": "KeyValueStoreClient", + "module": "apify_storage_client._key_value_store_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Key-value store resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [ + { + "id": 229, + "name": "get_public_url", + "module": "apify_storage_client._key_value_store_client", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"Arguments\": [{\"param\": \"key\", \"desc\": \"The key for which the URL should be generated.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py", + "line": 89, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py#L89" + } + ], + "signatures": [ + { + "id": 230, + "name": "get_public_url", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 231, + "name": "key", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 229 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py", + "line": 23, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_client.py#L23" + } + ] + }, + { + "id": 232, + "name": "KeyValueStoreCollectionClient", + "module": "apify_storage_client._key_value_store_collection_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Key-value store collection resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_collection_client.py", + "line": 14, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_key_value_store_collection_client.py#L14" + } + ] + }, + { + "id": 233, + "name": "RequestQueueClient", + "module": "apify_storage_client._request_queue_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Request queue resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_client.py", + "line": 24, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_client.py#L24" + } + ] + }, + { + "id": 234, + "name": "RequestQueueCollectionClient", + "module": "apify_storage_client._request_queue_collection_client", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Request queue collection resource client implementation based on the Apify platform storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_collection_client.py", + "line": 14, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/apify_storage_client/_request_queue_collection_client.py#L14" + } + ] + }, + { + "id": 235, + "name": "run_scrapy_actor", + "module": "scrapy._actor_runner", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start Twisted's reactor and execute the provided Actor coroutine.\\n\\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\\nenabling the Apify and Scrapy integration to work together.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/_actor_runner.py", + "line": 19, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_actor_runner.py#L19" + } + ], + "signatures": [ + { + "id": 236, + "name": "run_scrapy_actor", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Start Twisted's reactor and execute the provided Actor coroutine.\\n\\nThis function initiates the Twisted reactor and runs the given asyncio coroutine (typically the\\nActor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,\\nenabling the Apify and Scrapy integration to work together.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 237, + "name": "coro", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Coroutine" + } + } + ] + } + ] + }, + { + "id": 238, + "name": "AsyncThread", + "module": "scrapy._async_thread", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Class for running an asyncio event loop in a separate thread.\\n\\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\\nthat runs in its own dedicated thread.\"]}" + } + ] + }, + "children": [ + { + "id": 239, + "name": "run_coro", + "module": "scrapy._async_thread", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run a coroutine on an event loop running in a separate thread.\\n\\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\\nor the specified timeout is reached.\\n\", {\"Arguments\": [{\"param\": \"coro\", \"desc\": \"The coroutine to run.\"}, {\"param\": \"timeout\", \"desc\": \"The maximum number of seconds to wait for the coroutine to finish.\\n\"}]}, {\"Returns\": [\"The result returned by the coroutine.\", \"\"]}, {\"Raises\": [{\"param\": \"RuntimeError\", \"desc\": \"If the event loop is not running.\"}, {\"param\": \"TimeoutError\", \"desc\": \"If the coroutine does not complete within the timeout.\"}, {\"param\": \"Exception\", \"desc\": \"Any exception raised during coroutine execution.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", + "line": 33, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L33" + } + ], + "signatures": [ + { + "id": 240, + "name": "run_coro", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Run a coroutine on an event loop running in a separate thread.\\n\\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\\nor the specified timeout is reached.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Any" + }, + "parameters": [ + { + "id": 241, + "name": "coro", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Coroutine" + } + }, + { + "id": 242, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "timedelta" + }, + "defaultValue": "timedelta(seconds=60)" + } + ] + } + ] + }, + { + "id": 243, + "name": "close", + "module": "scrapy._async_thread", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the event loop and its thread gracefully.\\n\\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\\n\", {\"Arguments\": [{\"param\": \"timeout\", \"desc\": \"The maximum number of seconds to wait for the event loop thread to exit.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", + "line": 70, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L70" + } + ], + "signatures": [ + { + "id": 244, + "name": "close", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the event loop and its thread gracefully.\\n\\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 245, + "name": "timeout", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "timedelta" + }, + "defaultValue": "timedelta(seconds=60)" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 243, + 239 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/_async_thread.py", + "line": 16, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_async_thread.py#L16" + } + ] + }, + { + "id": 246, + "name": "initialize_logging", + "module": "scrapy._logging_config", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Configure logging for Apify Actors and adjust Scrapy's logging settings.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/_logging_config.py", + "line": 26, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/_logging_config.py#L26" + } + ], + "signatures": [ + { + "id": 247, + "name": "initialize_logging", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Configure logging for Apify Actors and adjust Scrapy's logging settings.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [] + } + ] + }, + { + "id": 248, + "name": "ApifyCacheStorage", + "module": "scrapy.extensions._httpcache", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\\n\\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\\nto be installed.\"]}" + } + ] + }, + "children": [ + { + "id": 249, + "name": "open_spider", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the cache storage for a spider.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 46, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L46" + } + ], + "signatures": [ + { + "id": 250, + "name": "open_spider", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the cache storage for a spider.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 251, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 252, + "name": "close_spider", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the cache storage for a spider.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 65, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L65" + } + ], + "signatures": [ + { + "id": 253, + "name": "close_spider", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the cache storage for a spider.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 254, + "name": "_", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + }, + { + "id": 255, + "name": "current_time", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 256, + "name": "retrieve_response", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve a response from the cache storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 108, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L108" + } + ], + "signatures": [ + { + "id": 257, + "name": "retrieve_response", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Retrieve a response from the cache storage.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Response | None" + }, + "parameters": [ + { + "id": 258, + "name": "_", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + }, + { + "id": 259, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 260, + "name": "current_time", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 261, + "name": "store_response", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store a response in the cache storage.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 140, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L140" + } + ], + "signatures": [ + { + "id": 262, + "name": "store_response", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Store a response in the cache storage.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 263, + "name": "_", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + }, + { + "id": 264, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 265, + "name": "response", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Response" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 252, + 249, + 256, + 261 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 29, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L29" + } + ] + }, + { + "id": 266, + "name": "to_gzip", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Dump a dictionary to a gzip-compressed byte stream.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 160, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L160" + } + ], + "signatures": [ + { + "id": 267, + "name": "to_gzip", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Dump a dictionary to a gzip-compressed byte stream.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "bytes" + }, + "parameters": [ + { + "id": 268, + "name": "data", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + }, + { + "id": 269, + "name": "mtime", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "int | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 270, + "name": "from_gzip", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Load a dictionary from a gzip-compressed byte stream.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 168, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L168" + } + ], + "signatures": [ + { + "id": 271, + "name": "from_gzip", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Load a dictionary from a gzip-compressed byte stream.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "dict" + }, + "parameters": [ + { + "id": 272, + "name": "gzip_bytes", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "bytes" + } + } + ] + } + ] + }, + { + "id": 273, + "name": "read_gzip_time", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Read the modification time from a gzip-compressed byte stream without decompressing the data.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 175, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L175" + } + ], + "signatures": [ + { + "id": 274, + "name": "read_gzip_time", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Read the modification time from a gzip-compressed byte stream without decompressing the data.\"]}" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "int" + }, + "parameters": [ + { + "id": 275, + "name": "gzip_bytes", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "bytes" + } + } + ] + } + ] + }, + { + "id": 276, + "name": "get_kvs_name", + "module": "scrapy.extensions._httpcache", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the key value store name for a spider.\\n\\nThe key value store name is derived from the spider name by replacing all special characters\\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\\n'httpcache-' and truncated to the maximum length.\\n\\nThe documentation\\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\\n\\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\\nThe name of the key value store should indicate to which spider it belongs, e.g. in\\nthe listing in the Apify's console.\\n\", {\"Arguments\": [{\"param\": \"spider_name\", \"desc\": \"Value of the Spider instance's name attribute.\"}, {\"param\": \"max_length\", \"desc\": \"Maximum length of the key value store name.\\n\"}]}, \"Returns: Key value store name.\\n\", {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If the spider name contains only special characters.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py", + "line": 183, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/extensions/_httpcache.py#L183" + } + ], + "signatures": [ + { + "id": 277, + "name": "get_kvs_name", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Get the key value store name for a spider.\\n\\nThe key value store name is derived from the spider name by replacing all special characters\\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\\n'httpcache-' and truncated to the maximum length.\\n\\nThe documentation\\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\\n\\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\\nThe name of the key value store should indicate to which spider it belongs, e.g. in\\nthe listing in the Apify's console.\\n\", {\"" + } + ], + "blockTags": [ + { + "tag": "@returns", + "content": [ + { + "kind": "text", + "text": "[{\"param\"" + } + ] + } + ] + }, + "type": { + "type": "intrinsic", + "name": "str" + }, + "parameters": [ + { + "id": 278, + "name": "spider_name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 279, + "name": "max_length", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "int" + }, + "defaultValue": "60" + } + ] + } + ] + }, + { + "id": 280, + "name": "ApifyHttpProxyMiddleware", + "module": "scrapy.middlewares.apify_proxy", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Apify HTTP proxy middleware for Scrapy.\\n\\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\\nprovided by the Actor input. An example of the proxy settings:\\n\\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}\"]}" + } + ] + }, + "children": [ + { + "id": 281, + "name": "__init__", + "module": "scrapy.middlewares.apify_proxy", + "kind": 512, + "kindString": "Constructor", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\\n\", {\"Arguments\": [{\"param\": \"proxy_settings\", \"desc\": \"Dictionary containing proxy settings, provided by the Actor input.\"}, {\"param\": \"auth_encoding\", \"desc\": \"Encoding for basic authentication (default is 'latin-1').\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 28, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L28" + } + ], + "signatures": [ + { + "id": 282, + "name": "__init__", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create a new instance.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 283, + "name": "proxy_settings", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "dict" + } + } + ] + } + ] + }, + { + "id": 284, + "name": "from_crawler", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"Arguments\": [{\"param\": \"crawler\", \"desc\": \"Scrapy Crawler object.\\n\"}]}, \"Returns: Instance of the class.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 39, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L39" + } + ], + "signatures": [ + { + "id": 285, + "name": "from_crawler", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyHttpProxyMiddleware", + "id": 280 + }, + "parameters": [ + { + "id": 286, + "name": "crawler", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Crawler" + } + } + ] + } + ] + }, + { + "id": 287, + "name": "process_request", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Raises\": [{\"param\": \"ValueError\", \"desc\": \"If username and password are not provided in the proxy URL.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 67, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L67" + } + ], + "signatures": [ + { + "id": 288, + "name": "process_request", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process a Scrapy request by assigning a new proxy.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 289, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 290, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 291, + "name": "process_exception", + "module": "scrapy.middlewares.apify_proxy", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"Scrapy Request object.\"}, {\"param\": \"exception\", \"desc\": \"Exception object.\"}, {\"param\": \"spider\", \"desc\": \"Scrapy Spider object.\\n\"}]}, {\"Returns\": [\"Returning None, meaning Scrapy will continue processing this exception, executing any other\", \"process_exception() methods of installed middleware, until no middleware is left and the default\", \"exception handling kicks in.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 89, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L89" + } + ], + "signatures": [ + { + "id": 292, + "name": "process_exception", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Process an exception that occurs during request processing.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 293, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + }, + { + "id": 294, + "name": "exception", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Exception" + } + }, + { + "id": 295, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Constructors", + "children": [ + 281 + ] + }, + { + "title": "Methods", + "children": [ + 284, + 291, + 287 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py", + "line": 17, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/middlewares/apify_proxy.py#L17" + } + ] + }, + { + "id": 296, + "name": "ActorDatasetPushPipeline", + "module": "scrapy.pipelines.actor_dataset_push", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A Scrapy pipeline for pushing items to an Actor's default dataset.\\n\\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor.\"]}" + } + ] + }, + "children": [ + { + "id": 297, + "name": "process_item", + "module": "scrapy.pipelines.actor_dataset_push", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py", + "line": 22, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py#L22" + } + ], + "signatures": [ + { + "id": 298, + "name": "process_item", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Pushes the provided Scrapy item to the Actor's default dataset.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "Item" + }, + "parameters": [ + { + "id": 299, + "name": "item", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Item" + } + }, + { + "id": 300, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 297 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py", + "line": 16, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/pipelines/actor_dataset_push.py#L16" + } + ] + }, + { + "id": 301, + "name": "to_apify_request", + "module": "scrapy.requests", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"Arguments\": [{\"param\": \"scrapy_request\", \"desc\": \"The Scrapy request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Returns\": [\"The converted Apify request if the conversion was successful, otherwise None.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/requests.py", + "line": 19, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/requests.py#L19" + } + ], + "signatures": [ + { + "id": 302, + "name": "to_apify_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert a Scrapy request to an Apify request.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ApifyRequest | None" + }, + "parameters": [ + { + "id": 303, + "name": "scrapy_request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "ScrapyRequest" + } + }, + { + "id": 304, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 305, + "name": "to_scrapy_request", + "module": "scrapy.requests", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"Arguments\": [{\"param\": \"apify_request\", \"desc\": \"The Apify request to be converted.\"}, {\"param\": \"spider\", \"desc\": \"The Scrapy spider that the request is associated with.\\n\"}]}, {\"Raises\": [{\"param\": \"TypeError\", \"desc\": \"If the Apify request is not an instance of the `ApifyRequest` class.\"}, {\"param\": \"ValueError\", \"desc\": \"If the Apify request does not contain the required keys.\\n\"}]}, {\"Returns\": [\"The converted Scrapy request.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/requests.py", + "line": 82, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/requests.py#L82" + } + ], + "signatures": [ + { + "id": 306, + "name": "to_scrapy_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Convert an Apify request to a Scrapy request.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "ScrapyRequest" + }, + "parameters": [ + { + "id": 307, + "name": "apify_request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "ApifyRequest" + } + }, + { + "id": 308, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 309, + "name": "ApifyScheduler", + "module": "scrapy.scheduler", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\\n\\nThis scheduler requires the asyncio Twisted reactor to be installed.\"]}" + } + ] + }, + "children": [ + { + "id": 310, + "name": "open", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the scheduler.\\n\", {\"Arguments\": [{\"param\": \"spider\", \"desc\": \"The spider that the scheduler is associated with.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 43, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L43" + } + ], + "signatures": [ + { + "id": 311, + "name": "open", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Open the scheduler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Deferred[None] | None" + }, + "parameters": [ + { + "id": 312, + "name": "spider", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Spider" + } + } + ] + } + ] + }, + { + "id": 313, + "name": "close", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the scheduler.\\n\\nShut down the event loop and its thread gracefully.\\n\", {\"Arguments\": [{\"param\": \"reason\", \"desc\": \"The reason for closing the spider.\"}]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 66, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L66" + } + ], + "signatures": [ + { + "id": 314, + "name": "close", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Close the scheduler.\\n\\nShut down the event loop and its thread gracefully.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "None" + }, + "parameters": [ + { + "id": 315, + "name": "reason", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + } + ] + } + ] + }, + { + "id": 316, + "name": "has_pending_requests", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"Returns\": [\"True if the scheduler has any pending requests, False otherwise.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 87, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L87" + } + ], + "signatures": [ + { + "id": 317, + "name": "has_pending_requests", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Check if the scheduler has any pending requests.\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [] + } + ] + }, + { + "id": 318, + "name": "enqueue_request", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"Arguments\": [{\"param\": \"request\", \"desc\": \"The request to add to the scheduler.\\n\"}]}, {\"Returns\": [\"True if the request was successfully enqueued, False otherwise.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 104, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L104" + } + ], + "signatures": [ + { + "id": 319, + "name": "enqueue_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Add a request to the scheduler.\\n\\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\\n\", {\"" + } + ] + }, + "type": { + "type": "intrinsic", + "name": "bool" + }, + "parameters": [ + { + "id": 320, + "name": "request", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "Request" + } + } + ] + } + ] + }, + { + "id": 321, + "name": "next_request", + "module": "scrapy.scheduler", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"Returns\": [\"The next request, or None if there are no more requests.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 138, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L138" + } + ], + "signatures": [ + { + "id": 322, + "name": "next_request", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Fetch the next request from the scheduler.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Request | None" + }, + "parameters": [] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 313, + 318, + 316, + 321, + 310 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/scheduler.py", + "line": 24, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/scheduler.py#L24" + } + ] + }, + { + "id": 323, + "name": "get_basic_auth_header", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/utils.py", + "line": 14, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/utils.py#L14" + } + ], + "signatures": [ + { + "id": 324, + "name": "get_basic_auth_header", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Generate a basic authentication header for the given username and password.\"]}" + } + ] + }, + "type": { + "type": "reference", + "name": "bytes" + }, + "parameters": [ + { + "id": 325, + "name": "username", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 326, + "name": "password", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + } + }, + { + "id": 327, + "name": "auth_encoding", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "intrinsic", + "name": "str" + }, + "defaultValue": "'latin-1'" + } + ] + } + ] + }, + { + "id": 328, + "name": "apply_apify_settings", + "module": "scrapy.utils", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"Arguments\": [{\"param\": \"settings\", \"desc\": \"Scrapy project settings to be modified.\"}, {\"param\": \"proxy_config\", \"desc\": \"Proxy configuration to be stored in the settings.\\n\"}]}, {\"Returns\": [\"Scrapy project settings with custom configurations.\"]}]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/scrapy/utils.py", + "line": 21, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/scrapy/utils.py#L21" + } + ], + "signatures": [ + { + "id": 329, + "name": "apply_apify_settings", + "modifiers": [], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Integrates Apify configuration into a Scrapy project settings.\\n\\nNote: The function directly modifies the passed `settings` object and also returns it.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "Settings" + }, + "parameters": [ + { + "id": 330, + "name": "settings", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "Settings | None" + }, + "defaultValue": "None" + }, + { + "id": 331, + "name": "proxy_config", + "kind": 32768, + "kindString": "Parameter", + "flags": { + "keyword-only": "true" + }, + "type": { + "type": "reference", + "name": "dict | None" + }, + "defaultValue": "None" + } + ] + } + ] + }, + { + "id": 332, + "name": "RequestList", + "module": "storages._request_list", + "docsGroup": "Classes", + "kind": 128, + "kindString": "Class", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Extends crawlee RequestList.\\n\\nMethod open is used to create RequestList from actor's requestListSources input.\"]}" + } + ] + }, + "children": [ + { + "id": 333, + "name": "open", + "module": "storages._request_list", + "kind": 2048, + "kindString": "Method", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize a new instance from request list source input.\\n\", {\"Arguments\": [{\"param\": \"name\", \"desc\": \"Name of the returned RequestList.\"}, {\"param\": \"request_list_sources_input\", \"desc\": \"List of dicts with either url key or requestsFromUrl key.\"}, {\"param\": \"http_client\", \"desc\": \"Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\\n\"}]}, {\"Returns\": [\"RequestList created from request_list_sources_input.\", \"\"]}, \"### Usage\\n\\n```python\\nexample_input = [\\n # Gather urls from response body.\\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\\n # Directly include this url.\\n {'url': 'https://crawlee.dev', 'method': 'GET'}\\n]\\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\\n```\"]}" + } + ] + }, + "children": [], + "groups": [], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/storages/_request_list.py", + "line": 49, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/storages/_request_list.py#L49" + } + ], + "signatures": [ + { + "id": 334, + "name": "open", + "modifiers": [ + "async" + ], + "kind": 4096, + "kindString": "Call signature", + "flags": {}, + "comment": { + "summary": [ + { + "kind": "text", + "text": "{\"content\": [\"Initialize a new instance from request list source input.\\n\", {\"" + } + ] + }, + "type": { + "type": "reference", + "name": "RequestList", + "id": 332 + }, + "parameters": [ + { + "id": 335, + "name": "name", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "str | None" + }, + "defaultValue": "None" + }, + { + "id": 336, + "name": "request_list_sources_input", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "list[dict[str, Any]] | None" + }, + "defaultValue": "None" + }, + { + "id": 337, + "name": "http_client", + "kind": 32768, + "kindString": "Parameter", + "flags": {}, + "type": { + "type": "reference", + "name": "HttpClient | None" + }, + "defaultValue": "None" + } + ] + } + ] + } + ], + "groups": [ + { + "title": "Methods", + "children": [ + 333 + ] + } + ], + "sources": [ + { + "filename": "/tmp-old-source-v27/src/apify/storages/_request_list.py", + "line": 42, + "character": 1, + "url": "https://github.com/apify/apify-sdk-python/blob/v2.7.3/tmp-old-source-v27/src/apify/storages/_request_list.py#L42" + } + ] + } + ], + "groups": [ + { + "title": "Classes", + "children": [ + 1, + 225, + 180, + 203, + 211, + 208, + 332 + ] + }, + { + "title": "Properties", + "children": [ + 158 + ] + }, + { + "title": "Interfaces", + "children": [ + 159 + ] + }, + { + "title": "Data structures", + "children": [ + 198, + 175, + 193, + 189, + 191, + 190, + 192, + 171, + 201, + 200, + 199, + 197, + 194, + 196, + 195, + 202, + 188 + ] + }, + { + "title": "Methods", + "children": [ + 328, + 219, + 222, + 185, + 270, + 323, + 276, + 246, + 273, + 235, + 301, + 266, + 305 + ] + }, + { + "title": "Other", + "children": [ + 296, + 248, + 280, + 309, + 238, + 226, + 227, + 228, + 232, + 233, + 234 + ] + } + ], + "sources": [ + { + "fileName": "src/index.ts", + "line": 1, + "character": 0, + "url": "http://example.com/blob/123456/src/dummy.py" + } + ] +} \ No newline at end of file diff --git a/website/versioned_docs/version-3.3/.gitignore b/website/versioned_docs/version-3.3/.gitignore new file mode 100644 index 00000000..1a13c363 --- /dev/null +++ b/website/versioned_docs/version-3.3/.gitignore @@ -0,0 +1 @@ +changelog.md diff --git a/website/versioned_docs/version-3.3/01_introduction/code/01_introduction.py b/website/versioned_docs/version-3.3/01_introduction/code/01_introduction.py new file mode 100644 index 00000000..3c875170 --- /dev/null +++ b/website/versioned_docs/version-3.3/01_introduction/code/01_introduction.py @@ -0,0 +1,24 @@ +import asyncio + +import httpx +from bs4 import BeautifulSoup + +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() or {} + url = actor_input.get('url', 'https://apify.com') + async with httpx.AsyncClient() as client: + response = await client.get(url) + soup = BeautifulSoup(response.content, 'html.parser') + data = { + 'url': url, + 'title': soup.title.string if soup.title else None, + } + await Actor.push_data(data) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/__init__.py b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/__main__.py b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/__main__.py new file mode 100644 index 00000000..8c4ab0b8 --- /dev/null +++ b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/__main__.py @@ -0,0 +1,6 @@ +import asyncio + +from .main import main + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/main.py b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/main.py new file mode 100644 index 00000000..1fb4b610 --- /dev/null +++ b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/main.py @@ -0,0 +1,8 @@ +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + await Actor.set_value('OUTPUT', 'Hello, world!') diff --git a/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/py.typed b/website/versioned_docs/version-3.3/01_introduction/code/actor_structure/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-3.3/01_introduction/images/apify-create.gif b/website/versioned_docs/version-3.3/01_introduction/images/apify-create.gif new file mode 100644 index 00000000..776643e6 Binary files /dev/null and b/website/versioned_docs/version-3.3/01_introduction/images/apify-create.gif differ diff --git a/website/versioned_docs/version-3.3/01_introduction/index.mdx b/website/versioned_docs/version-3.3/01_introduction/index.mdx new file mode 100644 index 00000000..33feb04a --- /dev/null +++ b/website/versioned_docs/version-3.3/01_introduction/index.mdx @@ -0,0 +1,48 @@ +--- +id: introduction +title: Overview +sidebar_label: Overview +slug: /overview +description: 'The official library for creating Apify Actors in Python, providing tools for web scraping, automation, and data storage integration.' +--- + +The Apify SDK for Python is the official library for creating [Apify Actors](https://docs.apify.com/platform/actors) in Python. It provides useful features like Actor lifecycle management, local storage emulation, and Actor event handling. + +```python +from apify import Actor +from bs4 import BeautifulSoup +import requests + +async def main(): + async with Actor: + input = await Actor.get_input() + response = requests.get(input['url']) + soup = BeautifulSoup(response.content, 'html.parser') + await Actor.push_data({ 'url': input['url'], 'title': soup.title.string }) +``` + +## What are Actors + +Actors are serverless cloud programs capable of performing tasks in a web browser, similar to what a human can do. These tasks can range from simple operations, such as filling out forms or unsubscribing from services, to complex jobs like scraping and processing large numbers of web pages. + +Actors can be executed locally or on the [Apify platform](https://docs.apify.com/platform). The Apify platform lets you run Actors at scale and provides features for monitoring, scheduling, publishing, and monetizing them. + +## Quick start + +To create and run Actors using Apify Console, check out [Apify Console documentation](https://docs.apify.com/platform/console). For creating and running Python Actors locally, refer to the [quick start guide](./quick-start). + +Explore the Guides section in the sidebar for a deeper understanding of the SDK's features and best practices. + +## Installation + +The Apify SDK for Python requires Python version 3.10 or above. It is typically installed when you create a new Actor project using the [Apify CLI](https://docs.apify.com/cli). To install it manually in an existing project, use: + +```bash +pip install apify +``` + +:::note API client alternative + +If you need to interact with the Apify API programmatically without creating Actors, use the [Apify API client for Python](https://docs.apify.com/api/client/python) instead. + +::: diff --git a/website/versioned_docs/version-3.3/01_introduction/quick-start.mdx b/website/versioned_docs/version-3.3/01_introduction/quick-start.mdx new file mode 100644 index 00000000..3c991045 --- /dev/null +++ b/website/versioned_docs/version-3.3/01_introduction/quick-start.mdx @@ -0,0 +1,116 @@ +--- +id: quick-start +title: Quick start +sidebar_label: Quick start +description: 'Get started with the Apify SDK for Python by creating your first Actor and learning the basics.' +--- + +Learn how to create and run Actors using the Apify SDK for Python. + +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +## Step 1: Create Actors + +To create and run Actors in Apify Console, refer to the [Console documentation](/platform/actors/development/quick-start/web-ide). + +To create a new Apify Actor on your computer, you can use the [Apify CLI](/cli), and select one of the [Python Actor templates](https://apify.com/templates?category=python). + +For example, to create an Actor from the "[beta] Python SDK" template, you can use the [`apify create` command](/cli/docs/reference#apify-create-actorname). + +```bash +apify create my-first-actor --template python-start +``` + +This will create a new folder called `my-first-actor`, download and extract the "Getting started with Python" Actor template there, create a virtual environment in `my-first-actor/.venv`, and install the Actor dependencies in it. + +![actor create command run](./images/apify-create.gif) + +## Step 2: Run Actors + +To run the Actor, you can use the [`apify run` command](/cli/docs/reference#apify-run): + +```bash +cd my-first-actor +apify run +``` + +This command: + +- Activates the virtual environment in `.venv` (if no other virtual environment is activated yet) +- Starts the Actor with the appropriate environment variables for local running +- Configures it to use local storages from the `storage` folder + +The Actor input, for example, will be in `storage/key_value_stores/default/INPUT.json`. + +## Step 3: Understand Actor structure + +All Python Actor templates follow the same structure. + +The `.actor` directory contains the [Actor configuration](/platform/actors/development/actor-config), such as the Actor's definition and input schema, and the Dockerfile necessary to run the Actor on the Apify platform. + +The Actor's runtime dependencies are specified in the `requirements.txt` file, which follows the [standard requirements file format](https://pip.pypa.io/en/stable/reference/requirements-file-format/). + +The Actor's source code is in the `src` folder. This folder contains two important files: + +- `main.py` - which contains the main function of the Actor +- `__main__.py` - which is the entrypoint of the Actor package setting up the Actor [logger](../concepts/logging) and executing the Actor's main function via [`asyncio.run()`](https://docs.python.org/3/library/asyncio-runner.html#asyncio.run). + +<Tabs> + <TabItem value="main.py" label="main.py" default> + <CodeBlock language="python">{ +`from apify import Actor +${''} +async def main(): + async with Actor: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!')` + }</CodeBlock> + </TabItem> + <TabItem value="__main__.py" label="__main.py__"> + <CodeBlock language="python">{ +`import asyncio +import logging +${''} +from apify.log import ActorLogFormatter +${''} +from .main import main +${''} +handler = logging.StreamHandler() +handler.setFormatter(ActorLogFormatter()) +${''} +apify_logger = logging.getLogger('apify') +apify_logger.setLevel(logging.DEBUG) +apify_logger.addHandler(handler) +${''} +asyncio.run(main())` + }</CodeBlock> + </TabItem> +</Tabs> + +If you want to modify the Actor structure, you need to make sure that your Actor is executable as a module, via `python -m src`, as that is the command started by `apify run` in the Apify CLI. +We recommend keeping the entrypoint for the Actor in the `src/__main__.py` file. + +## Next steps + +### Guides + +To see how you can integrate the Apify SDK with some of the most popular web scraping libraries, check out our guides for working with: + +- [Requests or HTTPX](../guides/requests-and-httpx) +- [Beautiful Soup](../guides/beautiful-soup) +- [Playwright](../guides/playwright) +- [Selenium](../guides/selenium) +- [Scrapy](../guides/scrapy) + +### Usage concepts + +To learn more about the features of the Apify SDK and how to use them, check out the Usage Concepts section in the sidebar, especially the guides for: + +- [Actor lifecycle](../concepts/actor-lifecycle) +- [Working with storages](../concepts/storages) +- [Handling Actor events](../concepts/actor-events) +- [How to use proxies](../concepts/proxy-management) diff --git a/website/versioned_docs/version-3.3/02_concepts/01_actor_lifecycle.mdx b/website/versioned_docs/version-3.3/02_concepts/01_actor_lifecycle.mdx new file mode 100644 index 00000000..bf07cf5d --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/01_actor_lifecycle.mdx @@ -0,0 +1,101 @@ +--- +id: actor-lifecycle +title: Actor lifecycle +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import ClassContextExample from '!!raw-loader!roa-loader!./code/01_class_context.py'; +import ClassManualExample from '!!raw-loader!roa-loader!./code/01_class_manual.py'; +import InstanceContextExample from '!!raw-loader!roa-loader!./code/01_instance_context.py'; +import InstanceManualExample from '!!raw-loader!roa-loader!./code/01_instance_manual.py'; + +import ErrorHandlingContextExample from '!!raw-loader!roa-loader!./code/01_error_handling_context.py'; +import ErrorHandlingManualExample from '!!raw-loader!roa-loader!./code/01_error_handling_manual.py'; + +import RebootExample from '!!raw-loader!roa-loader!./code/01_reboot.py'; + +import StatusMessageExample from '!!raw-loader!roa-loader!./code/01_status_message.py'; + +This guide explains how an **Apify Actor** starts, runs, and shuts down, describing the complete Actor lifecycle. For information about the core concepts such as Actors, the Apify Console, storages, and events, check out the [Apify platform documentation](https://docs.apify.com/platform). + +## Actor initialization + +During initialization, the SDK prepares all the components required to integrate with the Apify platform. It loads configuration from environment variables, initializes access to platform storages such as the [key-value store, dataset, and request queue](https://docs.apify.com/platform/storage), sets up event handling for [platform events](https://docs.apify.com/platform/integrations/webhooks/events), and configures logging. + +The recommended approach in Python is to use the global [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) class as an asynchronous context manager. This approach automatically manages setup and teardown and keeps your code concise. When entering the context, the SDK loads configuration and initializes clients lazily—for example, a dataset is opened only when it is first accessed. If the Actor runs on the Apify platform, it also begins listening for platform events. + +When the Actor exits, either normally or due to an exception, the SDK performs a graceful shutdown. It persists the final Actor state, stops event handling, and sets the terminal exit code together with the [status message](https://docs.apify.com/platform/actors/development/programming-interface/status-messages). + +<Tabs groupId="request_queue"> + <TabItem value="actor_class_with_context_manager" label="Actor class with context manager" default> + <RunnableCodeBlock className="language-python" language="python"> + {ClassContextExample} + </RunnableCodeBlock> + </TabItem> + <TabItem value="actor_class_with_manual_init_exit" label="Actor class with manual init/exit"> + <RunnableCodeBlock className="language-python" language="python"> + {ClassManualExample} + </RunnableCodeBlock> + </TabItem> +</Tabs> + +You can also create an [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) instance directly. This does not change its capabilities but allows you to specify optional parameters during initialization, such as disabling automatic `sys.exit()` calls or customizing timeouts. The choice between using a context manager or manual initialization depends on how much control you require over the Actor's startup and shutdown sequence. + +<Tabs groupId="request_queue"> + <TabItem value="actor_instance_with_context_manager" label="Actor instance with context manager" default> + <RunnableCodeBlock className="language-python" language="python"> + {InstanceContextExample} + </RunnableCodeBlock> + </TabItem> + <TabItem value="actor_instance_with_manual_init_exit" label="Actor instance with manual init/exit"> + <RunnableCodeBlock className="language-python" language="python"> + {InstanceManualExample} + </RunnableCodeBlock> + </TabItem> +</Tabs> + +## Error handling + +Good error handling lets your Actor fail fast on critical errors, retry transient issues safely, and keep data consistent. Normally you rely on the `async with Actor:` block—if it finishes, the run succeeds (exit code 0); if an unhandled exception occurs, the run fails (exit code 1). + +The SDK provides helper methods for explicit control: + +- [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit) - terminates the run successfully (default exit code 0). +- [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail) - marks the run as failed (default exit code 1). + +Any non-zero exit code is treated as a `FAILED` run. You rarely need to call these methods directly unless you want to perform a controlled shutdown or customize the exit behavior. + +Catch exceptions only when necessary - for example, to retry network timeouts or map specific errors to exit codes. Keep retry loops bounded with backoff and re-raise once exhausted. Make your processing idempotent so that restarts don't corrupt results. Both [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit) and [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail) perform the same cleanup, so complete any long-running persistence before calling them. + +Below is a minimal context-manager example where an unhandled exception automatically fails the run, followed by a manual pattern giving you more control. + +<RunnableCodeBlock className="language-python" language="python">{ErrorHandlingContextExample}</RunnableCodeBlock> + +If you need explicit control over exit codes or status messages, you can manage the Actor manually using [`Actor.init`](https://docs.apify.com/sdk/python/reference/class/Actor#init), [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit), and [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail). + +<RunnableCodeBlock className="language-python" language="python">{ErrorHandlingManualExample}</RunnableCodeBlock> + +## Reboot + +Rebooting (available on the Apify platform only) instructs the platform worker to restart your Actor from the beginning of its execution. Use this mechanism only for transient conditions that are likely to resolve after a fresh start — for example, rotating a blocked proxy pool or recovering from a stuck browser environment. + +Before triggering a reboot, persist any essential state externally (e.g., to the key-value store or dataset), as all in-memory data is lost after reboot. The example below tracks a reboot counter in the default key-value store and allows at most three restarts before exiting normally. + +<RunnableCodeBlock className="language-python" language="python">{RebootExample}</RunnableCodeBlock> + +## Status message + +[Status messages](https://docs.apify.com/platform/actors/development/programming-interface/status-messages) are lightweight, human-readable progress indicators displayed with the Actor run on the Apify platform (separate from logs). Use them to communicate high-level phases or milestones, such as "Fetching list", "Processed 120/500 pages", or "Uploading results". + +Update the status only when the user's understanding of progress changes - avoid frequent updates for every processed item. Detailed information should go to logs or storages (dataset, key-value store) instead. + +The SDK optimizes updates by sending an API request only when the message text changes, so repeating the same message incurs no additional cost. + +<RunnableCodeBlock className="language-python" language="python">{StatusMessageExample}</RunnableCodeBlock> + +## Conclusion + +This page has presented the full Actor lifecycle: initialization, execution, error handling, rebooting, shutdown and status messages. You've seen how the SDK supports both context-based and manual control patterns. For deeper dives, explore the [reference docs](https://docs.apify.com/sdk/python/reference), [guides](https://docs.apify.com/sdk/python/docs/guides/beautifulsoup-httpx), and [platform documentation](https://docs.apify.com/platform). diff --git a/website/versioned_docs/version-3.3/02_concepts/02_actor_input.mdx b/website/versioned_docs/version-3.3/02_concepts/02_actor_input.mdx new file mode 100644 index 00000000..3a4deabb --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/02_actor_input.mdx @@ -0,0 +1,18 @@ +--- +id: actor-input +title: Actor input +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import InputExample from '!!raw-loader!roa-loader!./code/02_input.py'; + +The Actor gets its [input](https://docs.apify.com/platform/actors/running/input) from the input record in its default [key-value store](https://docs.apify.com/platform/storage/key-value-store). + +To access it, instead of reading the record manually, you can use the [`Actor.get_input`](../../reference/class/Actor#get_input) convenience method. It will get the input record key from the Actor configuration, read the record from the default key-value store,and decrypt any [secret input fields](https://docs.apify.com/platform/actors/development/secret-input). + +For example, if an Actor received a JSON input with two fields, `{ "firstNumber": 1, "secondNumber": 2 }`, this is how you might process it: + +<RunnableCodeBlock className="language-python" language="python"> + {InputExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/03_storages.mdx b/website/versioned_docs/version-3.3/02_concepts/03_storages.mdx new file mode 100644 index 00000000..193ce36f --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/03_storages.mdx @@ -0,0 +1,173 @@ +--- +id: storages +title: Working with storages +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import OpeningStoragesExample from '!!raw-loader!roa-loader!./code/03_opening_storages.py'; +import DeletingStoragesExample from '!!raw-loader!roa-loader!./code/03_deleting_storages.py'; +import DatasetReadWriteExample from '!!raw-loader!roa-loader!./code/03_dataset_read_write.py'; +import DatasetExportsExample from '!!raw-loader!roa-loader!./code/03_dataset_exports.py'; +import KvsReadWriteExample from '!!raw-loader!roa-loader!./code/03_kvs_read_write.py'; +import KvsIteratingExample from '!!raw-loader!roa-loader!./code/03_kvs_iterating.py'; +import KvsPublicRecordExample from '!!raw-loader!roa-loader!./code/03_kvs_public_url.py'; +import RqExample from '!!raw-loader!roa-loader!./code/03_rq.py'; + +The `Actor` class provides methods to work either with the default storages of the Actor, or with any other storage, named or unnamed. + +## Types of storages + +There are three types of storages available to Actors. + +First are [datasets](https://docs.apify.com/platform/storage/dataset), which are append-only tables for storing the results of your Actors. You can open a dataset through the [`Actor.open_dataset`](../../reference/class/Actor#open_dataset) method, and work with it through the resulting [`Dataset`](../../reference/class/Dataset) class instance. + +Next there are [key-value stores](https://docs.apify.com/platform/storage/key-value-store), which function as a read/write storage for storing file-like objects, typically the Actor state or binary results. You can open a key-value store through the [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store) method, and work with it through the resulting [`KeyValueStore`](../../reference/class/KeyValueStore) class instance. + +Finally, there are [request queues](https://docs.apify.com/platform/storage/request-queue). These are queues into which you can put the URLs you want to scrape, and from which the Actor can dequeue them and process them. You can open a request queue through the [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) method, and work with it through the resulting [`RequestQueue`](../../reference/class/RequestQueue) class instance. + +Each Actor run has its default dataset, default key-value store and default request queue. + +## Local storage emulation + +To be able to develop Actors locally, the storages that the Apify platform provides are emulated on the local filesystem. + +The storage contents are loaded from and saved to the `storage` folder in the Actor's main folder. Each storage type is stored in its own subfolder, so for example datasets are stored in the `storage/datasets` folder. + +Each storage is then stored in its own folder, named after the storage, or called `default` if it's the default storage. For example, a request queue with the name `my-queue` would be stored in `storage/request_queues/my-queue`. + +Each dataset item, key-value store record, or request in a request queue is then stored in its own file in the storage folder. Dataset items and request queue requests are always JSON files, and key-value store records can be any file type, based on its content type. For example, the Actor input is typically stored in `storage/key_value_stores/default/INPUT.json`. + +## Local Actor run with remote storage + +When developing locally, opening any storage will by default use local storage. To change this behavior and to use remote storage you have to use `force_cloud=True` argument in [`Actor.open_dataset`](../../reference/class/Actor#open_dataset), [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) or [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store). Proper use of this argument allows you to work with both local and remote storages. + +Calling another remote Actor and accessing its default storage is typical use-case for using `force-cloud=True` argument to open remote Actor's storages. + +### Local storage persistence + +By default, the storage contents are persisted across multiple Actor runs. To clean up the Actor storages before the running the Actor, use the `--purge` flag of the [`apify run`](https://docs.apify.com/cli/docs/reference#apify-run) command of the Apify CLI. + +```bash +apify run --purge +``` + +## Convenience methods for working with default storages + +There are several methods for directly working with the default key-value store or default dataset of the Actor. + +- [`Actor.get_value('my-record')`](../../reference/class/Actor#get_value) reads a record from the default key-value store of the Actor. +- [`Actor.set_value('my-record', 'my-value')`](../../reference/class/Actor#set_value) saves a new value to the record in the default key-value store. +- [`Actor.get_input`](../../reference/class/Actor#get_input) reads the Actor input from the default key-value store of the Actor. +- [`Actor.push_data([{'result': 'Hello, world!'}, ...])`](../../reference/class/Actor#push_data) saves results to the default dataset of the Actor. + +## Opening named and unnamed storages + +The [`Actor.open_dataset`](../../reference/class/Actor#open_dataset), [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store) and [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) methods can be used to open any storage for reading and writing. You can either use them without arguments to open the default storages, or you can pass a storage ID or name to open another storage. + +<RunnableCodeBlock className="language-python" language="python"> + {OpeningStoragesExample} +</RunnableCodeBlock> + +## Deleting storages + +To delete a storage, you can use the [`Dataset.drop`](../../reference/class/Dataset#drop), +[`KeyValueStore.drop`](../../reference/class/KeyValueStore#drop) or [`RequestQueue.drop`](../../reference/class/RequestQueue#drop) methods. + +<RunnableCodeBlock className="language-python" language="python"> + {DeletingStoragesExample} +</RunnableCodeBlock> + +## Working with datasets + +In this section we will show you how to work with [datasets](https://docs.apify.com/platform/storage/dataset). + +### Reading & writing items + +To write data into a dataset, you can use the [`Dataset.push_data`](../../reference/class/Dataset#push_data) method. + +To read data from a dataset, you can use the [`Dataset.get_data`](../../reference/class/Dataset#get_data) method. + +To get an iterator of the data, you can use the [`Dataset.iterate_items`](../../reference/class/Dataset#iterate_items) method. + +<RunnableCodeBlock className="language-python" language="python"> + {DatasetReadWriteExample} +</RunnableCodeBlock> + +### Exporting items + +You can also export the dataset items into a key-value store, as either a CSV or a JSON record, +using the [`Dataset.export_to_csv`](../../reference/class/Dataset#export_to_csv) +or [`Dataset.export_to_json`](../../reference/class/Dataset#export_to_json) method. + +<RunnableCodeBlock className="language-python" language="python"> + {DatasetExportsExample} +</RunnableCodeBlock> + +## Working with key-value stores + +In this section we will show you how to work with [key-value stores](https://docs.apify.com/platform/storage/key-value-store). + +### Reading and writing records + +To read records from a key-value store, you can use the [`KeyValueStore.get_value`](../../reference/class/KeyValueStore#get_value) method. + +To write records into a key-value store, you can use the [`KeyValueStore.set_value`](../../reference/class/KeyValueStore#set_value) method. +You can set the content type of a record with the `content_type` argument. +To delete a record, set its value to `None`. + +<RunnableCodeBlock className="language-python" language="python"> + {KvsReadWriteExample} +</RunnableCodeBlock> + +### Iterating keys + +To get an iterator of the key-value store record keys, +you can use the [`KeyValueStore.iterate_keys`](../../reference/class/KeyValueStore#iterate_keys) method. + +<RunnableCodeBlock className="language-python" language="python"> + {KvsIteratingExample} +</RunnableCodeBlock> + +### Public URLs of records + +To get a publicly accessible URL of a key-value store record, +you can use the [`KeyValueStore.get_public_url`](../../reference/class/KeyValueStore#get_public_url) method. + +<RunnableCodeBlock className="language-python" language="python"> + {KvsPublicRecordExample} +</RunnableCodeBlock> + +## Working with request queues + +In this section we will show you how to work with [request queues](https://docs.apify.com/platform/storage/request-queue). + +### Adding requests to a queue + +To add a request into the queue, you can use the [`RequestQueue.add_request`](../../reference/class/RequestQueue#add_request) method. + +You can use the `forefront` boolean argument to specify whether the request should go to the beginning of the queue, or to the end. + +You can use the `unique_key` of the request to uniquely identify a request. If you try to add more requests with the same unique key, only the first one will be added. + +Check out the [`Request`](../../reference/class/Request) for more information on how to create requests and what properties they have. + +### Reading requests + +To fetch the next request from the queue for processing, you can use the [`RequestQueue.fetch_next_request`](../../reference/class/RequestQueue#fetch_next_request) method. + +To get info about a specific request from the queue, you can use the [`RequestQueue.get_request`](../../reference/class/RequestQueue#get_request) method. + +### Handling requests + +To mark a request as handled, you can use the [`RequestQueue.mark_request_as_handled`](../../reference/class/RequestQueue#mark_request_as_handled) method. + +To mark a request as not handled, so that it gets retried, you can use the [`RequestQueue.reclaim_request`](../../reference/class/RequestQueue#reclaim_request) method. + +To check if all the requests in the queue are handled, you can use the [`RequestQueue.is_finished`](../../reference/class/RequestQueue#is_finished) method. + +### Full example + +<RunnableCodeBlock className="language-python" language="python"> + {RqExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/04_actor_events.mdx b/website/versioned_docs/version-3.3/02_concepts/04_actor_events.mdx new file mode 100644 index 00000000..9ce7fd59 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/04_actor_events.mdx @@ -0,0 +1,81 @@ +--- +id: actor-events +title: Actor events & state persistence +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ActorEventsExample from '!!raw-loader!roa-loader!./code/04_actor_events.py'; + +During its runtime, the Actor receives Actor events sent by the Apify platform or generated by the Apify SDK itself. + +## Event types + +<table> + <thead> + <tr> + <th>Event</th> + <th>Data</th> + <th>Description</th> + </tr> + </thead> + <tbody> + <tr> + <td><code>SYSTEM_INFO</code></td> + <td><pre>{`{ + "created_at": datetime, + "cpu_current_usage": float, + "mem_current_bytes": int, + "is_cpu_overloaded": bool +}`} + </pre></td> + <td> + <p>This event is emitted regularly and it indicates the current resource usage of the Actor.</p> + The <code>is_cpu_overloaded</code> argument indicates whether the current CPU usage is higher than <code>Config.max_used_cpu_ratio</code> + </td> + </tr> + <tr> + <td><code>MIGRATING</code></td> + <td><code>None</code></td> + <td> + <p>Emitted when the Actor running on the Apify platform + is going to be <a href="https://docs.apify.com/platform/actors/development/state-persistence#what-is-a-migration">migrated</a> + {' '}to another worker server soon.</p> + You can use it to persist the state of the Actor so that once it is executed again on the new server, + it doesn't have to start over from the beginning. + Once you have persisted the state of your Actor, you can call <a href="../../reference/class/Actor#reboot"><code>Actor.reboot</code></a> + to reboot the Actor and trigger the migration immediately, to speed up the process. + </td> + </tr> + <tr> + <td><code>ABORTING</code></td> + <td><code>None</code></td> + <td> + When a user aborts an Actor run on the Apify platform, + they can choose to abort gracefully to allow the Actor some time before getting killed. + This graceful abort emits the <code>ABORTING</code> event which you can use to finish all running tasks and do cleanup. + </td> + </tr> + <tr> + <td><code>PERSIST_STATE</code></td> + <td><pre>{`{ "is_migrating": bool }`}</pre></td> + <td> + <p>Emitted in regular intervals (by default 60 seconds) to notify the Actor that it should persist its state, + in order to avoid repeating all work when the Actor restarts.</p> + <p>This event is also emitted automatically when the <code>MIGRATING</code> event happens, + in which case the <code>is_migrating</code> flag is set to <code>True</code>.</p> + Note that the <code>PERSIST_STATE</code> event is provided merely for user convenience, + you can achieve the same effect by persisting the state regularly in an interval and listening for the migrating event. + </td> + </tr> + </tbody> +</table> + +## Adding handlers to events + +To add handlers to these events, you use the [`Actor.on`](../../reference/class/Actor#on) method, +and to remove them, you use the [`Actor.off`](../../reference/class/Actor#off) method. + +<RunnableCodeBlock className="language-python" language="python"> + {ActorEventsExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/05_proxy_management.mdx b/website/versioned_docs/version-3.3/02_concepts/05_proxy_management.mdx new file mode 100644 index 00000000..64420eee --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/05_proxy_management.mdx @@ -0,0 +1,108 @@ +--- +id: proxy-management +title: Proxy management +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ApifyProxyExample from '!!raw-loader!roa-loader!./code/05_apify_proxy.py'; +import CustomProxyExample from '!!raw-loader!roa-loader!./code/05_custom_proxy.py'; +import ProxyRotationExample from '!!raw-loader!roa-loader!./code/05_proxy_rotation.py'; +import ApifyProxyConfig from '!!raw-loader!roa-loader!./code/05_apify_proxy_config.py'; +import CustomProxyFunctionExample from '!!raw-loader!roa-loader!./code/05_custom_proxy_function.py'; +import ProxyActorInputExample from '!!raw-loader!roa-loader!./code/05_proxy_actor_input.py'; +import ProxyHttpxExample from '!!raw-loader!roa-loader!./code/05_proxy_httpx.py'; + +[IP address blocking](https://en.wikipedia.org/wiki/IP_address_blocking) is one of the oldest and most effective ways of preventing access to a website. It is therefore paramount for a good web scraping library to provide easy to use but powerful tools which can work around IP blocking. The most powerful weapon in your anti IP blocking arsenal is a [proxy server](https://en.wikipedia.org/wiki/Proxy_server). + +With the Apify SDK, you can use your own proxy servers, proxy servers acquired from third-party providers, or you can rely on [Apify Proxy](https://apify.com/proxy) for your scraping needs. + +## Quick start + +If you want to use Apify Proxy locally, make sure that you run your Actors via the Apify CLI and that you are [logged in](https://docs.apify.com/cli/docs/installation#login-with-your-apify-account) with your Apify account in the CLI. + +### Using Apify proxy + +<RunnableCodeBlock className="language-python" language="python"> + {ApifyProxyExample} +</RunnableCodeBlock> + +### Using your own proxies + +<RunnableCodeBlock className="language-python" language="python"> + {CustomProxyExample} +</RunnableCodeBlock> + +## Proxy configuration + +All your proxy needs are managed by the [`ProxyConfiguration`](../../reference/class/ProxyConfiguration) class. You create an instance using the [`Actor.create_proxy_configuration()`](../../reference/class/Actor#create_proxy_configuration) method. Then you generate proxy URLs using the [`ProxyConfiguration.new_url()`](../../reference/class/ProxyConfiguration#new_url) method. + +### Apify proxy vs. your own proxies + +The `ProxyConfiguration` class covers both Apify Proxy and custom proxy URLs, so that you can easily switch between proxy providers. However, some features of the class are available only to Apify Proxy users, mainly because Apify Proxy is what one would call a super-proxy. It's not a single proxy server, but an API endpoint that allows connectionthrough millions of different IP addresses. So the class essentially has two modes: Apify Proxy or Your proxy. + +The difference is easy to remember. Using the `proxy_url` or `new_url_function` arguments enables use of your custom proxy URLs, whereas all the other options are there to configure Apify Proxy. Visit the [Apify Proxy docs](https://docs.apify.com/proxy) for more info on how these parameters work. + +### IP rotation and session management + +`ProxyConfiguration.new_url` allows you to pass a `session_id` parameter. It will then be used to create a `session_id`-`proxy_url` pair, and subsequent `new_url()` calls with the same `session_id` will always return the same `proxy_url`. This is extremely useful in scraping, because you want to create the impression of a real user. + +When no `session_id` is provided, your custom proxy URLs are rotated round-robin, whereas Apify Proxy manages their rotation using black magic to get the best performance. + +<RunnableCodeBlock className="language-python" language="python"> + {ProxyRotationExample} +</RunnableCodeBlock> + +### Apify proxy configuration + +With Apify Proxy, you can select specific proxy groups to use, or countries to connect from. This allows you to get better proxy performance after some initial research. + +<RunnableCodeBlock className="language-python" language="python"> + {ApifyProxyConfig} +</RunnableCodeBlock> + +Now your connections using proxy_url will use only Residential proxies from the US. Note that you must first get access to a proxy group before you are able to use it. You can find your available proxy groups in the [proxy dashboard](https://console.apify.com/proxy). + +If you don't specify any proxy groups, automatic proxy selection will be used. + +### Your own proxy configuration + +There are two options how to make `ProxyConfiguration` work with your own proxies. + +Either you can pass it a list of your own proxy servers: + +<RunnableCodeBlock className="language-python" language="python"> + {CustomProxyExample} +</RunnableCodeBlock> + +Or you can pass it a method (accepting one optional argument, the session ID), to generate proxy URLs automatically: + +<RunnableCodeBlock className="language-python" language="python"> + {CustomProxyFunctionExample} +</RunnableCodeBlock> + +### Configuring proxy based on Actor input + +To make selecting the proxies that the Actor uses easier, you can use an input field with the editor [`proxy` in your input schema](https://docs.apify.com/platform/actors/development/input-schema#object). This input will then be filled with a dictionary containing the proxy settings you or the users of your Actor selected for the Actor run. + +You can then use that input to create the proxy configuration: + +<RunnableCodeBlock className="language-python" language="python"> + {ProxyActorInputExample} +</RunnableCodeBlock> + +## Using the generated proxy URLs + +### HTTPX + +To use the generated proxy URLs with the `httpx` library, use the [`proxies`](https://www.python-httpx.org/advanced/#http-proxying) argument: + +<RunnableCodeBlock className="language-python" language="python"> + {ProxyHttpxExample} +</RunnableCodeBlock> + +Make sure you have the `httpx` library installed: + +```bash +pip install httpx +``` diff --git a/website/versioned_docs/version-3.3/02_concepts/06_interacting_with_other_actors.mdx b/website/versioned_docs/version-3.3/02_concepts/06_interacting_with_other_actors.mdx new file mode 100644 index 00000000..880cbb89 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/06_interacting_with_other_actors.mdx @@ -0,0 +1,51 @@ +--- +id: interacting-with-other-actors +title: Interacting with other Actors +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import InteractingStartExample from '!!raw-loader!roa-loader!./code/06_interacting_start.py'; +import InteractingCallExample from '!!raw-loader!roa-loader!./code/06_interacting_call.py'; +import InteractingCallTaskExample from '!!raw-loader!roa-loader!./code/06_interacting_call_task.py'; +import InteractingMetamorphExample from '!!raw-loader!roa-loader!./code/06_interacting_metamorph.py'; + +There are several methods that interact with other Actors and Actor tasks on the Apify platform. + +## Actor start + +The [`Actor.start`](../../reference/class/Actor#start) method starts another Actor on the Apify platform, and immediately returns the details of the started Actor run. + +<RunnableCodeBlock className="language-python" language="python"> + {InteractingStartExample} +</RunnableCodeBlock> + +## Actor call + +The [`Actor.call`](../../reference/class/Actor#call) method starts another Actor on the Apify platform, and waits for the started Actor run to finish. + +<RunnableCodeBlock className="language-python" language="python"> + {InteractingCallExample} +</RunnableCodeBlock> + +## Actor call task + +The [`Actor.call_task`](../../reference/class/Actor#call_task) method starts an [Actor task](https://docs.apify.com/platform/actors/tasks) on the Apify platform, and waits for the started Actor run to finish. + +<RunnableCodeBlock className="language-python" language="python"> + {InteractingCallTaskExample} +</RunnableCodeBlock> + +## Actor metamorph + +The [`Actor.metamorph`](../../reference/class/Actor#metamorph) operation transforms an Actor run into a run of another Actor with a new input. This feature is useful if you want to use another Actor to finish the work of your current Actor, instead of internally starting a new Actor run and waiting for its finish. With metamorph, you can easily create new Actors on top of existing ones, and give your users nicer input structure and user interface for the final Actor. For the users of your Actors, the metamorph operation is completely transparent; they will just see your Actor got the work done. + +Internally, the system stops the container corresponding to the original Actor run and starts a new container using a different container image. All the default storages are preserved,and the new Actor input is stored under the `INPUT-METAMORPH-1` key in the same default key-value store. + +To make you Actor compatible with the metamorph operation, use [`Actor.get_input`](../../reference/class/Actor#get_input) instead of [`Actor.get_value('INPUT')`](../../reference/class/Actor#get_value) to read your Actor input. This method will fetch the input using the right key in a case of metamorphed run. + +For example, imagine you have an Actor that accepts a hotel URL on input, and then internally uses the [`apify/web-scraper`](https://apify.com/apify/web-scraper) public Actor to scrape all the hotel reviews. The metamorphing code would look as follows: + +<RunnableCodeBlock className="language-python" language="python"> + {InteractingMetamorphExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/07_webhooks.mdx b/website/versioned_docs/version-3.3/02_concepts/07_webhooks.mdx new file mode 100644 index 00000000..04697bbb --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/07_webhooks.mdx @@ -0,0 +1,31 @@ +--- +id: webhooks +title: Creating webhooks +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import WebhookExample from '!!raw-loader!roa-loader!./code/07_webhook.py'; +import WebhookPreventingExample from '!!raw-loader!roa-loader!./code/07_webhook_preventing.py'; + +Webhooks allow you to configure the Apify platform to perform an action when a certain event occurs. For example, you can use them to start another Actor when the current run finishes or fails. + +You can learn more in the [documentation for webhooks](https://docs.apify.com/platform/integrations/webhooks). + +## Creating an ad-hoc webhook dynamically + +Besides creating webhooks manually in Apify Console, or through the Apify API,you can also create [ad-hoc webhooks](https://docs.apify.com/platform/integrations/webhooks/ad-hoc-webhooks) dynamically from the code of your Actor using the [`Actor.add_webhook`](../../reference/class/Actor#add_webhook) method: + +<RunnableCodeBlock className="language-python" language="python"> + {WebhookExample} +</RunnableCodeBlock> + +Note that webhooks are only supported when running on the Apify platform. When running the Actor locally, the method will print a warning and have no effect. + +## Preventing duplicate webhooks + +To ensure that duplicate ad-hoc webhooks won't get created in a case of Actor restart, you can use the `idempotency_key` parameter. The idempotency key must be unique across all the webhooks of a user so that only one webhook gets created for a given value. You can use, for example, the Actor run ID as the idempotency key: + +<RunnableCodeBlock className="language-python" language="python"> + {WebhookPreventingExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/08_access_apify_api.mdx b/website/versioned_docs/version-3.3/02_concepts/08_access_apify_api.mdx new file mode 100644 index 00000000..ff6fefb1 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/08_access_apify_api.mdx @@ -0,0 +1,31 @@ +--- +id: access-apify-api +title: Accessing Apify API +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ActorClientExample from '!!raw-loader!roa-loader!./code/08_actor_client.py'; +import ActorNewClientExample from '!!raw-loader!roa-loader!./code/08_actor_new_client.py'; + +The Apify SDK contains many useful features for making Actor development easier. However, it does not cover all the features the Apify API offers. + +For working with the Apify API directly, you can use the provided instance of the [Apify API Client](https://docs.apify.com/api/client/python) library. + +## Actor client + +To access the provided instance of [`ApifyClientAsync`](https://docs.apify.com/api/client/python/reference/class/ApifyClientAsync), you can use the [`Actor.apify_client`](../../reference/class/Actor#apify_client) property. + +For example, to get the details of your user, you can use this snippet: + +<RunnableCodeBlock className="language-python" language="python"> + {ActorClientExample} +</RunnableCodeBlock> + +## Actor new client + +If you want to create a completely new instance of the client, for example, to get a client for a different user or change the configuration of the client,you can use the [`Actor.new_client`](../../reference/class/Actor#new_client) method: + +<RunnableCodeBlock className="language-python" language="python"> + {ActorNewClientExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/09_logging.mdx b/website/versioned_docs/version-3.3/02_concepts/09_logging.mdx new file mode 100644 index 00000000..c1bd26c6 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/09_logging.mdx @@ -0,0 +1,114 @@ +--- +id: logging +title: Logging +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import LogConfigExample from '!!raw-loader!roa-loader!./code/09_log_config.py'; +import LoggerUsageExample from '!!raw-loader!roa-loader!./code/09_logger_usage.py'; +import RedirectLog from '!!raw-loader!roa-loader!./code/09_redirect_log.py'; +import RedirectLogExistingRun from '!!raw-loader!roa-loader!./code/09_redirect_log_existing_run.py'; + +The Apify SDK is logging useful information through the [`logging`](https://docs.python.org/3/library/logging.html) module from Python's standard library, into the logger with the name `apify`. + +## Automatic configuration + +When you create an Actor from an Apify-provided template, either in Apify Console or through the Apify CLI, you do not have to configure the logger yourself. The template already contains initialization code for the logger,which sets the logger level to `DEBUG` and the log formatter to [`ActorLogFormatter`](../../reference/class/ActorLogFormatter). + +## Manual configuration + +### Configuring the log level + +In Python's default behavior, if you don't configure the logger otherwise, only logs with level `WARNING` or higher are printed out to the standard output, without any formatting. To also have logs with `DEBUG` and `INFO` level printed out, you need to call the [`Logger.setLevel`](https://docs.python.org/3/library/logging.html#logging.Logger.setLevel) method on the logger, with the desired minimum level as an argument. + +### Configuring the log formatting + +By default, only the log message is printed out to the output, without any formatting. To have a nicer output, with the log level printed in color, the messages nicely aligned, and extra log fields printed out,you can use the [`ActorLogFormatter`](../../reference/class/ActorLogFormatter) class from the `apify.log` module. + +### Example log configuration + +To configure and test the logger, you can use this snippet: + +<RunnableCodeBlock className="language-python" language="python"> + {LogConfigExample} +</RunnableCodeBlock> + +This configuration will cause all levels of messages to be printed to the standard output, with some pretty formatting. + +## Logger usage + +Here you can see how all the log levels would look like. + +You can use the `extra` argument for all log levels, it's not specific to the warning level. When you use `Logger.exception`, there is no need to pass the Exception object to the log manually, it will automatiacally infer it from the current execution context and print the exception details. + +<RunnableCodeBlock className="language-python" language="python"> + {LoggerUsageExample} +</RunnableCodeBlock> + +Result: + +<!-- TODO: This is an ugly ugly hack, we should make a component for this in the docs theme --> +<!-- markdownlint-disable no-inline-html --> +<style>{` +.actor-log-block .ansi-blue-fg { + color: rgb(0, 160, 228); +} +.actor-log-block .ansi-green-fg { + color: rgb(0, 162, 82); +} +.actor-log-block .ansi-red-fg { + color: rgb(219, 45, 32); +} +.actor-log-block .ansi-yellow-fg { + color: rgb(255, 201, 0); +} +.actor-log-block .ansi-bright-black-fg { + color: rgb(128, 128, 128); +} +[data-theme='dark'] .actor-log-block .ansi-yellow-fg { + color: rgb(253, 237, 0); +} + +`}</style> +<pre className="actor-log-block"> + <div><span class="ansi-blue-fg">DEBUG</span> This is a debug message</div> + <div><span class="ansi-green-fg">INFO </span> This is an info message</div> + <div><span class="ansi-yellow-fg">WARN </span> This is a warning message <span class="ansi-bright-black-fg">{`({"reason": "Bad Actor!"})`}</span></div> + <div><span class="ansi-red-fg">ERROR</span> This is an error message</div> + <div><span class="ansi-red-fg">ERROR</span> This is an exceptional message</div> + <div> Traceback (most recent call last):</div> + <div> File "main.py", line 6, in <module></div> + <div> raise RuntimeError('Ouch!')</div> + <div> RuntimeError: Ouch!</div> +</pre> + +<!-- markdownlint-enable no-inline-html --> + +## Redirect logs from other Actor runs + +In some situations, one Actor is going to start one or more other Actors and wait for them to finish and produce some results. In such cases, you might want to redirect the logs and status messages of the started Actors runs back to the parent Actor run, so that you can see the progress of the started Actors' runs in the parent Actor's logs. This guide will show possibilities on how to do it. + +### Redirecting logs from Actor.call + +Typical use case for log redirection is to call another Actor using the [`Actor.call`](../../reference/class/Actor#call) method. This method has an optional `logger` argument, which is by default set to the `default` literal. This means that the logs of the called Actor will be automatically redirected to the parent Actor's logs with default formatting and filtering. If you set the `logger` argument to `None`, then no log redirection happens. The third option is to pass your own `Logger` instance with the possibility to define your own formatter, filter, and handler. Below you can see those three possible ways of log redirection when starting another Actor run through [`Actor.call`](../../reference/class/Actor#call). + +<RunnableCodeBlock className="language-python" language="python"> + {RedirectLog} +</RunnableCodeBlock> + +Each default redirect logger log entry will have a specific format. After the timestamp, it will contain cyan colored text that will contain the redirect information - the other actor's name and the run ID. The rest of the log message will be printed in the same manner as the parent Actor's logger is configured. + +The log redirection can be deep, meaning that if the other actor also starts another actor and is redirecting logs from it, then in the top-level Actor, you can see it as well. See the following example screenshot of the Apify log console when one actor recursively starts itself (there are 2 levels of recursion in the example). + +![Console with redirected logs](/img/guides/redirected_logs_example.webp 'Example of console with redirected logs from recursively started actor.') + +### Redirecting logs from already running Actor run + +In some cases, you might want to connect to an already running Actor run and redirect its logs to your current Actor run. This can be done using the [ApifyClient](../../reference/class/Actor#apify_client) and getting the streamed log from a specific Actor run. You can then use it as a context manager, and the log redirection will be active in the context, or you can control the log redirection manually by explicitly calling `start` and `stop` methods. + +You can further decide whether you want to redirect just new logs of the ongoing Actor run, or if you also want to redirect historical logs from that Actor's run, so all logs it has produced since it was started. Both options are shown in the example code below. + +<RunnableCodeBlock className="language-python" language="python"> + {RedirectLogExistingRun} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/02_concepts/10_configuration.mdx b/website/versioned_docs/version-3.3/02_concepts/10_configuration.mdx new file mode 100644 index 00000000..4d1e83e8 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/10_configuration.mdx @@ -0,0 +1,34 @@ +--- +id: actor-configuration +title: Actor configuration +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ConfigExample from '!!raw-loader!roa-loader!./code/10_config.py'; + +The [`Actor`](../../reference/class/Actor) class gets configured using the [`Configuration`](../../reference/class/Configuration) class, which initializes itself based on the provided environment variables. + +If you're using the Apify SDK in your Actors on the Apify platform, or Actors running locally through the Apify CLI, you don't need to configure the `Actor` class manually,unless you have some specific requirements, everything will get configured automatically. + +If you need some special configuration, you can adjust it either through the `Configuration` class directly,or by setting environment variables when running the Actor locally. + +To see the full list of configuration options, check the `Configuration` class or the list of environment variables that the Actor understands. + +## Configuring from code + +This will cause the Actor to persist its state every 10 seconds: + +<RunnableCodeBlock className="language-python" language="python"> + {ConfigExample} +</RunnableCodeBlock> + +## Configuring via environment variables + +All the configuration options can be set via environment variables. The environment variables are prefixed with `APIFY_`, and the configuration options are in uppercase, with underscores as separators. See the [`Configuration`](../../reference/class/Configuration) API reference for the full list of configuration options. + +This Actor run will not persist its local storages to the filesystem: + +```bash +APIFY_PERSIST_STORAGE=0 apify run +``` diff --git a/website/versioned_docs/version-3.3/02_concepts/11_pay_per_event.mdx b/website/versioned_docs/version-3.3/02_concepts/11_pay_per_event.mdx new file mode 100644 index 00000000..0a422332 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/11_pay_per_event.mdx @@ -0,0 +1,69 @@ +--- +id: pay-per-event +title: Pay-per-event monetization +description: Monetize your Actors using the pay-per-event pricing model +--- + +import ActorChargeSource from '!!raw-loader!roa-loader!./code/11_actor_charge.py'; +import ConditionalActorChargeSource from '!!raw-loader!roa-loader!./code/11_conditional_actor_charge.py'; +import ChargeLimitCheckSource from '!!raw-loader!roa-loader!./code/11_charge_limit_check.py'; +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +Apify provides several [pricing models](https://docs.apify.com/platform/actors/publishing/monetize) for monetizing your Actors. The most recent and most flexible one is [pay-per-event](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event), which lets you charge your users programmatically directly from your Actor. As the name suggests, you may charge the users each time a specific event occurs, for example a call to an external API or when you return a result. + +To use the pay-per-event pricing model, you first need to [set it up](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event) for your Actor in the Apify console. After that, you're free to start charging for events. + +:::info How pay-per-event pricing works + +If you want more details about PPE pricing, please refer to our [PPE documentation](https://docs.apify.com/platform/actors/publishing/monetize/pay-per-event). + +::: + +## Charging for events + +After monetization is set in the Apify console, you can add <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> calls to your code and start monetizing! + +<RunnableCodeBlock className="language-python" language="python"> + {ActorChargeSource} +</RunnableCodeBlock> + +Then you just push your code to Apify and that's it! The SDK will even keep track of the max total charge setting for you, so you will not provide more value than what the user chose to pay for. + +If you need finer control over charging, you can access call <ApiLink to="class/Actor#get_charging_manager">`Actor.get_charging_manager()`</ApiLink> to access the <ApiLink to="class/ChargingManager">`ChargingManager`</ApiLink>, which can provide more detailed information - for example how many events of each type can be charged before reaching the configured limit. + +### Handling the charge limit + +While the SDK automatically prevents overcharging by limiting how many events are charged and how many items are pushed, **it does not stop your Actor from running**. When the charge limit is reached, <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> and `Actor.push_data` will silently stop charging and pushing data, but your Actor will keep running — potentially doing expensive work (scraping pages, calling APIs) for no purpose. This means your Actor may never terminate on its own if you don't check the charge limit yourself. + +To avoid this, you should check the `event_charge_limit_reached` field in the result returned by <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> or `Actor.push_data` and stop your Actor when the limit is reached. You can also use the `chargeable_within_limit` field from the result to plan ahead — it tells you how many events of each type can still be charged within the remaining budget. + +<RunnableCodeBlock className="language-python" language="python"> + {ChargeLimitCheckSource} +</RunnableCodeBlock> + +Alternatively, you can periodically check the remaining budget via <ApiLink to="class/Actor#get_charging_manager">`Actor.get_charging_manager()`</ApiLink> instead of inspecting every `ChargeResult`. This can be useful when charging happens in multiple places across your code, or when using a crawler where you don't directly control the main loop. + +:::caution +Always check the charge limit in your Actor, whether through `ChargeResult` return values or the `ChargingManager`. Without this check, your Actor will continue running and consuming platform resources after the budget is exhausted, producing no output. +::: + +## Transitioning from a different pricing model + +When you plan to start using the pay-per-event pricing model for an Actor that is already monetized with a different pricing model, your source code will need support both pricing models during the transition period enforced by the Apify platform. Arguably the most frequent case is the transition from the pay-per-result model which utilizes the `ACTOR_MAX_PAID_DATASET_ITEMS` environment variable to prevent returning unpaid dataset items. The following is an example how to handle such scenarios. The key part is the <ApiLink to="class/ChargingManager#get_pricing_info">`ChargingManager.get_pricing_info()`</ApiLink> method which returns information about the current pricing model. + +<RunnableCodeBlock className="language-python" language="python"> + {ConditionalActorChargeSource} +</RunnableCodeBlock> + +## Local development + +It is encouraged to test your monetization code on your machine before releasing it to the public. To tell your Actor that it should work in pay-per-event mode, pass it the `ACTOR_TEST_PAY_PER_EVENT` environment variable: + +```sh +ACTOR_TEST_PAY_PER_EVENT=true python -m youractor +``` + +If you also wish to see a log of all the events charged throughout the run, the Apify SDK keeps a log of charged events in a so called charging dataset. Your charging dataset can be found under the `charging-log` name (unless you change your storage settings, this dataset is stored in `storage/datasets/charging-log/`). Please note that this log is not available when running the Actor in production on the Apify platform. + +Because pricing configuration is stored by the Apify platform, all events will have a default price of $1. diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_class_context.py b/website/versioned_docs/version-3.3/02_concepts/code/01_class_context.py new file mode 100644 index 00000000..86b0875a --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_class_context.py @@ -0,0 +1,21 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Get input + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await Actor.push_data(data) + + # Set status message + await Actor.set_status_message('Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_class_manual.py b/website/versioned_docs/version-3.3/02_concepts/code/01_class_manual.py new file mode 100644 index 00000000..fbc1c434 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_class_manual.py @@ -0,0 +1,26 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + await Actor.init() + + try: + # Get input + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await Actor.push_data(data) + + # Set status message + await Actor.set_status_message('Actor completed successfully') + + finally: + await Actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_context_manager.py b/website/versioned_docs/version-3.3/02_concepts/code/01_context_manager.py new file mode 100644 index 00000000..a98c6ebe --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_context_manager.py @@ -0,0 +1,15 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_context.py b/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_context.py new file mode 100644 index 00000000..435563a3 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_context.py @@ -0,0 +1,13 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Any unhandled exception triggers Actor.fail() automatically + raise RuntimeError('Boom') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_manual.py b/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_manual.py new file mode 100644 index 00000000..367695a8 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_error_handling_manual.py @@ -0,0 +1,33 @@ +import asyncio +import random + +from apify import Actor + + +async def do_work() -> None: + # Simulate random outcomes: success or one of two exception types. + outcome = random.random() + + if outcome < 0.33: + raise ValueError('Invalid input data encountered') + if outcome < 0.66: + raise RuntimeError('Unexpected runtime failure') + + # Simulate successful work + Actor.log.info('Work completed successfully') + + +async def main() -> None: + await Actor.init() + try: + await do_work() + except ValueError as exc: # Specific error mapping example + await Actor.fail(exit_code=10, exception=exc) + except Exception as exc: # Catch-all for unexpected errors + await Actor.fail(exit_code=91, exception=exc) + else: + await Actor.exit(status_message='Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_init_exit.py b/website/versioned_docs/version-3.3/02_concepts/code/01_init_exit.py new file mode 100644 index 00000000..38b53465 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_init_exit.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + await Actor.init() + + try: + Actor.log.info('Actor input:', await Actor.get_input()) + await Actor.set_value('OUTPUT', 'Hello, world!') + raise RuntimeError('Ouch!') + + except Exception as exc: + Actor.log.exception('Error while running Actor') + await Actor.fail(exit_code=91, exception=exc) + + await Actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_instance_context.py b/website/versioned_docs/version-3.3/02_concepts/code/01_instance_context.py new file mode 100644 index 00000000..5803d543 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_instance_context.py @@ -0,0 +1,27 @@ +import asyncio +from datetime import timedelta + +from apify import Actor + + +async def main() -> None: + actor = Actor( + event_listeners_timeout=timedelta(seconds=30), + cleanup_timeout=timedelta(seconds=30), + ) + + async with actor: + # Get input + actor_input = await actor.get_input() + actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor instance!', 'input': actor_input} + await actor.push_data(data) + + # Set status message + await actor.set_status_message('Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_instance_manual.py b/website/versioned_docs/version-3.3/02_concepts/code/01_instance_manual.py new file mode 100644 index 00000000..b5a2dcaf --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_instance_manual.py @@ -0,0 +1,32 @@ +import asyncio +from datetime import timedelta + +from apify import Actor + + +async def main() -> None: + actor = Actor( + event_listeners_timeout=timedelta(seconds=30), + cleanup_timeout=timedelta(seconds=30), + ) + + await actor.init() + + try: + # Get input + actor_input = await actor.get_input() + actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await actor.push_data(data) + + # Set status message + await actor.set_status_message('Actor completed successfully') + + finally: + await actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_reboot.py b/website/versioned_docs/version-3.3/02_concepts/code/01_reboot.py new file mode 100644 index 00000000..b12066c5 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_reboot.py @@ -0,0 +1,23 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Use the KVS to persist a simple reboot counter across restarts. + kvs = await Actor.open_key_value_store() + reboot_counter = await kvs.get_value('reboot_counter', 0) + + # Limit the number of reboots to avoid infinite loops. + if reboot_counter < 3: + await kvs.set_value('reboot_counter', reboot_counter + 1) + Actor.log.info(f'Reboot attempt {reboot_counter + 1}/3') + # Trigger a platform reboot; after restart the code runs from the beginning. + await Actor.reboot() + + Actor.log.info('Reboot limit reached, finishing run') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/01_status_message.py b/website/versioned_docs/version-3.3/02_concepts/code/01_status_message.py new file mode 100644 index 00000000..9f239874 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/01_status_message.py @@ -0,0 +1,24 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + await Actor.set_status_message('Here we go!') + # Do some work... + await asyncio.sleep(3) + await Actor.set_status_message('So far so good...') + await asyncio.sleep(3) + # Do some more work... + await Actor.set_status_message('Steady as she goes...') + await asyncio.sleep(3) + # Do even more work... + await Actor.set_status_message('Almost there...') + await asyncio.sleep(3) + # Finish the job + await Actor.set_status_message('Phew! That was not that hard!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/02_input.py b/website/versioned_docs/version-3.3/02_concepts/code/02_input.py new file mode 100644 index 00000000..7dd0f86e --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/02_input.py @@ -0,0 +1,15 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() or {} + first_number = actor_input.get('firstNumber', 0) + second_number = actor_input.get('secondNumber', 0) + Actor.log.info('Sum: %s', first_number + second_number) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_exports.py b/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_exports.py new file mode 100644 index 00000000..bd879fc6 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_exports.py @@ -0,0 +1,37 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a dataset and write some data in it + dataset = await Actor.open_dataset(name='my-cool-dataset') + await dataset.push_data([{'itemNo': i} for i in range(1000)]) + + # Export the data as CSV + await dataset.export_to( + content_type='csv', + key='data.csv', + to_kvs_name='my-cool-key-value-store', + ) + + # Export the data as JSON + await dataset.export_to( + content_type='json', + key='data.json', + to_kvs_name='my-cool-key-value-store', + ) + + # Print the exported records + store = await Actor.open_key_value_store(name='my-cool-key-value-store') + + csv_data = await store.get_value('data.csv') + Actor.log.info(f'CSV data: {csv_data}') + + json_data = await store.get_value('data.json') + Actor.log.info(f'JSON data: {json_data}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_read_write.py b/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_read_write.py new file mode 100644 index 00000000..f5c6333e --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_dataset_read_write.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a dataset and write some data in it + dataset = await Actor.open_dataset(name='my-cool-dataset') + await dataset.push_data([{'itemNo': i} for i in range(1000)]) + + # Read back the first half of the data + first_half = await dataset.get_data(limit=500) + Actor.log.info(f'The first half of items = {first_half.items}') + + # Iterate over the second half + second_half = [item async for item in dataset.iterate_items(offset=500)] + Actor.log.info(f'The second half of items = {second_half}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_deleting_storages.py b/website/versioned_docs/version-3.3/02_concepts/code/03_deleting_storages.py new file mode 100644 index 00000000..75d8a69a --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_deleting_storages.py @@ -0,0 +1,19 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a key-value store with the name 'my-cool-store' + key_value_store = await Actor.open_key_value_store(name='my-cool-store') + await key_value_store.set_value('record', 'Hello, world!') + + # Do something ... + + # Now we don't want it anymore + await key_value_store.drop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_iterating.py b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_iterating.py new file mode 100644 index 00000000..a5a3009b --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_iterating.py @@ -0,0 +1,24 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + kvs = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Write some data to it + await kvs.set_value('automatic_text', 'abcd') + await kvs.set_value('automatic_json', {'ab': 'cd'}) + await kvs.set_value('explicit_csv', 'a,b\nc,d', content_type='text/csv') + + # Print the info for each record + Actor.log.info('Records in store:') + + async for key, info in kvs.iterate_keys(): + Actor.log.info(f'key={key}, info={info}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_public_url.py b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_public_url.py new file mode 100644 index 00000000..f2a870e4 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_public_url.py @@ -0,0 +1,17 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + store = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Get the public URL of a record + my_record_url = await store.get_public_url('my_record') + Actor.log.info(f'URL of "my_record": {my_record_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_read_write.py b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_read_write.py new file mode 100644 index 00000000..ba4a1dd2 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_kvs_read_write.py @@ -0,0 +1,31 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Open a named key-value store + kvs = await Actor.open_key_value_store(name='my-cool-key-value-store') + + # Write some data to it + await kvs.set_value('automatic_text', 'abcd') + await kvs.set_value('automatic_json', {'ab': 'cd'}) + await kvs.set_value('explicit_csv', 'a,b\nc,d', content_type='text/csv') + + # Get the values and log them + automatic_text = await kvs.get_value('automatic_text') + Actor.log.info(f'Automatic text: {automatic_text}') + + automatic_json = await kvs.get_value('automatic_json') + Actor.log.info(f'Automatic JSON: {automatic_json}') + + explicit_csv = await kvs.get_value('explicit_csv') + Actor.log.info(f'Explicit CSV: {explicit_csv}') + + # Delete the `automatic_text` value + await kvs.set_value('automatic_text', None) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_opening_storages.py b/website/versioned_docs/version-3.3/02_concepts/code/03_opening_storages.py new file mode 100644 index 00000000..39730dfb --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_opening_storages.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor, Request + + +async def main() -> None: + async with Actor: + # Work with the default dataset of the Actor + dataset = await Actor.open_dataset() + await dataset.push_data({'result': 'Hello, world!'}) + + # Work with the key-value store with ID 'mIJVZsRQrDQf4rUAf' + key_value_store = await Actor.open_key_value_store(id='mIJVZsRQrDQf4rUAf') + await key_value_store.set_value('record', 'Hello, world!') + + # Work with the request queue with the name 'my-queue' + request_queue = await Actor.open_request_queue(name='my-queue') + await request_queue.add_request(Request.from_url('https://apify.com')) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/03_rq.py b/website/versioned_docs/version-3.3/02_concepts/code/03_rq.py new file mode 100644 index 00000000..4823570b --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/03_rq.py @@ -0,0 +1,54 @@ +import asyncio +import random + +from apify import Actor, Request + +FAILURE_RATE = 0.3 + + +async def main() -> None: + async with Actor: + # Open the queue + queue = await Actor.open_request_queue() + + # Add some requests to the queue + for i in range(1, 10): + await queue.add_request(Request.from_url(f'http://example.com/{i}')) + + # Add a request to the start of the queue, for priority processing + await queue.add_request(Request.from_url('http://example.com/0'), forefront=True) + + # If you try to add an existing request again, it will not do anything + add_request_info = await queue.add_request( + Request.from_url('http://example.com/5') + ) + Actor.log.info(f'Add request info: {add_request_info}') + + # Finally, process the queue until all requests are handled + while not await queue.is_finished(): + # Fetch the next unhandled request in the queue + request = await queue.fetch_next_request() + # This can happen due to the eventual consistency of the underlying request + # queue storage, best solution is just to sleep a bit. + if request is None: + await asyncio.sleep(1) + continue + + Actor.log.info(f'Processing request {request.unique_key}...') + Actor.log.info(f'Scraping URL {request.url}...') + + # Do some fake work, which fails 30% of the time + await asyncio.sleep(1) + if random.random() > FAILURE_RATE: + # If processing the request was successful, mark it as handled + Actor.log.info('Request successful.') + await queue.mark_request_as_handled(request) + else: + # If processing the request was unsuccessful, reclaim it so it can be + # processed again. + Actor.log.warning('Request failed, will retry!') + await queue.reclaim_request(request) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/04_actor_events.py b/website/versioned_docs/version-3.3/02_concepts/code/04_actor_events.py new file mode 100644 index 00000000..591fddcd --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/04_actor_events.py @@ -0,0 +1,42 @@ +import asyncio +from typing import Any + +from apify import Actor, Event + + +async def main() -> None: + async with Actor: + total_items = 1000 + + # Load the state if it's saved from some previous execution + processed_items = 0 + actor_state = await Actor.get_value('STATE') + if actor_state is not None: + processed_items = actor_state + + # Save the state when the `PERSIST_STATE` event happens + async def save_state(event_data: Any) -> None: + nonlocal processed_items + Actor.log.info('Saving Actor state', extra=event_data) + await Actor.set_value('STATE', processed_items) + + Actor.on(Event.PERSIST_STATE, save_state) + + # Do some fake work + for i in range(processed_items, total_items): + Actor.log.info(f'Processing item {i}...') + processed_items = i + await asyncio.sleep(0.1) + + # Suppose we can stop saving the state now + Actor.off(Event.PERSIST_STATE, save_state) + + # Do some more fake work, this time something that can't be restarted, + # so no point persisting the state + for j in range(10): + Actor.log.info(f'Processing item {j} of another kind...') + await asyncio.sleep(1) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy.py b/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy.py new file mode 100644 index 00000000..12d1dcca --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy.py @@ -0,0 +1,18 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration() + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy_config.py b/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy_config.py new file mode 100644 index 00000000..68f39e09 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_apify_proxy_config.py @@ -0,0 +1,21 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration( + groups=['RESIDENTIAL'], + country_code='US', + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() + Actor.log.info(f'Proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy.py b/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy.py new file mode 100644 index 00000000..43ad2dbf --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy.py @@ -0,0 +1,23 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy_function.py b/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy_function.py new file mode 100644 index 00000000..0fa233e9 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_custom_proxy_function.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import asyncio + +from apify import Actor, Request + + +async def custom_new_url_function( + session_id: str | None = None, + _: Request | None = None, +) -> str | None: + if session_id is not None: + return f'http://my-custom-proxy-supporting-sessions.com?session-id={session_id}' + return 'http://my-custom-proxy-not-supporting-sessions.com' + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration( + new_url_function=custom_new_url_function, # ty: ignore[invalid-argument-type] + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url_with_session = await proxy_cfg.new_url('a') + Actor.log.info(f'Using proxy URL: {proxy_url_with_session}') + + proxy_url_without_session = await proxy_cfg.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url_without_session}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_actor_input.py b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_actor_input.py new file mode 100644 index 00000000..adf0d8d3 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_actor_input.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + actor_input = await Actor.get_input() or {} + proxy_settings = actor_input.get('proxySettings') + proxy_cfg = await Actor.create_proxy_configuration( + actor_proxy_input=proxy_settings + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() + Actor.log.info(f'Using proxy URL: {proxy_url}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_httpx.py b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_httpx.py new file mode 100644 index 00000000..dbabacaa --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_httpx.py @@ -0,0 +1,28 @@ +import asyncio + +import httpx + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() + + async with httpx.AsyncClient(proxy=proxy_url) as httpx_client: + response = await httpx_client.get('http://example.com') + Actor.log.info(f'Response: {response}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_rotation.py b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_rotation.py new file mode 100644 index 00000000..427eb585 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/05_proxy_rotation.py @@ -0,0 +1,29 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + proxy_cfg = await Actor.create_proxy_configuration( + proxy_urls=[ + 'http://proxy-1.com', + 'http://proxy-2.com', + ], + ) + + if not proxy_cfg: + raise RuntimeError('No proxy configuration available.') + + proxy_url = await proxy_cfg.new_url() # http://proxy-1.com + proxy_url = await proxy_cfg.new_url() # http://proxy-2.com + proxy_url = await proxy_cfg.new_url() # http://proxy-1.com + proxy_url = await proxy_cfg.new_url() # http://proxy-2.com + proxy_url = await proxy_cfg.new_url(session_id='a') # http://proxy-1.com + proxy_url = await proxy_cfg.new_url(session_id='b') # http://proxy-2.com + proxy_url = await proxy_cfg.new_url(session_id='b') # http://proxy-2.com + proxy_url = await proxy_cfg.new_url(session_id='a') # http://proxy-1.com + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call.py b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call.py new file mode 100644 index 00000000..da798a61 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call.py @@ -0,0 +1,32 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start the apify/screenshot-url Actor. + actor_run = await Actor.call( + actor_id='apify/screenshot-url', + run_input={ + 'urls': [{'url': 'https://www.apify.com/'}], + 'delay': 1000, + 'waitUntil': 'load', + }, + ) + + if actor_run is None: + raise RuntimeError('Actor task failed to start.') + + # Wait for the Actor run to finish. + run_client = Actor.apify_client.run(actor_run.id) + await run_client.wait_for_finish() + + # Get the Actor output from the dataset. + dataset_client = run_client.dataset() + item_list = await dataset_client.list_items() + Actor.log.info(f'Actor output: {item_list.items}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call_task.py b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call_task.py new file mode 100644 index 00000000..4796be4d --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_call_task.py @@ -0,0 +1,25 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start the Actor task by its ID. + actor_run = await Actor.call_task(task_id='Z3m6FPSj0GYZ25rQc') + + if actor_run is None: + raise RuntimeError('Actor task failed to start.') + + # Wait for the task run to finish. + run_client = Actor.apify_client.run(actor_run.id) + await run_client.wait_for_finish() + + # Get the task run dataset items + dataset_client = run_client.dataset() + items = await dataset_client.list_items() + Actor.log.info(f'Task run dataset items: {items}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_metamorph.py b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_metamorph.py new file mode 100644 index 00000000..b1db959a --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_metamorph.py @@ -0,0 +1,30 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Get the original Actor input. + actor_input = await Actor.get_input() or {} + hotel_url = actor_input.get('hotel_url') + + # Create new input for apify/web-scraper Actor. + web_scraper_input = { + 'startUrls': [{'url': hotel_url}], + 'pageFunction': """async function pageFunction(context) { + // Here you pass the JavaScript page function + // that scrapes all the reviews from the hotel's URL + }""", + } + + # Metamorph the Actor run to `apify/web-scraper` with the new input. + await Actor.metamorph('apify/web-scraper', web_scraper_input) + + # This code will not be called, since the `metamorph` action terminates + # the current Actor run container. + Actor.log.info('You will not see this!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_start.py b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_start.py new file mode 100644 index 00000000..b5e5fe54 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/06_interacting_start.py @@ -0,0 +1,19 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Start your own Actor named 'my-fancy-actor'. + actor_run = await Actor.start( + actor_id='~my-fancy-actor', + run_input={'foo': 'bar'}, + ) + + # Log the Actor run ID. + Actor.log.info(f'Actor run ID: {actor_run.id}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/07_webhook.py b/website/versioned_docs/version-3.3/02_concepts/code/07_webhook.py new file mode 100644 index 00000000..76c9153c --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/07_webhook.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor, Webhook + + +async def main() -> None: + async with Actor: + # Create a webhook that will be triggered when the Actor run fails. + webhook = Webhook( + event_types=['ACTOR.RUN.FAILED'], # ty: ignore[invalid-argument-type] + request_url='https://example.com/run-failed', + ) + + # Add the webhook to the Actor. + await Actor.add_webhook(webhook) + + # Raise an error to simulate a failed run. + raise RuntimeError('I am an error and I know it!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/07_webhook_preventing.py b/website/versioned_docs/version-3.3/02_concepts/code/07_webhook_preventing.py new file mode 100644 index 00000000..3ace707b --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/07_webhook_preventing.py @@ -0,0 +1,22 @@ +import asyncio + +from apify import Actor, Webhook + + +async def main() -> None: + async with Actor: + # Create a webhook that will be triggered when the Actor run fails. + webhook = Webhook( + event_types=['ACTOR.RUN.FAILED'], # ty: ignore[invalid-argument-type] + request_url='https://example.com/run-failed', + ) + + # Add the webhook to the Actor. + await Actor.add_webhook(webhook, idempotency_key=Actor.configuration.actor_run_id) + + # Raise an error to simulate a failed run. + raise RuntimeError('I am an error and I know it!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/08_actor_client.py b/website/versioned_docs/version-3.3/02_concepts/code/08_actor_client.py new file mode 100644 index 00000000..304fdf09 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/08_actor_client.py @@ -0,0 +1,17 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Create a new user client. + user_client = Actor.apify_client.user('me') + + # Get information about the current user. + me = await user_client.get() + Actor.log.info(f'User: {me}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/08_actor_new_client.py b/website/versioned_docs/version-3.3/02_concepts/code/08_actor_new_client.py new file mode 100644 index 00000000..0569dfa6 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/08_actor_new_client.py @@ -0,0 +1,20 @@ +import asyncio + +from apify import Actor + +TOKEN = 'ANOTHER_USERS_TOKEN' + + +async def main() -> None: + async with Actor: + # Create a new user client with a custom token. + apify_client = Actor.new_client(token=TOKEN, max_retries=2) + user_client = apify_client.user('me') + + # Get information about the another user. + them = await user_client.get() + Actor.log.info(f'Another user: {them}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/09_log_config.py b/website/versioned_docs/version-3.3/02_concepts/code/09_log_config.py new file mode 100644 index 00000000..c3c93610 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/09_log_config.py @@ -0,0 +1,17 @@ +import asyncio +import logging + +from apify.log import ActorLogFormatter + + +async def main() -> None: + handler = logging.StreamHandler() + handler.setFormatter(ActorLogFormatter()) + + apify_logger = logging.getLogger('apify') + apify_logger.setLevel(logging.DEBUG) + apify_logger.addHandler(handler) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/09_logger_usage.py b/website/versioned_docs/version-3.3/02_concepts/code/09_logger_usage.py new file mode 100644 index 00000000..74f2beb1 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/09_logger_usage.py @@ -0,0 +1,28 @@ +import asyncio +import logging + +from apify import Actor +from apify.log import ActorLogFormatter + + +async def main() -> None: + handler = logging.StreamHandler() + handler.setFormatter(ActorLogFormatter()) + + apify_logger = logging.getLogger('apify') + apify_logger.setLevel(logging.DEBUG) + apify_logger.addHandler(handler) + + async with Actor: + Actor.log.debug('This is a debug message') + Actor.log.info('This is an info message') + Actor.log.warning('This is a warning message', extra={'reason': 'Bad Actor!'}) + Actor.log.error('This is an error message') + try: + raise RuntimeError('Ouch!') + except RuntimeError: + Actor.log.exception('This is an exceptional message') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log.py b/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log.py new file mode 100644 index 00000000..387e6d1d --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log.py @@ -0,0 +1,21 @@ +import asyncio +import logging + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Default redirect logger + await Actor.call(actor_id='some_actor_id') + # No redirect logger + await Actor.call(actor_id='some_actor_id', logger=None) + # Custom redirect logger + await Actor.call( + actor_id='some_actor_id', + logger=logging.getLogger('custom_logger'), + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log_existing_run.py b/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log_existing_run.py new file mode 100644 index 00000000..cd982833 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/09_redirect_log_existing_run.py @@ -0,0 +1,28 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Lifecycle of redirected logs is handled by the context manager. + async with await Actor.apify_client.run('some_actor_id').get_streamed_log( + # Redirect all logs from the start of that run, even the logs from past. + from_start=True + ): + await asyncio.sleep(5) + # Logging will stop out of context + + # Lifecycle of redirected logs can be handled manually. + streamed_log = await Actor.apify_client.run('some_id').get_streamed_log( + # Do not redirect historical logs from this actor run. + # Redirect only new logs from now on. + from_start=False + ) + streamed_log.start() + await asyncio.sleep(5) + await streamed_log.stop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/10_config.py b/website/versioned_docs/version-3.3/02_concepts/code/10_config.py new file mode 100644 index 00000000..f7e00c16 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/10_config.py @@ -0,0 +1,23 @@ +import asyncio +from datetime import timedelta + +from apify import Actor, Configuration, Event + + +async def main() -> None: + configuration = Configuration( + persist_state_interval=timedelta(seconds=10) + # Set other configuration options here as needed. + ) + + async with Actor(configuration=configuration): + # Define a handler that will be called for every persist state event. + async def save_state() -> None: + await Actor.set_value('STATE', 'Hello, world!') + + # The save_state handler will be called every 10 seconds now. + Actor.on(Event.PERSIST_STATE, save_state) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/11_actor_charge.py b/website/versioned_docs/version-3.3/02_concepts/code/11_actor_charge.py new file mode 100644 index 00000000..fc8a4433 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/11_actor_charge.py @@ -0,0 +1,36 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # highlight-start + # Charge for a single occurrence of an event + await Actor.charge(event_name='init') + # highlight-end + + # Prepare some mock results + result = [ + {'word': 'Lorem'}, + {'word': 'Ipsum'}, + {'word': 'Dolor'}, + {'word': 'Sit'}, + {'word': 'Amet'}, + ] + # highlight-start + # Shortcut for charging for each pushed dataset item + await Actor.push_data(result, 'result-item') + # highlight-end + + # highlight-start + # Or you can charge for a given number of events manually + await Actor.charge( + event_name='result-item', + count=len(result), + ) + # highlight-end + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/11_charge_limit_check.py b/website/versioned_docs/version-3.3/02_concepts/code/11_charge_limit_check.py new file mode 100644 index 00000000..7f946a23 --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/11_charge_limit_check.py @@ -0,0 +1,29 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + urls = [ + 'https://example.com/1', + 'https://example.com/2', + 'https://example.com/3', + ] + + for url in urls: + # Do some expensive work (e.g. scraping, API calls) + result = {'url': url, 'data': f'Scraped data from {url}'} + + # highlight-start + # push_data returns a ChargeResult - check it to know if the budget ran out + charge_result = await Actor.push_data(result, 'result-item') + + if charge_result.event_charge_limit_reached: + Actor.log.info('Charge limit reached, stopping the Actor') + break + # highlight-end + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/02_concepts/code/11_conditional_actor_charge.py b/website/versioned_docs/version-3.3/02_concepts/code/11_conditional_actor_charge.py new file mode 100644 index 00000000..193284fd --- /dev/null +++ b/website/versioned_docs/version-3.3/02_concepts/code/11_conditional_actor_charge.py @@ -0,0 +1,24 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Check the dataset because there might already be items + # if the run migrated or was restarted + default_dataset = await Actor.open_dataset() + metadata = await default_dataset.get_metadata() + charged_items = metadata.item_count + + # highlight-start + if Actor.get_charging_manager().get_pricing_info().is_pay_per_event: + # highlight-end + await Actor.push_data({'hello': 'world'}, 'dataset-item') + elif charged_items < (Actor.configuration.max_paid_dataset_items or 0): + await Actor.push_data({'hello': 'world'}) + charged_items += 1 + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/01_beautifulsoup_httpx.mdx b/website/versioned_docs/version-3.3/03_guides/01_beautifulsoup_httpx.mdx new file mode 100644 index 00000000..42452a2a --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/01_beautifulsoup_httpx.mdx @@ -0,0 +1,30 @@ +--- +id: beautifulsoup-httpx +title: Using BeautifulSoup with HTTPX +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import BeautifulSoupHttpxExample from '!!raw-loader!roa-loader!./code/01_beautifulsoup_httpx.py'; + +In this guide, you'll learn how to use the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) library with the [HTTPX](https://www.python-httpx.org/) library in your Apify Actors. + +## Introduction + +[BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) is a Python library for extracting data from HTML and XML files. It provides simple methods and Pythonic idioms for navigating, searching, and modifying a website's element tree, enabling efficient data extraction. + +[HTTPX](https://www.python-httpx.org/) is a modern, high-level HTTP client library for Python. It provides a simple interface for making HTTP requests and supports both synchronous and asynchronous requests. + +To create an Actor which uses those libraries, start from the [BeautifulSoup & Python](https://apify.com/templates/categories/python) Actor template. This template includes the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) and [HTTPX](https://www.python-httpx.org/) libraries preinstalled, allowing you to begin development immediately. + +## Example Actor + +Below is a simple Actor that recursively scrapes titles from all linked websites, up to a specified maximum depth, starting from URLs provided in the Actor input. It uses [HTTPX](https://www.python-httpx.org/) for fetching pages and [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) for parsing their content to extract titles and links to other pages. + +<RunnableCodeBlock className="language-python" language="python"> + {BeautifulSoupHttpxExample} +</RunnableCodeBlock> + +## Conclusion + +In this guide, you learned how to use the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) with the [HTTPX](https://www.python-httpx.org/) in your Apify Actors. By combining these libraries, you can efficiently extract data from HTML or XML files, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-3.3/03_guides/02_parsel_impit.mdx b/website/versioned_docs/version-3.3/03_guides/02_parsel_impit.mdx new file mode 100644 index 00000000..0b572bf8 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/02_parsel_impit.mdx @@ -0,0 +1,28 @@ +--- +id: parsel-impit +title: Using Parsel with Impit +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ParselImpitExample from '!!raw-loader!roa-loader!./code/02_parsel_impit.py'; + +In this guide, you'll learn how to combine the [Parsel](https://github.com/scrapy/parsel) and [Impit](https://github.com/apify/impit) libraries when building Apify Actors. + +## Introduction + +[Parsel](https://github.com/scrapy/parsel) is a Python library for extracting data from HTML and XML documents using CSS selectors and [XPath](https://en.wikipedia.org/wiki/XPath) expressions. It offers an intuitive API for navigating and extracting structured data, making it a popular choice for web scraping. Compared to [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/), it also delivers better performance. + +[Impit](https://github.com/apify/impit) is Apify's high-performance HTTP client for Python. It supports both synchronous and asynchronous workflows and is built for large-scale web scraping, where making thousands of requests efficiently is essential. With built-in browser impersonation and anti-blocking features, it simplifies handling modern websites. + +## Example Actor + +The following example shows a simple Actor that recursively scrapes titles from linked pages, up to a user-defined maximum depth. It uses [Impit](https://github.com/apify/impit) to fetch pages and [Parsel](https://github.com/scrapy/parsel) to extract titles and discover new links. + +<RunnableCodeBlock className="language-python" language="python"> + {ParselImpitExample} +</RunnableCodeBlock> + +## Conclusion + +In this guide, you learned how to use [Parsel](https://github.com/scrapy/parsel) with [Impit](https://github.com/apify/impit) in your Apify Actors. By combining these libraries, you get a powerful and efficient solution for web scraping: [Parsel](https://github.com/scrapy/parsel) provides excellent CSS selector and XPath support for data extraction, while [Impit](https://github.com/apify/impit) offers a fast and simple HTTP client built by Apify. This combination makes it easy to build scalable web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-3.3/03_guides/03_playwright.mdx b/website/versioned_docs/version-3.3/03_guides/03_playwright.mdx new file mode 100644 index 00000000..2c7428a5 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/03_playwright.mdx @@ -0,0 +1,57 @@ +--- +id: playwright +title: Using Playwright +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import PlaywrightExample from '!!raw-loader!roa-loader!./code/03_playwright.py'; + +[Playwright](https://playwright.dev) is a tool for web automation and testing that can also be used for web scraping. It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Playwright for web scraping include: + +- **Cross-browser support** - Playwright supports the latest versions of major browsers like Chrome, Firefox, and Safari, so you can choose the one that suits your needs the best. +- **Headless mode** - Playwright can run in headless mode, meaning that the browser window is not visible on your screen while it is scraping, which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Playwright provides a variety of powerful selectors that allow you to target specific elements on a web page, including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Playwright allows you to emulate user interactions like clicking, scrolling, filling out forms, and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Playwright in Actors + +To create Actors which use Playwright, start from the [Playwright & Python](https://apify.com/templates/categories/python) Actor template. + +On the Apify platform, the Actor will already have Playwright and the necessary browsers preinstalled in its Docker image, including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to finish the Playwright setup yourself before you can run the Actor. + +<Tabs groupId="operating-systems"> + <TabItem value="unix" label="Linux / macOS" default> + <CodeBlock language="bash">{ +`source .venv/bin/activate +playwright install --with-deps` + }</CodeBlock> + </TabItem> + <TabItem value="win" label="Windows"> + <CodeBlock language="powershell">{ +`.venv\\Scripts\\activate +playwright install --with-deps` + }</CodeBlock> + </TabItem> +</Tabs> + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, up to a maximum depth, starting from URLs in the Actor input. + +It uses Playwright to open the pages in an automated Chrome browser, and to extract the title and anchor elements after the pages load. + +<RunnableCodeBlock className="language-python" language="python"> + {PlaywrightExample} +</RunnableCodeBlock> + +## Conclusion + +In this guide you learned how to create Actors that use Playwright to scrape websites. Playwright is a powerful tool that can be used to manage browser instances and scrape websites that require JavaScript execution. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-3.3/03_guides/04_selenium.mdx b/website/versioned_docs/version-3.3/03_guides/04_selenium.mdx new file mode 100644 index 00000000..bbc6abe1 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/04_selenium.mdx @@ -0,0 +1,46 @@ +--- +id: selenium +title: Using Selenium +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import SeleniumExample from '!!raw-loader!roa-loader!./code/04_selenium.py'; + +[Selenium](https://www.selenium.dev/) is a tool for web automation and testing that can also be used for web scraping. It allows you to control a web browser programmatically and interact with web pages just as a human would. + +Some of the key features of Selenium for web scraping include: + +- **Cross-browser support** - Selenium supports the latest versions of major browsers like Chrome, Firefox, and Safari, +so you can choose the one that suits your needs the best. +- **Headless mode** - Selenium can run in headless mode, +meaning that the browser window is not visible on your screen while it is scraping, +which can be useful for running scraping tasks in the background or in containers without a display. +- **Powerful selectors** - Selenium provides a variety of powerful selectors that allow you to target specific elements on a web page, +including CSS selectors, XPath, and text matching. +- **Emulation of user interactions** - Selenium allows you to emulate user interactions like clicking, scrolling, filling out forms, +and even typing in text, which can be useful for scraping websites that have dynamic content or require user input. + +## Using Selenium in Actors + +To create Actors which use Selenium, start from the [Selenium & Python](https://apify.com/templates/categories/python) Actor template. + +On the Apify platform, the Actor will already have Selenium and the necessary browsers preinstalled in its Docker image, +including the tools and setup necessary to run browsers in headful mode. + +When running the Actor locally, you'll need to install the Selenium browser drivers yourself. +Refer to the [Selenium documentation](https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/) for installation instructions. + +## Example Actor + +This is a simple Actor that recursively scrapes titles from all linked websites, up to a maximum depth, starting from URLs in the Actor input. + +It uses Selenium ChromeDriver to open the pages in an automated Chrome browser, and to extract the title and anchor elements after the pages load. + +<RunnableCodeBlock className="language-python" language="python"> + {SeleniumExample} +</RunnableCodeBlock> + +## Conclusion + +In this guide you learned how to use Selenium for web scraping in Apify Actors. You can now create your own Actors that use Selenium to scrape dynamic websites and interact with web pages just like a human would. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-3.3/03_guides/05_crawlee.mdx b/website/versioned_docs/version-3.3/03_guides/05_crawlee.mdx new file mode 100644 index 00000000..ed805dea --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/05_crawlee.mdx @@ -0,0 +1,46 @@ +--- +id: crawlee +title: Using Crawlee +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import CrawleeBeautifulSoupExample from '!!raw-loader!roa-loader!./code/05_crawlee_beautifulsoup.py'; +import CrawleeParselExample from '!!raw-loader!roa-loader!./code/05_crawlee_parsel.py'; +import CrawleePlaywrightExample from '!!raw-loader!roa-loader!./code/05_crawlee_playwright.py'; + +In this guide you'll learn how to use the [Crawlee](https://crawlee.dev/python) library in your Apify Actors. + +## Introduction + +[Crawlee](https://crawlee.dev/python) is a Python library for web scraping and browser automation that provides a robust and flexible framework for building web scraping tasks. It seamlessly integrates with the Apify platform and supports a variety of scraping techniques, from static HTML parsing to dynamic JavaScript-rendered content handling. Crawlee offers a range of crawlers, including HTTP-based crawlers like [`HttpCrawler`](https://crawlee.dev/python/api/class/HttpCrawler), [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) and [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler), and browser-based crawlers like [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler), to suit different scraping needs. + +In this guide, you'll learn how to use Crawlee with [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler), [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler), and [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler) to build Apify Actors for web scraping. + +## Actor with BeautifulSoupCrawler + +The [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) is ideal for extracting data from static HTML pages. It uses [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) for parsing and [`ImpitHttpClient`](https://crawlee.dev/python/api/class/ImpitHttpClient) for HTTP communication, ensuring efficient and lightweight scraping. If you do not need to execute JavaScript on the page, [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) is a great choice for your scraping tasks. Below is an example of how to use it` in an Apify Actor. + +<RunnableCodeBlock className="language-python" language="python"> + {CrawleeBeautifulSoupExample} +</RunnableCodeBlock> + +## Actor with ParselCrawler + +The [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler) works in the same way as [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler), but it uses the [Parsel](https://parsel.readthedocs.io/en/latest/) library for HTML parsing. This allows for more powerful and flexible data extraction using [XPath](https://en.wikipedia.org/wiki/XPath) selectors. It should be faster than [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler). Below is an example of how to use [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler) in an Apify Actor. + +<RunnableCodeBlock className="language-python" language="python"> + {CrawleeParselExample} +</RunnableCodeBlock> + +## Actor with PlaywrightCrawler + +The [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler) is built for handling dynamic web pages that rely on JavaScript for content rendering. Using the [Playwright](https://playwright.dev/) library, it provides a browser-based automation environment to interact with complex websites. Below is an example of how to use [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler) in an Apify Actor. + +<RunnableCodeBlock className="language-python" language="python"> + {CrawleePlaywrightExample} +</RunnableCodeBlock> + +## Conclusion + +In this guide, you learned how to use the [Crawlee](https://crawlee.dev/python) library in your Apify Actors. By using the [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler), [`ParselCrawler`](https://crawlee.dev/python/api/class/ParselCrawler), and [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler) crawlers, you can efficiently scrape static or dynamic web pages, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/website/versioned_docs/version-3.3/03_guides/06_scrapy.mdx b/website/versioned_docs/version-3.3/03_guides/06_scrapy.mdx new file mode 100644 index 00000000..7d790b7d --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/06_scrapy.mdx @@ -0,0 +1,114 @@ +--- +id: scrapy +title: Using Scrapy +--- + +import CodeBlock from '@theme/CodeBlock'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import UnderscoreMainExample from '!!raw-loader!./code/scrapy_project/src/__main__.py'; +import MainExample from '!!raw-loader!./code/scrapy_project/src/main.py'; +import ItemsExample from '!!raw-loader!./code/scrapy_project/src/items.py'; +import SpidersExample from '!!raw-loader!./code/scrapy_project/src/spiders/title.py'; +import SettingsExample from '!!raw-loader!./code/scrapy_project/src/settings.py'; + +[Scrapy](https://scrapy.org/) is an open-source web scraping framework for Python. It provides tools for defining scrapers, extracting data from web pages, following links, and handling pagination. With the Apify SDK, Scrapy projects can be converted into Apify [Actors](https://docs.apify.com/platform/actors), integrated with Apify [storages](https://docs.apify.com/platform/storage), and executed on the Apify [platform](https://docs.apify.com/platform). + +## Integrating Scrapy with the Apify platform + +The Apify SDK provides an Apify-Scrapy integration. The main challenge of this is to combine two asynchronous frameworks that use different event loop implementations. Scrapy uses [Twisted](https://twisted.org/) for asynchronous execution, while the Apify SDK is based on [asyncio](https://docs.python.org/3/library/asyncio.html). The key thing is to install the Twisted's `asyncioreactor` to run Twisted's asyncio compatible event loop. The `apify.scrapy.run_scrapy_actor` function handles this reactor installation automatically. This allows both Twisted and asyncio to run on a single event loop, enabling a Scrapy spider to run as an Apify Actor with minimal modifications. + +<CodeBlock className="language-python" title="__main.py__: The Actor entry point "> + {UnderscoreMainExample} +</CodeBlock> + +In this setup, `apify.scrapy.initialize_logging` configures an Apify log formatter and reconfigures loggers to ensure consistent logging across Scrapy, the Apify SDK, and other libraries. The `apify.scrapy.run_scrapy_actor` installs Twisted's asyncio-compatible reactor and bridges asyncio coroutines with Twisted's reactor, enabling the Actor's main coroutine, which contains the Scrapy spider, to be executed. + +Make sure the `SCRAPY_SETTINGS_MODULE` environment variable is set to the path of the Scrapy settings module. This variable is also used by the `Actor` class to detect that the project is a Scrapy project, triggering additional actions. + +<CodeBlock className="language-python" title="main.py: The Actor main coroutine"> + {MainExample} +</CodeBlock> + +Within the Actor's main coroutine, the Actor's input is processed as usual. The function `apify.scrapy.apply_apify_settings` is then used to configure Scrapy settings with Apify-specific components before the spider is executed. The key components and other helper functions are described in the next section. + +## Key integration components + +The Apify SDK provides several custom components to support integration with the Apify platform: + +- [`apify.scrapy.ApifyScheduler`](https://docs.apify.com/sdk/python/reference/class/ApifyScheduler) - Replaces Scrapy's default [scheduler](https://docs.scrapy.org/en/latest/topics/scheduler.html) with one that uses Apify's [request queue](https://docs.apify.com/platform/storage/request-queue) for storing requests. It manages enqueuing, dequeuing, and maintaining the state and priority of requests. +- [`apify.scrapy.ActorDatasetPushPipeline`](https://docs.apify.com/sdk/python/reference/class/ActorDatasetPushPipeline) - A Scrapy [item pipeline](https://docs.scrapy.org/en/latest/topics/item-pipeline.html) that pushes scraped items to Apify's [dataset](https://docs.apify.com/platform/storage/dataset). When enabled, every item produced by the spider is sent to the dataset. +- [`apify.scrapy.ApifyHttpProxyMiddleware`](https://docs.apify.com/sdk/python/reference/class/ApifyHttpProxyMiddleware) - A Scrapy [middleware](https://docs.scrapy.org/en/latest/topics/downloader-middleware.html) that manages proxy configurations. This middleware replaces Scrapy's default `HttpProxyMiddleware` to facilitate the use of Apify's proxy service. +- [`apify.scrapy.extensions.ApifyCacheStorage`](https://docs.apify.com/sdk/python/reference/class/ApifyCacheStorage) - A storage backend for Scrapy's built-in [HTTP cache middleware](https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpcache). This backend uses Apify's [key-value store](https://docs.apify.com/platform/storage/key-value-store). Make sure to set `HTTPCACHE_ENABLED` and `HTTPCACHE_EXPIRATION_SECS` in your settings, or caching won't work. + +Additional helper functions in the [`apify.scrapy`](https://github.com/apify/apify-sdk-python/tree/master/src/apify/scrapy) subpackage include: + +- `apply_apify_settings` - Applies Apify-specific components to Scrapy settings. +- `to_apify_request` and `to_scrapy_request` - Convert between Apify and Scrapy request objects. +- `initialize_logging` - Configures logging for the Actor environment. +- `run_scrapy_actor` - Installs Twisted's asyncio reactor and bridges asyncio and Twisted event loops. + +## Create a new Apify-Scrapy project + +The simplest way to start using Scrapy in Apify Actors is to use the [Scrapy Actor template](https://apify.com/templates/python-scrapy). The template provides a pre-configured project structure and setup that includes all necessary components to run Scrapy spiders as Actors and store their output in Apify datasets. If you prefer manual setup, refer to the example Actor section below for configuration details. + +## Wrapping an existing Scrapy project + +The Apify CLI supports converting an existing Scrapy project into an Apify Actor with a single command. The CLI expects the project to follow the standard Scrapy layout (including a `scrapy.cfg` file in the project root). During the wrapping process, the CLI: + +- Creates the necessary files and directories for an Apify Actor. +- Installs the Apify SDK and required dependencies. +- Updates Scrapy settings to include Apify-specific components. + +For further details, see the [Scrapy migration guide](https://docs.apify.com/cli/docs/integrating-scrapy). + +## Example Actor + +The following example demonstrates a Scrapy Actor that scrapes page titles and enqueues links found on each page. This example aligns with the structure provided in the Apify Actor templates. + +<Tabs> + <TabItem value="__main__.py" label="__main.py__"> + <CodeBlock className="language-python"> + {UnderscoreMainExample} + </CodeBlock> + </TabItem> + <TabItem value="main.py" label="main.py"> + <CodeBlock className="language-python"> + {MainExample} + </CodeBlock> + </TabItem> + <TabItem value="settings.py" label="settings.py"> + <CodeBlock className="language-python"> + {SettingsExample} + </CodeBlock> + </TabItem> + <TabItem value="items.py" label="items.py"> + <CodeBlock className="language-python"> + {ItemsExample} + </CodeBlock> + </TabItem> + <TabItem value="spiders/title.py" label="spiders/title.py"> + <CodeBlock className="language-python"> + {SpidersExample} + </CodeBlock> + </TabItem> +</Tabs> + +## Dealing with imminent migration to another host + +Under some circumstances, the platform may decide to [migrate your Actor](https://docs.apify.com/academy/expert-scraping-with-apify/migrations-maintaining-state) from one piece of infrastructure to another while it's in progress. While [Crawlee](https://crawlee.dev/python)-based projects can pause and resume their work after a restart, achieving the same with a Scrapy-based project can be challenging. + +As a workaround for this issue (tracked as [apify/actor-templates#303](https://github.com/apify/actor-templates/issues/303)), turn on caching with `HTTPCACHE_ENABLED` and set `HTTPCACHE_EXPIRATION_SECS` to at least a few minutes—the exact value depends on your use case. If your Actor gets migrated and restarted, the subsequent run will hit the cache, making it fast and avoiding unnecessary resource consumption. + +## Conclusion + +In this guide you learned how to use Scrapy in Apify Actors. You can now start building your own web scraping projects using Scrapy, the Apify SDK and host them on the Apify platform. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! + +## Additional resources + +- [Apify CLI: Integrating Scrapy projects](https://docs.apify.com/cli/docs/integrating-scrapy) +- [Apify: Run Scrapy spiders on Apify](https://apify.com/run-scrapy-in-cloud) +- [Apify templates: Python Actor Scrapy template](https://apify.com/templates/python-scrapy) +- [Apify store: Scrapy Books Example Actor](https://apify.com/vdusek/scrapy-books-example) +- [Scrapy: Official documentation](https://docs.scrapy.org/) diff --git a/website/versioned_docs/version-3.3/03_guides/07_running_webserver.mdx b/website/versioned_docs/version-3.3/03_guides/07_running_webserver.mdx new file mode 100644 index 00000000..d9deedc1 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/07_running_webserver.mdx @@ -0,0 +1,26 @@ +--- +id: running-webserver +title: Running webserver in your Actor +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import WebserverExample from '!!raw-loader!roa-loader!./code/07_webserver.py'; + +Each Actor run on the Apify platform is assigned a unique hard-to-guess URL (for example `https://8segt5i81sokzm.runs.apify.net`), which enables HTTP access to an optional web server running inside the Actor run's container. + +The URL is available in the following places: + +- In Apify Console, on the Actor run details page as the **Container URL** field. +- In the API as the `container_url` property of the [Run object](https://docs.apify.com/api/v2#/reference/actors/run-object/get-run). +- In the Actor as the `Actor.configuration.container_url` property. + +The web server running inside the container must listen at the port defined by the `Actor.configuration.container_port` property. When running Actors locally, the port defaults to `4321`, so the web server will be accessible at `http://localhost:4321`. + +## Example + +The following example demonstrates how to start a simple web server in your Actor,which will respond to every GET request with the number of items that the Actor has processed so far: + +<RunnableCodeBlock className="language-python" language="python"> + {WebserverExample} +</RunnableCodeBlock> diff --git a/website/versioned_docs/version-3.3/03_guides/code/01_beautifulsoup_httpx.py b/website/versioned_docs/version-3.3/03_guides/code/01_beautifulsoup_httpx.py new file mode 100644 index 00000000..5dbfab2a --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/01_beautifulsoup_httpx.py @@ -0,0 +1,88 @@ +import asyncio +from urllib.parse import urljoin + +import httpx +from bs4 import BeautifulSoup + +from apify import Actor, Request + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + # Create an HTTPX client to fetch the HTML content of the URLs. + async with httpx.AsyncClient() as client: + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Fetch the HTTP response from the specified URL using HTTPX. + response = await client.get(url, follow_redirects=True) + + # Parse the HTML content using Beautiful Soup. + soup = BeautifulSoup(response.content, 'html.parser') + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in soup.find_all('a'): + link_href = link.get('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': soup.title.string if soup.title else None, + 'h1s': [h1.text for h1 in soup.find_all('h1')], + 'h2s': [h2.text for h2 in soup.find_all('h2')], + 'h3s': [h3.text for h3 in soup.find_all('h3')], + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(new_request) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/02_parsel_impit.py b/website/versioned_docs/version-3.3/03_guides/code/02_parsel_impit.py new file mode 100644 index 00000000..21b5e74f --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/02_parsel_impit.py @@ -0,0 +1,94 @@ +import asyncio +from urllib.parse import urljoin + +import impit +import parsel + +from apify import Actor, Request + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + # Create an Impit client to fetch the HTML content of the URLs. + async with impit.AsyncClient() as client: + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Fetch the HTTP response from the specified URL using Impit. + response = await client.get(url) + + # Parse the HTML content using Parsel Selector. + selector = parsel.Selector(text=response.text) + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + # Extract all links using CSS selector + links = selector.css('a::attr(href)').getall() + for link_href in links: + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data using Parsel selectors. + title = selector.css('title::text').get() + h1s = selector.css('h1::text').getall() + h2s = selector.css('h2::text').getall() + h3s = selector.css('h3::text').getall() + + data = { + 'url': url, + 'title': title, + 'h1s': h1s, + 'h2s': h2s, + 'h3s': h3s, + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(request) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/03_playwright.py b/website/versioned_docs/version-3.3/03_guides/code/03_playwright.py new file mode 100644 index 00000000..3eecb4ac --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/03_playwright.py @@ -0,0 +1,97 @@ +import asyncio +from urllib.parse import urljoin + +from playwright.async_api import async_playwright + +from apify import Actor, Request + +# Note: To run this Actor locally, ensure that Playwright browsers are installed. +# Run `playwright install --with-deps` in the Actor's virtual environment to install them. +# When running on the Apify platform, these dependencies are already included +# in the Actor's Docker image. + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + Actor.log.info('Launching Playwright...') + + # Launch Playwright and open a new browser context. + async with async_playwright() as playwright: + # Configure the browser to launch in headless mode as per Actor configuration. + browser = await playwright.chromium.launch( + headless=Actor.configuration.headless, + args=['--disable-gpu'], + ) + context = await browser.new_context() + + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Open a new page in the browser context and navigate to the URL. + page = await context.new_page() + await page.goto(url) + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in await page.locator('a').all(): + link_href = await link.get_attribute('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': await page.title(), + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + await page.close() + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(request) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/04_selenium.py b/website/versioned_docs/version-3.3/03_guides/code/04_selenium.py new file mode 100644 index 00000000..4b427a7a --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/04_selenium.py @@ -0,0 +1,106 @@ +import asyncio +from urllib.parse import urljoin + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.common.by import By + +from apify import Actor, Request + +# To run this Actor locally, you need to have the Selenium Chromedriver installed. +# Follow the installation guide at: +# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/ +# When running on the Apify platform, the Chromedriver is already included +# in the Actor's Docker image. + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + max_depth = actor_input.get('max_depth', 1) + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in actor input, exiting...') + await Actor.exit() + + # Open the default request queue for handling URLs to be processed. + request_queue = await Actor.open_request_queue() + + # Enqueue the start URLs with an initial crawl depth of 0. + for start_url in start_urls: + url = start_url.get('url') + Actor.log.info(f'Enqueuing {url} ...') + new_request = Request.from_url(url, user_data={'depth': 0}) + await request_queue.add_request(new_request) + + # Launch a new Selenium Chrome WebDriver and configure it. + Actor.log.info('Launching Chrome WebDriver...') + chrome_options = ChromeOptions() + + if Actor.configuration.headless: + chrome_options.add_argument('--headless') + + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + driver = webdriver.Chrome(options=chrome_options) + + # Test WebDriver setup by navigating to an example page. + driver.get('http://www.example.com') + if driver.title != 'Example Domain': + raise ValueError('Failed to open example page.') + + # Process the URLs from the request queue. + while request := await request_queue.fetch_next_request(): + url = request.url + + if not isinstance(request.user_data['depth'], (str, int)): + raise TypeError('Request.depth is an unexpected type.') + + depth = int(request.user_data['depth']) + Actor.log.info(f'Scraping {url} (depth={depth}) ...') + + try: + # Navigate to the URL using Selenium WebDriver. Use asyncio.to_thread + # for non-blocking execution. + await asyncio.to_thread(driver.get, url) + + # If the current depth is less than max_depth, find nested links + # and enqueue them. + if depth < max_depth: + for link in driver.find_elements(By.TAG_NAME, 'a'): + link_href = link.get_attribute('href') + link_url = urljoin(url, link_href) + + if link_url.startswith(('http://', 'https://')): + Actor.log.info(f'Enqueuing {link_url} ...') + new_request = Request.from_url( + link_url, + user_data={'depth': depth + 1}, + ) + await request_queue.add_request(new_request) + + # Extract the desired data. + data = { + 'url': url, + 'title': driver.title, + } + + # Store the extracted data to the default dataset. + await Actor.push_data(data) + + except Exception: + Actor.log.exception(f'Cannot extract data from {url}.') + + finally: + # Mark the request as handled to ensure it is not processed again. + await request_queue.mark_request_as_handled(request) + + driver.quit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_beautifulsoup.py b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_beautifulsoup.py new file mode 100644 index 00000000..4d3a81d7 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_beautifulsoup.py @@ -0,0 +1,55 @@ +import asyncio + +from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext + +from apify import Actor + +# Create a crawler. +crawler = BeautifulSoupCrawler( + # Limit the crawl to max requests. Remove or increase it for crawling all links. + max_requests_per_crawl=50, +) + + +# Define a request handler, which will be called for every request. +@crawler.router.default_handler +async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + Actor.log.info(f'Scraping {context.request.url}...') + + # Extract the desired data. + data = { + 'url': context.request.url, + 'title': context.soup.title.string if context.soup.title else None, + 'h1s': [h1.text for h1 in context.soup.find_all('h1')], + 'h2s': [h2.text for h2 in context.soup.find_all('h2')], + 'h3s': [h3.text for h3 in context.soup.find_all('h3')], + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional links found on the current page. + await context.enqueue_links(strategy='same-domain') + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [ + url.get('url') + for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + ] + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Run the crawler with the starting requests. + await crawler.run(start_urls) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_parsel.py b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_parsel.py new file mode 100644 index 00000000..31f39d8b --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_parsel.py @@ -0,0 +1,55 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + +from apify import Actor + +# Create a crawler. +crawler = ParselCrawler( + # Limit the crawl to max requests. Remove or increase it for crawling all links. + max_requests_per_crawl=50, +) + + +# Define a request handler, which will be called for every request. +@crawler.router.default_handler +async def request_handler(context: ParselCrawlingContext) -> None: + Actor.log.info(f'Scraping {context.request.url}...') + + # Extract the desired data. + data = { + 'url': context.request.url, + 'title': context.selector.xpath('//title/text()').get(), + 'h1s': context.selector.xpath('//h1/text()').getall(), + 'h2s': context.selector.xpath('//h2/text()').getall(), + 'h3s': context.selector.xpath('//h3/text()').getall(), + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional links found on the current page. + await context.enqueue_links(strategy='same-domain') + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [ + url.get('url') + for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + ] + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Run the crawler with the starting requests. + await crawler.run(start_urls) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_playwright.py b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_playwright.py new file mode 100644 index 00000000..be4ea29e --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/05_crawlee_playwright.py @@ -0,0 +1,58 @@ +import asyncio + +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + +from apify import Actor + +# Create a crawler. +crawler = PlaywrightCrawler( + # Limit the crawl to max requests. Remove or increase it for crawling all links. + max_requests_per_crawl=50, + # Run the browser in a headless mode. + headless=True, + browser_launch_options={'args': ['--disable-gpu']}, +) + + +# Define a request handler, which will be called for every request. +@crawler.router.default_handler +async def request_handler(context: PlaywrightCrawlingContext) -> None: + Actor.log.info(f'Scraping {context.request.url}...') + + # Extract the desired data. + data = { + 'url': context.request.url, + 'title': await context.page.title(), + 'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()], + 'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()], + 'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()], + } + + # Store the extracted data to the default dataset. + await context.push_data(data) + + # Enqueue additional links found on the current page. + await context.enqueue_links(strategy='same-domain') + + +async def main() -> None: + # Enter the context of the Actor. + async with Actor: + # Retrieve the Actor input, and use default values if not provided. + actor_input = await Actor.get_input() or {} + start_urls = [ + url.get('url') + for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}]) + ] + + # Exit if no start URLs are provided. + if not start_urls: + Actor.log.info('No start URLs specified in Actor input, exiting...') + await Actor.exit() + + # Run the crawler with the starting requests. + await crawler.run(start_urls) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/07_webserver.py b/website/versioned_docs/version-3.3/03_guides/code/07_webserver.py new file mode 100644 index 00000000..d4bc0655 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/07_webserver.py @@ -0,0 +1,53 @@ +import asyncio +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from apify import Actor + +processed_items = 0 +http_server = None + + +# Just a simple handler that will print the number of processed items so far +# on every GET request. +class RequestHandler(BaseHTTPRequestHandler): + def do_get(self) -> None: + self.log_request() + self.send_response(200) + self.end_headers() + self.wfile.write(bytes(f'Processed items: {processed_items}', encoding='utf-8')) + + +def run_server() -> None: + # Start the HTTP server on the provided port, + # and save a reference to the server. + global http_server + with ThreadingHTTPServer( + ('', Actor.configuration.web_server_port), RequestHandler + ) as server: + Actor.log.info(f'Server running on {Actor.configuration.web_server_port}') + http_server = server + server.serve_forever() + + +async def main() -> None: + global processed_items + async with Actor: + # Start the HTTP server in a separate thread. + run_server_task = asyncio.get_running_loop().run_in_executor(None, run_server) + + # Simulate doing some work. + for _ in range(100): + await asyncio.sleep(1) + processed_items += 1 + Actor.log.info(f'Processed items: {processed_items}') + + if http_server is None: + raise RuntimeError('HTTP server not started') + + # Signal the HTTP server to shut down, and wait for it to finish. + http_server.shutdown() + await run_server_task + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/__init__.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/__main__.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/__main__.py new file mode 100644 index 00000000..807447c9 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/__main__.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +import os + +from apify.scrapy import initialize_logging, run_scrapy_actor + +# Import your main Actor coroutine here. +from .main import main + +# Ensure the location to the Scrapy settings module is defined. +os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings' + + +if __name__ == '__main__': + initialize_logging() + run_scrapy_actor(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/items.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/items.py new file mode 100644 index 00000000..6579083f --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/items.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from scrapy import Field, Item + + +class TitleItem(Item): + """Represents a title item scraped from a web page.""" + + url = Field() + title = Field() diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/main.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/main.py new file mode 100644 index 00000000..d8b67984 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/main.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import asyncio + +from scrapy.crawler import AsyncCrawlerRunner + +from apify import Actor +from apify.scrapy import apply_apify_settings + +# Import your Scrapy spider here. +from .spiders import TitleSpider as Spider + + +async def main() -> None: + """Apify Actor main coroutine for executing the Scrapy spider.""" + async with Actor: + # Retrieve and process Actor input. + actor_input = await Actor.get_input() or {} + start_urls = [url['url'] for url in actor_input.get('startUrls', [])] + allowed_domains = actor_input.get('allowedDomains') + proxy_config = actor_input.get('proxyConfiguration') + + # Apply Apify settings, which will override the Scrapy project settings. + settings = apply_apify_settings(proxy_config=proxy_config) + + # Create AsyncCrawlerRunner and execute the Scrapy spider. + crawler_runner = AsyncCrawlerRunner(settings) + await crawler_runner.crawl( + Spider, + start_urls=start_urls, + allowed_domains=allowed_domains, + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/py.typed b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/settings.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/settings.py new file mode 100644 index 00000000..5c0e56e3 --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/settings.py @@ -0,0 +1,11 @@ +BOT_NAME = 'titlebot' +DEPTH_LIMIT = 1 +LOG_LEVEL = 'INFO' +NEWSPIDER_MODULE = 'src.spiders' +ROBOTSTXT_OBEY = True +SPIDER_MODULES = ['src.spiders'] +TELNETCONSOLE_ENABLED = False +# Do not change the Twisted reactor unless you really know what you are doing. +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' +HTTPCACHE_ENABLED = True +HTTPCACHE_EXPIRATION_SECS = 7200 diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/__init__.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/__init__.py new file mode 100644 index 00000000..3745a12c --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/__init__.py @@ -0,0 +1,3 @@ +from .title import TitleSpider + +__all__ = ['TitleSpider'] diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/py.typed b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/title.py b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/title.py new file mode 100644 index 00000000..7223a53d --- /dev/null +++ b/website/versioned_docs/version-3.3/03_guides/code/scrapy_project/src/spiders/title.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +from scrapy import Request, Spider + +from ..items import TitleItem + +if TYPE_CHECKING: + from collections.abc import Generator + + from scrapy.http.response import Response + + +class TitleSpider(Spider): + """A spider that scrapes web pages to extract titles and discover new links. + + This spider retrieves the content of the <title> element from each page and queues + any valid hyperlinks for further crawling. + """ + + name = 'title_spider' + + # Limit the number of pages to scrape. + custom_settings = {'CLOSESPIDER_PAGECOUNT': 10} + + def __init__( + self, + start_urls: list[str], + allowed_domains: list[str], + *args: Any, + **kwargs: Any, + ) -> None: + """A default constructor. + + Args: + start_urls: URLs to start the scraping from. + allowed_domains: Domains that the scraper is allowed to crawl. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + super().__init__(*args, **kwargs) + self.start_urls = start_urls + self.allowed_domains = allowed_domains + + def parse(self, response: Response) -> Generator[TitleItem | Request, None, None]: + """Parse the web page response. + + Args: + response: The web page response. + + Yields: + Yields scraped `TitleItem` and new `Request` objects for links. + """ + self.logger.info('TitleSpider is parsing %s...', response) + + # Extract and yield the TitleItem + url = response.url + title = response.css('title::text').extract_first() + yield TitleItem(url=url, title=title) + + # Extract all links from the page, create `Request` objects out of them, + # and yield them. + for link_href in response.css('a::attr("href")'): + link_url = urljoin(response.url, link_href.get()) + if link_url.startswith(('http://', 'https://')): + yield Request(link_url) diff --git a/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v2.md b/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v2.md new file mode 100644 index 00000000..1fd1d111 --- /dev/null +++ b/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v2.md @@ -0,0 +1,48 @@ +--- +id: upgrading-to-v2 +title: Upgrading to v2 +--- + +This page summarizes the breaking changes between Apify Python SDK v1.x and v2.0. + +## Python version support + +Support for Python 3.8 has been dropped. The Apify Python SDK v2.x now requires Python 3.9 or later. Make sure your environment is running a compatible version before upgrading. + +## Storages + +- The SDK now uses [crawlee](https://github.com/apify/crawlee-python) for local storage emulation. This change should not affect intended usage (working with `Dataset`, `KeyValueStore` and `RequestQueue` classes from the `apify.storages` module or using the shortcuts exposed by the `Actor` class) in any way. +- There is a difference in the `RequestQueue.add_request` method: it accepts an `apify.Request` object instead of a free-form dictionary. + - A quick way to migrate from dict-based arguments is to wrap it with a `Request.model_validate()` call. + - The preferred way is using the `Request.from_url` helper which prefills the `unique_key` and `id` attributes, or instantiating it directly, e.g., `Request(url='https://example.tld', ...)`. + - For simple use cases, `add_request` also accepts plain strings that contain an URL, e.g. `queue.add_request('https://example.tld')`. +- Removing the `StorageClientManager` class is a significant change. If you need to change the storage client, use `crawlee.service_container` instead. + +## Configuration + +The `apify.Configuration` class now uses `pydantic_settings` to load configuration from environment variables. This eliminates the need for the helper functions which handled environment variables in `apify._utils`. + +Attributes suffixed with `_millis` were renamed to remove said suffix and have the `datetime.timedelta` type now. + +## Actor + +- The `Actor.main` method has been removed as it brings no benefits compared to using `async with Actor`. +- The `Actor.add_webhook`, `Actor.start`, `Actor.call` and `Actor.start_task` methods now accept instances of the `apify.Webhook` model instead of an untyped `dict`. +- `Actor.start`, `Actor.call`, `Actor.start_task`, `Actor.set_status_message` and `Actor.abort` return instances of the `ActorRun` model instead of an untyped `dict`. +- Upon entering the context manager (`async with Actor`), the `Actor` puts the default logging configuration in place. This can be disabled using the `configure_logging` parameter. +- The `config` parameter of `Actor` has been renamed to `configuration`. +- Event handlers registered via `Actor.on` will now receive Pydantic objects instead of untyped dicts. For example, where you would do `event['isMigrating']`, you should now use `event.is_migrating` + +## Scrapy integration + +The `apify.scrapy.utils.open_queue_with_custom_client` function is not necessary anymore and has been removed. + +## Subpackage visibility + +The following modules were made private: + +- `apify.proxy_configuration` (`ProxyConfiguration` is still exported from `apify`) +- `apify.config` (`Configuration` is still exported from `apify`) +- `apify.actor` (`Actor` is still exported from `apify`) +- `apify.event_manager` +- `apify.consts` diff --git a/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v3.md b/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v3.md new file mode 100644 index 00000000..803db6d8 --- /dev/null +++ b/website/versioned_docs/version-3.3/04_upgrading/upgrading_to_v3.md @@ -0,0 +1,159 @@ +--- +id: upgrading-to-v3 +title: Upgrading to v3 +--- + +This page summarizes the breaking changes between Apify Python SDK v2.x and v3.0. + +## Python version support + +Support for Python 3.9 has been dropped. The Apify Python SDK v3.x now requires Python 3.10 or later. Make sure your environment is running a compatible version before upgrading. + +## Changes in storages + +Apify Python SDK v3.0 includes Crawlee v1.0, which brings significant changes to the storage APIs. In Crawlee v1.0, the `Dataset`, `KeyValueStore`, and `RequestQueue` storage APIs have been updated for consistency and simplicity. Below is a detailed overview of what's new, what's changed, and what's been removed. + +See the Crawlee's [Storages guide](https://crawlee.dev/python/docs/guides/storages) for more details. + +### Dataset + +The `Dataset` API now includes several new methods, such as: + +- `get_metadata` - retrieves metadata information for the dataset. +- `purge` - completely clears the dataset, including all items (keeps the metadata only). +- `list_items` - returns the dataset's items in a list format. + +Some older methods have been removed or replaced: + +- `from_storage_object` constructor has been removed. You should now use the `open` method with either a `name` or `id` parameter. +- `get_info` method and the `storage_object` property have been replaced by the new `get_metadata` method. +- `set_metadata` method has been removed. +- `write_to_json` and `write_to_csv` methods have been removed; instead, use the `export_to` method for exporting data in different formats. + +### Key-value store + +The `KeyValueStore` API now includes several new methods, such as: + +- `get_metadata` - retrieves metadata information for the key-value store. +- `purge` - completely clears the key-value store, removing all keys and values (keeps the metadata only). +- `delete_value` - deletes a specific key and its associated value. +- `list_keys` - lists all keys in the key-value store. + +Some older methods have been removed or replaced: + +- `from_storage_object` - removed; use the `open` method with either a `name` or `id` instead. +- `get_info` and `storage_object` - replaced by the new `get_metadata` method. +- `set_metadata` method has been removed. + +### Request queue + +The `RequestQueue` API now includes several new methods, such as: + +- `get_metadata` - retrieves metadata information for the request queue. +- `purge` - completely clears the request queue, including all pending and processed requests (keeps the metadata only). +- `add_requests` - replaces the previous `add_requests_batched` method, offering the same functionality under a simpler name. + +Some older methods have been removed or replaced: + +- `from_storage_object` - removed; use the `open` method with either a `name` or `id` instead. +- `get_info` and `storage_object` - replaced by the new `get_metadata` method. +- `get_request` has argument `unique_key` instead of `request_id` as the `id` field was removed from the `Request`. +- `set_metadata` method has been removed. + +Some changes in the related model classes: + +- `resource_directory` in `RequestQueueMetadata` - removed; use the corresponding `path_to_*` property instead. +- `stats` field in `RequestQueueMetadata` - removed as it was unused. +- `RequestQueueHead` - replaced by `RequestQueueHeadWithLocks`. + +## Removed Actor.config property +- `Actor.config` property has been removed. Use `Actor.configuration` instead. + +## Default storage ids in configuration changed to None +- `Configuration.default_key_value_store_id` changed from `'default'` to `None`. +- `Configuration.default_dataset_id` changed from `'default'` to `None`. +- `Configuration.default_request_queue_id` changed from `'default'` to `None`. + +Previously using the default storage without specifying its `id` in `Configuration` would lead to using specific storage with id `'default'`. Now it will use newly created unnamed storage with `'id'` assigned by the Apify platform, consecutive calls to get the default storage will return the same storage. + +## Actor initialization and ServiceLocator changes + +`Actor` initialization and global `service_locator` services setup is more strict and predictable. +- Services in `Actor` can't be changed after calling `Actor.init`, entering the `async with Actor` context manager or after requesting them from the `Actor`. +- Services in `Actor` can be different from services in Crawler. + + +**Now (v3.0):** + +```python +from crawlee.crawlers import BasicCrawler +from crawlee.storage_clients import MemoryStorageClient +from crawlee.configuration import Configuration +from crawlee.events import LocalEventManager +from apify import Actor + +async def main(): + + async with Actor(): + # This crawler will use same services as Actor and global service_locator + crawler_1 = BasicCrawler() + + # This crawler will use custom services + custom_configuration = Configuration() + custom_event_manager = LocalEventManager.from_config(custom_configuration) + custom_storage_client = MemoryStorageClient() + crawler_2 = BasicCrawler( + configuration=custom_configuration, + event_manager=custom_event_manager, + storage_client=custom_storage_client, + ) +``` + +### Changes in storage clients + +## Explicit control over storage clients used in Actor +- It is now possible to have full control over which storage clients are used by the `Actor`. To make development of Actors convenient, the `Actor` has two storage clients. One that is used when running on Apify platform or when opening storages with `force_cloud=True` and the other client that is used when running outside the Apify platform. The `Actor` has reasonable defaults and for the majority of use-cases there is no need to change it. However, if you need to use a different storage client, you can set it up before entering `Actor` context through `service_locator`. + +**Now (v3.0):** + +```python +from crawlee import service_locator +from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient, MemoryStorageClient +from apify import Actor + + +async def main(): + service_locator.set_storage_client( + SmartApifyStorageClient( + cloud_storage_client=ApifyStorageClient(request_queue_access="single"), + local_storage_client=MemoryStorageClient() + ) + ) + async with Actor: + rq = await Actor.open_request_queue() +``` + + +## The default use of optimized ApifyRequestQueueClient + +- The default client for working with Apify platform based `RequestQueue` is now optimized and simplified client which does significantly lower amount of API calls, but does not support multiple consumers working on the same queue. It is cheaper and faster and is suitable for the majority of the use cases. +- The full client is still available, but it has to be explicitly requested via `request_queue_access="shared"` argument when using the `ApifyStorageClient`. + +**Now (v3.0):** + +```python +from crawlee import service_locator +from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient +from apify import Actor + + +async def main(): + # Full client that supports multiple consumers of the Apify Request Queue + service_locator.set_storage_client( + SmartApifyStorageClient( + cloud_storage_client=ApifyStorageClient(request_queue_access="shared"), + ) + ) + async with Actor: + rq = await Actor.open_request_queue() +``` diff --git a/website/versioned_docs/version-3.3/api-packages.json b/website/versioned_docs/version-3.3/api-packages.json new file mode 100644 index 00000000..4672c024 --- /dev/null +++ b/website/versioned_docs/version-3.3/api-packages.json @@ -0,0 +1 @@ +[{"entryPoints":{"index":{"label":"Index","path":"src/index.ts"}},"packageRoot":".","packagePath":".","packageSlug":".","packageName":"apify-sdk-python"}] \ No newline at end of file diff --git a/website/versioned_docs/version-3.3/api-typedoc.json b/website/versioned_docs/version-3.3/api-typedoc.json new file mode 100644 index 00000000..21992456 --- /dev/null +++ b/website/versioned_docs/version-3.3/api-typedoc.json @@ -0,0 +1,76320 @@ +{ + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1, + "module": "_consts", + "name": "EVENT_LISTENERS_TIMEOUT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_consts.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 6 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 2, + "module": "_consts", + "name": "BASE64_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_consts.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 8 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 3, + "module": "_consts", + "name": "ENCRYPTED_STRING_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_consts.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 9 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 4, + "module": "_consts", + "name": "ENCRYPTED_JSON_VALUE_PREFIX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_consts.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 10 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 5, + "module": "_consts", + "name": "ENCRYPTED_INPUT_VALUE_REGEXP", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_consts.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 11 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 6, + "module": "_crypto", + "name": "ENCRYPTION_KEY_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 7, + "module": "_crypto", + "name": "ENCRYPTION_IV_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 8, + "module": "_crypto", + "name": "ENCRYPTION_AUTH_TAG_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encrypts the given value using AES cipher and the password for encryption using the public key.\n\nThe encryption password is a string of encryption key and initial vector used for cipher.\nIt returns the encrypted password and encrypted value in BASE64 format.\n\n\nReturns: Encrypted password and value." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 9, + "module": "_crypto", + "name": "public_encrypt", + "parsedDocstring": { + "text": "Encrypts the given value using AES cipher and the password for encryption using the public key.\n\nThe encryption password is a string of encryption key and initial vector used for cipher.\nIt returns the encrypted password and encrypted value in BASE64 format.\n\n\nReturns: Encrypted password and value.", + "args": { + "value": "The value which should be encrypted.", + "public_key": "Public key to use for encryption.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encrypts the given value using AES cipher and the password for encryption using the public key.\n\nThe encryption password is a string of encryption key and initial vector used for cipher.\nIt returns the encrypted password and encrypted value in BASE64 format.\n\n\nReturns: Encrypted password and value." + } + ] + }, + "flags": {}, + "id": 10, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "public_encrypt", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value which should be encrypted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 11, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Public key to use for encryption.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 12, + "kind": 32768, + "kindString": "Parameter", + "name": "public_key", + "type": { + "name": "rsa.RSAPublicKey", + "type": "reference" + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypts the given encrypted value using the private key and password.\n\n\nReturns: Decrypted value." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 13, + "module": "_crypto", + "name": "private_decrypt", + "parsedDocstring": { + "text": "Decrypts the given encrypted value using the private key and password.\n\n\nReturns: Decrypted value.", + "args": { + "encrypted_password": "Password used to encrypt the private key encoded as base64 string.", + "encrypted_value": "Encrypted value to decrypt as base64 string.", + "private_key": "Private key to use for decryption.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypts the given encrypted value using the private key and password.\n\n\nReturns: Decrypted value." + } + ] + }, + "flags": {}, + "id": 14, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "private_decrypt", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password used to encrypt the private key encoded as base64 string." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15, + "kind": 32768, + "kindString": "Parameter", + "name": "encrypted_password", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encrypted value to decrypt as base64 string." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 16, + "kind": 32768, + "kindString": "Parameter", + "name": "encrypted_value", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Private key to use for decryption.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 17, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key", + "type": { + "name": "rsa.RSAPrivateKey", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 18, + "module": "_crypto", + "name": "load_private_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 118 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 19, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "load_private_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 20, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key_file_base64", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 21, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key_password", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "rsa.RSAPrivateKey", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypt input secrets." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 22, + "module": "_crypto", + "name": "decrypt_input_secrets", + "parsedDocstring": { + "text": "Decrypt input secrets." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 137 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Decrypt input secrets." + } + ] + }, + "flags": {}, + "id": 23, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "decrypt_input_secrets", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 24, + "kind": 32768, + "kindString": "Parameter", + "name": "private_key", + "type": { + "name": "rsa.RSAPrivateKey", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 25, + "kind": 32768, + "kindString": "Parameter", + "name": "input_data", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 26, + "module": "_crypto", + "name": "CHARSET", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 163 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encode the given number to base62." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 27, + "module": "_crypto", + "name": "encode_base62", + "parsedDocstring": { + "text": "Encode the given number to base62." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 166 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Encode the given number to base62." + } + ] + }, + "flags": {}, + "id": 28, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "encode_base62", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 29, + "kind": 32768, + "kindString": "Parameter", + "name": "num", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.\n\nHMAC signature is truncated to 30 characters to make it shorter.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 30, + "module": "_crypto", + "name": "create_hmac_signature", + "parsedDocstring": { + "text": "Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.\n\nHMAC signature is truncated to 30 characters to make it shorter.\n", + "args": { + "secret_key": "Secret key used for signing signatures.", + "message": "Message to be signed.\n" + }, + "returns": "Base62 encoded signature." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_crypto.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 178 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Base62 encoded signature." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.\n\nHMAC signature is truncated to 30 characters to make it shorter.\n" + } + ] + }, + "flags": {}, + "id": 31, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "create_hmac_signature", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Secret key used for signing signatures." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 32, + "kind": 32768, + "kindString": "Parameter", + "name": "secret_key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Message to be signed.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33, + "kind": 32768, + "kindString": "Parameter", + "name": "message", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 34, + "module": "_proxy_configuration", + "name": "APIFY_PROXY_VALUE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 35, + "module": "_proxy_configuration", + "name": "COUNTRY_CODE_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 36, + "module": "_proxy_configuration", + "name": "SESSION_ID_MAX_LENGTH", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the given string is a valid URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 37, + "module": "_proxy_configuration", + "name": "is_url", + "parsedDocstring": { + "text": "Check if the given string is a valid URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the given string is a valid URL." + } + ] + }, + "flags": {}, + "id": 38, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 39, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\nthe proxy will select the groups automatically." + } + ] + }, + "flags": {}, + "groups": [], + "id": 41, + "module": "_proxy_configuration", + "name": "groups", + "parsedDocstring": { + "text": "An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy). If not provided,\nthe proxy will select the groups automatically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + } + ] + }, + "flags": {}, + "groups": [], + "id": 42, + "module": "_proxy_configuration", + "name": "country_code", + "parsedDocstring": { + "text": "If set and relevant proxies are available in your Apify account, all proxied requests will use IP addresses\nthat are geolocated to the specified country. For example `GB` for IPs from Great Britain. Note that online\nservices often have their own rules for handling geolocation and thus the country selection is a best attempt\nat geolocation, rather than a guaranteed hit. This parameter is optional, by default, each proxied request is\nassigned an IP address from a random country. The country code needs to be a two letter ISO country code.\nSee the [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).\nThis parameter is optional, by default, the proxy uses all available proxy servers from all countries." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 82 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides information about a proxy connection that is used for requests." + } + ] + }, + "decorations": [ + { + "args": "('Configuration')", + "name": "docs_group" + }, + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 42, + 41 + ], + "title": "Properties" + } + ], + "id": 40, + "module": "_proxy_configuration", + "name": "ProxyInfo", + "parsedDocstring": { + "text": "Provides information about a proxy connection that is used for requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 75 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration instance.\n\nIt is highly recommended to use `Actor.create_proxy_configuration()` instead of this.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44, + "module": "_proxy_configuration", + "name": "__init__", + "parsedDocstring": { + "text": "Create a ProxyConfiguration instance.\n\nIt is highly recommended to use `Actor.create_proxy_configuration()` instead of this.\n", + "args": { + "password": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available.", + "groups": "Proxy groups which the Apify Proxy should use, if provided.", + "country_code": "Country which the Apify Proxy should use, if provided.", + "proxy_urls": "Custom proxy server URLs which should be rotated through.", + "new_url_function": "Function which returns a custom proxy URL to be used.", + "tiered_proxy_urls": "Proxy URLs arranged into tiers" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration instance.\n\nIt is highly recommended to use `Actor.create_proxy_configuration()` instead of this.\n" + } + ] + }, + "flags": {}, + "id": 45, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 46, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy groups which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 47, + "kind": 32768, + "kindString": "Parameter", + "name": "groups", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Country which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 48, + "kind": 32768, + "kindString": "Parameter", + "name": "country_code", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom proxy server URLs which should be rotated through." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 49, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_urls", + "type": { + "name": "list[str | None] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function which returns a custom proxy URL to be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 50, + "kind": 32768, + "kindString": "Parameter", + "name": "new_url_function", + "type": { + "name": "_NewUrlFunction | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "_NewUrlFunction" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy URLs arranged into tiers" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 51, + "kind": 32768, + "kindString": "Parameter", + "name": "tiered_proxy_urls", + "type": { + "name": "list[list[str | None]] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 52, + "kind": 32768, + "kindString": "Parameter", + "name": "_actor_config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "231" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 53, + "kind": 32768, + "kindString": "Parameter", + "name": "_apify_client", + "type": { + "name": "ApifyClientAsync | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ApifyClientAsync" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 54, + "module": "_proxy_configuration", + "name": "initialize", + "parsedDocstring": { + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 180 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if using proxy, if so, check the access.\n\nLoad the Apify Proxy password from API (only if not passed to constructor or through env var).\n\nOnly called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups\nif the country code is provided.\n\nYou should use the Actor.create_proxy_configuration function to create a pre-initialized\n`ProxyConfiguration` instance instead of calling this manually." + } + ] + }, + "flags": {}, + "id": 55, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 56, + "module": "_proxy_configuration", + "name": "new_proxy_info", + "parsedDocstring": { + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n", + "args": { + "session_id": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier will use the same\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\nand include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`.", + "request": "request for which the proxy info is being issued, used in proxy tier handling.", + "proxy_tier": "allows forcing the proxy tier to be used.\n" + }, + "returns": "Dictionary that represents information about the proxy and its configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 205 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Dictionary that represents information about the proxy and its configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new ProxyInfo object.\n\nUse it if you want to work with a rich representation of a proxy URL. If you need the URL string only,\nuse `ProxyConfiguration.new_url`.\n" + } + ] + }, + "flags": {}, + "id": 57, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "new_proxy_info", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the identifier of a proxy session (https://docs.apify.com/proxy#sessions).\nAll the HTTP requests going through the proxy with the same session identifier will use the same\ntarget proxy server (i.e. the same IP address). The identifier must not be longer than 50 characters\nand include only the following: `0-9`, `a-z`, `A-Z`, `\".\"`, `\"_\"` and `\"~\"`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 58, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "request for which the proxy info is being issued, used in proxy tier handling." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 59, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "allows forcing the proxy tier to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 60, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_tier", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "40" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\nno configuration option, the proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\nof proxy URLs will be rotated by the configuration, if this option is provided." + } + ] + }, + "decorations": [ + { + "args": "('Configuration')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 44, + 54, + 56 + ], + "title": "Methods" + } + ], + "id": 43, + "module": "_proxy_configuration", + "name": "ProxyConfiguration", + "parsedDocstring": { + "text": "Configures a connection to a proxy server with the provided options.\n\nProxy servers are used to prevent target websites from blocking your crawlers based on IP address rate limits or\nblacklists. The default servers used by this class are managed by [Apify Proxy](https://docs.apify.com/proxy).\nTo be able to use Apify Proxy, you need an Apify account and access to the selected proxies. If you provide\nno configuration option, the proxies will be managed automatically using a smart algorithm.\n\nIf you want to use your own proxies, use the `proxy_urls` or `new_url_function` constructor options. Your list\nof proxy URLs will be rotated by the configuration, if this option is provided." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 94 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 61, + "module": "log", + "name": "logger_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 8 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 62, + "module": "log", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 11 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 63, + "module": "log", + "name": "ActorLogFormatter", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/log.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 64, + "module": "__init__", + "name": "__version__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/__init__.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pricing model for an Actor." + } + ] + }, + "flags": {}, + "groups": [], + "id": 65, + "module": "_models", + "name": "PricingModel", + "parsedDocstring": { + "text": "Pricing model for an Actor." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Defines the general access level for the resource." + } + ] + }, + "flags": {}, + "groups": [], + "id": 66, + "module": "_models", + "name": "GeneralAccess", + "parsedDocstring": { + "text": "Defines the general access level for the resource." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 68, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 69, + "module": "_models", + "name": "actor_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='actorId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 70, + "module": "_models", + "name": "actor_task_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='actorTaskId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 71, + "module": "_models", + "name": "actor_run_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 27 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='actorRunId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Condition for triggering a webhook." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 69, + 71, + 70, + 68 + ], + "title": "Properties" + } + ], + "id": 67, + "module": "_models", + "name": "WebhookCondition", + "parsedDocstring": { + "text": "Condition for triggering a webhook." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Status of a webhook dispatch." + } + ] + }, + "flags": {}, + "groups": [], + "id": 72, + "module": "_models", + "name": "WebhookDispatchStatus", + "parsedDocstring": { + "text": "Status of a webhook dispatch." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 74, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 75, + "module": "_models", + "name": "status", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 39 + } + ], + "type": { + "name": "WebhookDispatchStatus", + "type": "reference", + "target": "72" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 76, + "module": "_models", + "name": "finished_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Information about a webhook dispatch." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 76, + 74, + 75 + ], + "title": "Properties" + } + ], + "id": 73, + "module": "_models", + "name": "ExampleWebhookDispatch", + "parsedDocstring": { + "text": "Information about a webhook dispatch." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 78, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 79, + "module": "_models", + "name": "total_dispatches", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 48 + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Statistics about webhook dispatches." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 78, + 79 + ], + "title": "Properties" + } + ], + "id": 77, + "module": "_models", + "name": "WebhookStats", + "parsedDocstring": { + "text": "Statistics about webhook dispatches." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 81, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 82, + "module": "_models", + "name": "event_types", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "WebhookEventType" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 83, + "module": "_models", + "name": "request_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 84, + "module": "_models", + "name": "id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='id')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 85, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 65 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 86, + "module": "_models", + "name": "modified_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 66 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='modifiedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 87, + "module": "_models", + "name": "user_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='userId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 88, + "module": "_models", + "name": "is_ad_hoc", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 68 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='isAdHoc')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 89, + "module": "_models", + "name": "should_interpolate_strings", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='shouldInterpolateStrings')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 90, + "module": "_models", + "name": "condition", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Annotated[WebhookCondition | None, Field(alias='condition')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "WebhookCondition", + "target": "67" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 91, + "module": "_models", + "name": "ignore_ssl_errors", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='ignoreSslErrors')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 92, + "module": "_models", + "name": "do_not_retry", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 72 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='doNotRetry')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 93, + "module": "_models", + "name": "payload_template", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73 + } + ], + "type": { + "name": "Annotated[ str | None, Field(alias='payloadTemplate', description='Template for the payload sent by the webhook'), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 94, + "module": "_models", + "name": "headers_template", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 77 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='headersTemplate')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 95, + "module": "_models", + "name": "description", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='description')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 96, + "module": "_models", + "name": "last_dispatch", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 79 + } + ], + "type": { + "name": "Annotated[ExampleWebhookDispatch | None, Field(alias='lastDispatch')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ExampleWebhookDispatch", + "target": "73" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 97, + "module": "_models", + "name": "stats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 80 + } + ], + "type": { + "name": "Annotated[WebhookStats | None, Field(alias='stats')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "WebhookStats", + "target": "77" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 90, + 85, + 95, + 92, + 82, + 94, + 84, + 91, + 88, + 96, + 81, + 86, + 93, + 83, + 89, + 97, + 87 + ], + "title": "Properties" + } + ], + "id": 80, + "module": "_models", + "name": "Webhook", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 99, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 85 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 100, + "module": "_models", + "name": "origin", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "type": { + "name": "MetaOrigin", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 101, + "module": "_models", + "name": "client_ip", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 88 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='clientIp')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 102, + "module": "_models", + "name": "user_agent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='userAgent')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 103, + "module": "_models", + "name": "schedule_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 90 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='scheduleId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 104, + "module": "_models", + "name": "scheduled_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='scheduledAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 101, + 99, + 100, + 103, + 104, + 102 + ], + "title": "Properties" + } + ], + "id": 98, + "module": "_models", + "name": "ActorRunMeta", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 84 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 106, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 96 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 107, + "module": "_models", + "name": "input_body_len", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 98 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='inputBodyLen')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 108, + "module": "_models", + "name": "migration_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='migrationCount')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 109, + "module": "_models", + "name": "reboot_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='rebootCount')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 110, + "module": "_models", + "name": "restart_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 101 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 111, + "module": "_models", + "name": "resurrect_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 102 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 112, + "module": "_models", + "name": "mem_avg_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='memAvgBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 113, + "module": "_models", + "name": "mem_max_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='memMaxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 114, + "module": "_models", + "name": "mem_current_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 105 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='memCurrentBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 115, + "module": "_models", + "name": "cpu_avg_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuAvgUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 116, + "module": "_models", + "name": "cpu_max_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 107 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuMaxUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 117, + "module": "_models", + "name": "cpu_current_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='cpuCurrentUsage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 118, + "module": "_models", + "name": "net_rx_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 109 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='netRxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 119, + "module": "_models", + "name": "net_tx_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 110 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='netTxBytes')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 120, + "module": "_models", + "name": "duration_millis", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 111 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='durationMillis')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 121, + "module": "_models", + "name": "run_time_secs", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='runTimeSecs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 122, + "module": "_models", + "name": "metamorph", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='metamorph')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 123, + "module": "_models", + "name": "compute_units", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 114 + } + ], + "type": { + "name": "float", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 123, + 115, + 117, + 116, + 120, + 107, + 112, + 114, + 113, + 122, + 108, + 106, + 118, + 119, + 109, + 110, + 111, + 121 + ], + "title": "Properties" + } + ], + "id": 105, + "module": "_models", + "name": "ActorRunStats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 95 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 125, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 119 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 126, + "module": "_models", + "name": "build", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 121 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 127, + "module": "_models", + "name": "timeout_secs", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 122 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 128, + "module": "_models", + "name": "memory_mbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 123 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 129, + "module": "_models", + "name": "disk_mbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 124 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 130, + "module": "_models", + "name": "max_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 125 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='maxItems')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 131, + "module": "_models", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 126 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='maxTotalChargeUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 126, + 129, + 130, + 131, + 128, + 125, + 127 + ], + "title": "Properties" + } + ], + "id": 124, + "module": "_models", + "name": "ActorRunOptions", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 118 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 133, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 131 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 134, + "module": "_models", + "name": "actor_compute_units", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 133 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 135, + "module": "_models", + "name": "dataset_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 134 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='DATASET_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 136, + "module": "_models", + "name": "dataset_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 135 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='DATASET_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 137, + "module": "_models", + "name": "key_value_store_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 136 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='KEY_VALUE_STORE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 138, + "module": "_models", + "name": "key_value_store_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 137 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='KEY_VALUE_STORE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 139, + "module": "_models", + "name": "key_value_store_lists", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 138 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='KEY_VALUE_STORE_LISTS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 140, + "module": "_models", + "name": "request_queue_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 139 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='REQUEST_QUEUE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 141, + "module": "_models", + "name": "request_queue_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='REQUEST_QUEUE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 142, + "module": "_models", + "name": "data_transfer_internal_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 141 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 143, + "module": "_models", + "name": "data_transfer_external_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 142 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 144, + "module": "_models", + "name": "proxy_residential_transfer_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 143 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 145, + "module": "_models", + "name": "proxy_serps", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 144 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='PROXY_SERPS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 134, + 143, + 142, + 135, + 136, + 139, + 137, + 138, + 133, + 144, + 145, + 140, + 141 + ], + "title": "Properties" + } + ], + "id": 132, + "module": "_models", + "name": "ActorRunUsage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 130 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 147, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 151 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 148, + "module": "_models", + "name": "actor_compute_units", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 153 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 149, + "module": "_models", + "name": "dataset_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 154 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATASET_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 150, + "module": "_models", + "name": "dataset_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 155 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATASET_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 151, + "module": "_models", + "name": "key_value_store_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 156 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 152, + "module": "_models", + "name": "key_value_store_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 153, + "module": "_models", + "name": "key_value_store_lists", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 158 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='KEY_VALUE_STORE_LISTS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 154, + "module": "_models", + "name": "request_queue_reads", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 159 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='REQUEST_QUEUE_READS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 155, + "module": "_models", + "name": "request_queue_writes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 160 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='REQUEST_QUEUE_WRITES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 156, + "module": "_models", + "name": "data_transfer_internal_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 161 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 157, + "module": "_models", + "name": "data_transfer_external_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 162 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 158, + "module": "_models", + "name": "proxy_residential_transfer_gbytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 163 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 159, + "module": "_models", + "name": "proxy_serps", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 164 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='PROXY_SERPS')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Resource usage costs in USD." + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 148, + 157, + 156, + 149, + 150, + 153, + 151, + 152, + 147, + 158, + 159, + 154, + 155 + ], + "title": "Properties" + } + ], + "id": 146, + "module": "_models", + "name": "ActorRunUsageUsd", + "parsedDocstring": { + "text": "Resource usage costs in USD." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 148 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 161, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 170 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 162, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 172 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 163, + "module": "_models", + "name": "actor_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 173 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 164, + "module": "_models", + "name": "build_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 174 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 165, + "module": "_models", + "name": "input_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 175 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='inputKey')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Information about a metamorph event that occurred during the run." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 163, + 164, + 162, + 165, + 161 + ], + "title": "Properties" + } + ], + "id": 160, + "module": "_models", + "name": "Metamorph", + "parsedDocstring": { + "text": "Information about a metamorph event that occurred during the run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 167 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 167, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 168, + "module": "_models", + "name": "apify_margin_percentage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='apifyMarginPercentage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 169, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 182 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 170, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='startedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 171, + "module": "_models", + "name": "notified_about_future_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 184 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutFutureChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 172, + "module": "_models", + "name": "notified_about_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 173, + "module": "_models", + "name": "reason_for_change", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 186 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='reasonForChange')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 168, + 169, + 167, + 172, + 171, + 173, + 170 + ], + "title": "Properties" + } + ], + "id": 166, + "module": "_models", + "name": "CommonActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 178 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedBy": [ + { + "name": "FreeActorPricingInfo", + "target": "174", + "type": "reference" + }, + { + "name": "FlatPricePerMonthActorPricingInfo", + "target": "176", + "type": "reference" + }, + { + "name": "PricePerDatasetItemActorPricingInfo", + "target": "180", + "type": "reference" + }, + { + "name": "PayPerEventActorPricingInfo", + "target": "192", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 175, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 190 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "FREE" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1167, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.model_config", + "target": 167, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1168, + "module": "_models", + "name": "apify_margin_percentage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='apifyMarginPercentage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.apify_margin_percentage", + "target": 168, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1169, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 182 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.created_at", + "target": 169, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1170, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='startedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.started_at", + "target": 170, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1171, + "module": "_models", + "name": "notified_about_future_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 184 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutFutureChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_future_change_at", + "target": 171, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1172, + "module": "_models", + "name": "notified_about_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_change_at", + "target": 172, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1173, + "module": "_models", + "name": "reason_for_change", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 186 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='reasonForChange')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.reason_for_change", + "target": 173, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1168, + 1169, + 1167, + 1172, + 1171, + 175, + 1173, + 1170 + ], + "title": "Properties" + } + ], + "id": 174, + "module": "_models", + "name": "FreeActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 189 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "CommonActorPricingInfo", + "target": "166", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 177, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 194 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "FLAT_PRICE_PER_MONTH" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 178, + "module": "_models", + "name": "trial_minutes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 195 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 179, + "module": "_models", + "name": "price_per_unit_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 196 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1174, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.model_config", + "target": 167, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1175, + "module": "_models", + "name": "apify_margin_percentage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='apifyMarginPercentage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.apify_margin_percentage", + "target": 168, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1176, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 182 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.created_at", + "target": 169, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1177, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='startedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.started_at", + "target": 170, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1178, + "module": "_models", + "name": "notified_about_future_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 184 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutFutureChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_future_change_at", + "target": 171, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1179, + "module": "_models", + "name": "notified_about_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_change_at", + "target": 172, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1180, + "module": "_models", + "name": "reason_for_change", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 186 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='reasonForChange')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.reason_for_change", + "target": 173, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1175, + 1176, + 1174, + 1179, + 1178, + 179, + 177, + 1180, + 1177, + 178 + ], + "title": "Properties" + } + ], + "id": 176, + "module": "_models", + "name": "FlatPricePerMonthActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 193 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "CommonActorPricingInfo", + "target": "166", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 181, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 200 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "PRICE_PER_DATASET_ITEM" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 182, + "module": "_models", + "name": "unit_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 201 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 183, + "module": "_models", + "name": "price_per_unit_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 202 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1181, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.model_config", + "target": 167, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1182, + "module": "_models", + "name": "apify_margin_percentage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='apifyMarginPercentage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.apify_margin_percentage", + "target": 168, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1183, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 182 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.created_at", + "target": 169, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1184, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='startedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.started_at", + "target": 170, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1185, + "module": "_models", + "name": "notified_about_future_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 184 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutFutureChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_future_change_at", + "target": 171, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1186, + "module": "_models", + "name": "notified_about_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_change_at", + "target": 172, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1187, + "module": "_models", + "name": "reason_for_change", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 186 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='reasonForChange')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.reason_for_change", + "target": 173, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1182, + 1183, + 1181, + 1186, + 1185, + 183, + 181, + 1187, + 1184, + 182 + ], + "title": "Properties" + } + ], + "id": 180, + "module": "_models", + "name": "PricePerDatasetItemActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 199 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "CommonActorPricingInfo", + "target": "166", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 185, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 206 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 186, + "module": "_models", + "name": "event_price_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 208 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 187, + "module": "_models", + "name": "event_title", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 209 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 188, + "module": "_models", + "name": "event_description", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 210 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='eventDescription')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 188, + 186, + 187, + 185 + ], + "title": "Properties" + } + ], + "id": 184, + "module": "_models", + "name": "ActorChargeEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 205 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 190, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 214 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 191, + "module": "_models", + "name": "actor_charge_events", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 216 + } + ], + "type": { + "name": "Annotated[dict[str, ActorChargeEvent] | None, Field(alias='actorChargeEvents')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "ActorChargeEvent", + "target": "184" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 191, + 190 + ], + "title": "Properties" + } + ], + "id": 189, + "module": "_models", + "name": "PricingPerEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 213 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 193, + "module": "_models", + "name": "pricing_model", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 220 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "PAY_PER_EVENT" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 194, + "module": "_models", + "name": "pricing_per_event", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 221 + } + ], + "type": { + "name": "PricingPerEvent", + "type": "reference", + "target": "189" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 195, + "module": "_models", + "name": "minimal_max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 222 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='minimalMaxTotalChargeUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1188, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.model_config", + "target": 167, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1189, + "module": "_models", + "name": "apify_margin_percentage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='apifyMarginPercentage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.apify_margin_percentage", + "target": 168, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1190, + "module": "_models", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 182 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='createdAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.created_at", + "target": 169, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1191, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='startedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.started_at", + "target": 170, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1192, + "module": "_models", + "name": "notified_about_future_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 184 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutFutureChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_future_change_at", + "target": 171, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1193, + "module": "_models", + "name": "notified_about_change_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='notifiedAboutChangeAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.notified_about_change_at", + "target": 172, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1194, + "module": "_models", + "name": "reason_for_change", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 186 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='reasonForChange')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + }, + "inheritedFrom": { + "name": "CommonActorPricingInfo.reason_for_change", + "target": 173, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1189, + 1190, + 195, + 1188, + 1193, + 1192, + 193, + 194, + 1194, + 1191 + ], + "title": "Properties" + } + ], + "id": 192, + "module": "_models", + "name": "PayPerEventActorPricingInfo", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 219 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "CommonActorPricingInfo", + "target": "166", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 197, + "module": "_models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 227 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 198, + "module": "_models", + "name": "id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 229 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 199, + "module": "_models", + "name": "act_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 230 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 200, + "module": "_models", + "name": "user_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 231 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 201, + "module": "_models", + "name": "actor_task_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 232 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='actorTaskId')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 202, + "module": "_models", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 233 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 203, + "module": "_models", + "name": "finished_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 234 + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='finishedAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 204, + "module": "_models", + "name": "status", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 235 + } + ], + "type": { + "name": "ActorJobStatus", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 205, + "module": "_models", + "name": "status_message", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 236 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='statusMessage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 206, + "module": "_models", + "name": "is_status_message_terminal", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 237 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='isStatusMessageTerminal')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 207, + "module": "_models", + "name": "meta", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 238 + } + ], + "type": { + "name": "ActorRunMeta", + "type": "reference", + "target": "98" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 208, + "module": "_models", + "name": "stats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "type": { + "name": "ActorRunStats", + "type": "reference", + "target": "105" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 209, + "module": "_models", + "name": "options", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 240 + } + ], + "type": { + "name": "ActorRunOptions", + "type": "reference", + "target": "124" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 210, + "module": "_models", + "name": "build_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 241 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 211, + "module": "_models", + "name": "exit_code", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 242 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='exitCode')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 212, + "module": "_models", + "name": "general_access", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 243 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='generalAccess')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 213, + "module": "_models", + "name": "default_key_value_store_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 244 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 214, + "module": "_models", + "name": "default_dataset_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 245 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 215, + "module": "_models", + "name": "default_request_queue_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 246 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 216, + "module": "_models", + "name": "build_number", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 247 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='buildNumber')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 217, + "module": "_models", + "name": "container_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 248 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='containerUrl')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 218, + "module": "_models", + "name": "is_container_server_ready", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='isContainerServerReady')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 219, + "module": "_models", + "name": "git_branch_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 250 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='gitBranchName')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 220, + "module": "_models", + "name": "usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 251 + } + ], + "type": { + "name": "Annotated[ActorRunUsage | None, Field(alias='usage')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRunUsage", + "target": "132" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 221, + "module": "_models", + "name": "usage_total_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 252 + } + ], + "type": { + "name": "Annotated[float | None, Field(alias='usageTotalUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 222, + "module": "_models", + "name": "usage_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 253 + } + ], + "type": { + "name": "Annotated[ActorRunUsageUsd | None, Field(alias='usageUsd')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRunUsageUsd", + "target": "146" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 223, + "module": "_models", + "name": "pricing_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 254 + } + ], + "type": { + "name": "Annotated[ FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo | PricePerDatasetItemActorPricingInfo | PayPerEventActorPricingInfo | None, Field(alias='pricingInfo', discriminator='pricing_model'), ]", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "FreeActorPricingInfo", + "target": "174" + }, + { + "type": "reference", + "name": "FlatPricePerMonthActorPricingInfo", + "target": "176" + } + ] + }, + { + "type": "reference", + "name": "PricePerDatasetItemActorPricingInfo", + "target": "180" + } + ] + }, + { + "type": "reference", + "name": "PayPerEventActorPricingInfo", + "target": "192" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 224, + "module": "_models", + "name": "charged_event_counts", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 262 + } + ], + "type": { + "name": "Annotated[ dict[str, int] | None, Field(alias='chargedEventCounts'), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "int" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 225, + "module": "_models", + "name": "metamorphs", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 266 + } + ], + "type": { + "name": "Annotated[list[Metamorph] | None, Field(alias='metamorphs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Metamorph", + "target": "160" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 199, + 201, + 210, + 216, + 224, + 217, + 214, + 213, + 215, + 211, + 203, + 212, + 219, + 198, + 218, + 206, + 207, + 225, + 197, + 209, + 223, + 202, + 208, + 204, + 205, + 220, + 221, + 222, + 200 + ], + "title": "Properties" + } + ], + "id": 196, + "module": "_models", + "name": "ActorRun", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 226 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 226, + "module": "_configuration", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 228, + "module": "_configuration", + "name": "key_value_stores", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 229, + "module": "_configuration", + "name": "datasets", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 46 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 230, + "module": "_configuration", + "name": "request_queues", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 47 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mapping of storage aliases to their IDs, grouped by storage type.\n\nPopulated from the `ACTOR_STORAGES_JSON` env var that the Apify platform sets when an Actor declares\nnamed storages in its `actor.json` schema. Each key maps a user-defined alias (e.g. `'custom'`)\nto the platform-assigned storage ID." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 229, + 228, + 230 + ], + "title": "Properties" + } + ], + "id": 227, + "module": "_configuration", + "name": "ActorStorages", + "parsedDocstring": { + "text": "Mapping of storage aliases to their IDs, grouped by storage type.\n\nPopulated from the `ACTOR_STORAGES_JSON` env var that the Apify platform sets when an Actor declares\nnamed storages in its `actor.json` schema. Each key maps a user-defined alias (e.g. `'custom'`)\nto the platform-assigned storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 232, + "module": "_configuration", + "name": "actor_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 82 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_id', 'apify_actor_id', 'apify_act_id', ), description='ID of the Actor', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 233, + "module": "_configuration", + "name": "actor_full_name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 94 + } + ], + "type": { + "name": "Annotated[ str | None, Field( description='Full name of the Actor', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 234, + "module": "_configuration", + "name": "actor_permission_level", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 101 + } + ], + "type": { + "name": "Annotated[ str | None, Field( description='Permission level the Actor is run under.', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 235, + "module": "_configuration", + "name": "actor_run_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_run_id', 'apify_actor_run_id', 'apify_act_run_id', ), description='ID of the Actor run', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 236, + "module": "_configuration", + "name": "actor_build_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 120 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_build_id', 'apify_actor_build_id', ), description='ID of the Actor build used in the run', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 237, + "module": "_configuration", + "name": "actor_build_number", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 131 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_build_number', 'apify_actor_build_number', ), description='Build number of the Actor build used in the run', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 238, + "module": "_configuration", + "name": "actor_build_tags", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 142 + } + ], + "type": { + "name": "Annotated[ list[str] | None, Field( description='Build tags of the Actor build used in the run', ), BeforeValidator(_transform_to_list), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 239, + "module": "_configuration", + "name": "actor_task_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 150 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_task_id', 'apify_actor_task_id', ), description='ID of the Actor task. Empty if Actor is run outside of any task, e.g. directly using the API', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 240, + "module": "_configuration", + "name": "actor_events_ws_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 161 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_events_websocket_url', 'apify_actor_events_ws_url', ), description='Websocket URL where Actor may listen for events from Actor platform', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 241, + "module": "_configuration", + "name": "api_base_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 172 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 242, + "module": "_configuration", + "name": "api_public_base_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 180 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 243, + "module": "_configuration", + "name": "dedicated_cpus", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 188 + } + ], + "type": { + "name": "Annotated[ float | None, Field( alias='apify_dedicated_cpus', description='Number of CPU cores reserved for the actor, based on allocated memory', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "float" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 244, + "module": "_configuration", + "name": "default_dataset_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 196 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_default_dataset_id', 'apify_default_dataset_id', ), description='Default dataset ID used by the Apify storage client when no ID or name is provided.', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 245, + "module": "_configuration", + "name": "default_key_value_store_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 207 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_default_key_value_store_id', 'apify_default_key_value_store_id', ), description='Default key-value store ID for the Apify storage client when no ID or name is provided.', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 246, + "module": "_configuration", + "name": "default_request_queue_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 218 + } + ], + "type": { + "name": "Annotated[ str | None, Field( validation_alias=AliasChoices( 'actor_default_request_queue_id', 'apify_default_request_queue_id', ), description='Default request queue ID for the Apify storage client when no ID or name is provided.', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 247, + "module": "_configuration", + "name": "disable_outdated_warning", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 229 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 248, + "module": "_configuration", + "name": "fact", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 238 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='apify_fact')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 249, + "module": "_configuration", + "name": "input_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 240 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 250, + "module": "_configuration", + "name": "input_secrets_private_key_file", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 252 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_input_secrets_private_key_file', description='Path to the secret key used to decrypt Secret inputs.', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 251, + "module": "_configuration", + "name": "input_secrets_private_key_passphrase", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 260 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_input_secrets_private_key_passphrase', description='Passphrase for the input secret key', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 252, + "module": "_configuration", + "name": "is_at_home", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 268 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 253, + "module": "_configuration", + "name": "latest_sdk_version", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 276 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_sdk_latest_version', description='Specifies the most recent release version of the Apify SDK for Javascript. Used for ' 'checking for updates.', ), deprecated('SDK version checking is not supported for the Python SDK'), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 254, + "module": "_configuration", + "name": "log_format", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 286 + } + ], + "type": { + "name": "Annotated[ str | None, Field(alias='apify_log_format'), deprecated('Adjust the log format in code instead'), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 255, + "module": "_configuration", + "name": "max_paid_dataset_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 292 + } + ], + "type": { + "name": "Annotated[ int | None, Field( alias='actor_max_paid_dataset_items', description='For paid-per-result Actors, the user-set limit on returned results. Do not exceed this limit', ), BeforeValidator(lambda val: val if val != '' else None), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 256, + "module": "_configuration", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 301 + } + ], + "type": { + "name": "Annotated[ Decimal | None, Field( alias='actor_max_total_charge_usd', description='For pay-per-event Actors, the user-set limit on total charges. Do not exceed this limit', ), BeforeValidator(lambda val: val if val != '' else None), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Decimal" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 257, + "module": "_configuration", + "name": "test_pay_per_event", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 310 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 258, + "module": "_configuration", + "name": "meta_origin", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 318 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_meta_origin', description='Specifies how an Actor run was started', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 259, + "module": "_configuration", + "name": "metamorph_after_sleep", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 326 + } + ], + "type": { + "name": "timedelta_ms", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 260, + "module": "_configuration", + "name": "proxy_hostname", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 334 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 261, + "module": "_configuration", + "name": "proxy_password", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 342 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_proxy_password', description='Password to the Apify proxy', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 262, + "module": "_configuration", + "name": "proxy_port", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 350 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 263, + "module": "_configuration", + "name": "proxy_status_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 358 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 264, + "module": "_configuration", + "name": "started_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 366 + } + ], + "type": { + "name": "Annotated[ datetime | None, Field( validation_alias=AliasChoices( 'actor_started_at', 'apify_started_at', ), description='Date when the Actor was started', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 265, + "module": "_configuration", + "name": "timeout_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 377 + } + ], + "type": { + "name": "Annotated[ datetime | None, Field( validation_alias=AliasChoices( 'actor_timeout_at', 'apify_timeout_at', ), description='Date when the Actor will time out', ), BeforeValidator(lambda val: val if val != '' else None), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 266, + "module": "_configuration", + "name": "standby_port", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 389 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 267, + "module": "_configuration", + "name": "standby_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 398 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 268, + "module": "_configuration", + "name": "token", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 407 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_token', description='API token of the user who started the Actor', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 269, + "module": "_configuration", + "name": "user_id", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 415 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_user_id', description='ID of the user who started the Actor. May differ from the Actor owner', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 270, + "module": "_configuration", + "name": "user_is_paying", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 423 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 271, + "module": "_configuration", + "name": "web_server_port", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 432 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 272, + "module": "_configuration", + "name": "web_server_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 444 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 273, + "module": "_configuration", + "name": "workflow_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 455 + } + ], + "type": { + "name": "Annotated[ str | None, Field( alias='apify_workflow_key', description='Identifier used for grouping related runs and API calls together', ), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 274, + "module": "_configuration", + "name": "actor_pricing_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 463 + } + ], + "type": { + "name": "Annotated[ FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo | PricePerDatasetItemActorPricingInfo | PayPerEventActorPricingInfo | None, Field( alias='apify_actor_pricing_info', description='JSON string with prising info of the actor', discriminator='pricing_model', ), BeforeValidator(lambda data: json.loads(data) if isinstance(data, str) else data or None), ]", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "FreeActorPricingInfo", + "target": "174" + }, + { + "type": "reference", + "name": "FlatPricePerMonthActorPricingInfo", + "target": "176" + } + ] + }, + { + "type": "reference", + "name": "PricePerDatasetItemActorPricingInfo", + "target": "180" + } + ] + }, + { + "type": "reference", + "name": "PayPerEventActorPricingInfo", + "target": "192" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 275, + "module": "_configuration", + "name": "charged_event_counts", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 477 + } + ], + "type": { + "name": "Annotated[ dict[str, int] | None, Field( alias='apify_charged_actor_event_counts', description='Counts of events that were charged for the actor', ), BeforeValidator(lambda data: json.loads(data) if isinstance(data, str) else data or None), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "int" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 276, + "module": "_configuration", + "name": "actor_storages", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 486 + } + ], + "type": { + "name": "Annotated[ ActorStorages | None, Field( alias='actor_storages_json', description='Mapping of storage aliases to their platform-assigned IDs.', ), BeforeValidator(_load_storage_keys), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorStorages", + "target": "227" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + } + ] + }, + "decorations": [ + { + "args": "(mode='after')", + "name": "model_validator" + } + ], + "flags": {}, + "groups": [], + "id": 277, + "module": "_configuration", + "name": "disable_browser_sandbox_on_platform", + "parsedDocstring": { + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 496 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Disable the browser sandbox mode when running on the Apify platform.\n\nRunning in environment where `is_at_home` is True does not benefit from browser sandbox as it is already running\nin a container. It can be on the contrary undesired as the process in the container might be running as root and\nthis will crash chromium that was started with browser sandbox mode." + } + ] + }, + "flags": {}, + "id": 278, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "disable_browser_sandbox_on_platform", + "parameters": [], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 279, + "module": "_configuration", + "name": "canonical_input_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 509 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 280, + "module": "_configuration", + "name": "input_key_candidates", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 513 + } + ], + "type": { + "name": "set", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the global instance of the configuration.\n\nThis method ensures that ApifyConfiguration is returned, even if CrawleeConfiguration was set in the\nservice locator." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 281, + "module": "_configuration", + "name": "get_global_configuration", + "parsedDocstring": { + "text": "Retrieve the global instance of the configuration.\n\nThis method ensures that ApifyConfiguration is returned, even if CrawleeConfiguration was set in the\nservice locator." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 517 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the global instance of the configuration.\n\nThis method ensures that ApifyConfiguration is returned, even if CrawleeConfiguration was set in the\nservice locator." + } + ] + }, + "flags": {}, + "id": 282, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_global_configuration", + "parameters": [], + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create Apify Configuration from existing Crawlee Configuration.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 283, + "module": "_configuration", + "name": "from_configuration", + "parsedDocstring": { + "text": "Create Apify Configuration from existing Crawlee Configuration.\n", + "args": { + "configuration": "The existing Crawlee Configuration.\n" + }, + "returns": "The created Apify Configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 538 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The created Apify Configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create Apify Configuration from existing Crawlee Configuration.\n" + } + ] + }, + "flags": {}, + "id": 284, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_configuration", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The existing Crawlee Configuration.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 285, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration", + "type": "reference" + } + } + ], + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A class for specifying the configuration of an Actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + } + ] + }, + "decorations": [ + { + "args": "('Configuration')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 277, + 283, + 281 + ], + "title": "Methods" + }, + { + "children": [ + 236, + 237, + 238, + 240, + 233, + 232, + 234, + 274, + 235, + 276, + 239, + 241, + 242, + 279, + 275, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 280, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 258, + 259, + 260, + 261, + 262, + 263, + 266, + 267, + 264, + 257, + 265, + 268, + 269, + 270, + 271, + 272, + 273 + ], + "title": "Properties" + } + ], + "id": 231, + "module": "_configuration", + "name": "Configuration", + "parsedDocstring": { + "text": "A class for specifying the configuration of an Actor.\n\nCan be used either globally via `Configuration.get_global_configuration()`,\nor it can be specific to each `Actor` instance on the `actor.config` property." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 75 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 286, + "module": "_actor", + "name": "MainReturnType", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 54 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 288, + "module": "_actor", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "configuration": "The Actor configuration to use. If not provided, a default configuration is created.", + "configure_logging": "Whether to set up the default logging configuration.", + "exit_process": "Whether the Actor should call `sys.exit` when the context manager exits.\nDefaults to True, except in IPython, Pytest, and Scrapy environments.", + "exit_code": "The exit code the Actor should use when exiting.", + "status_message": "Final status message to display upon Actor termination.", + "event_listeners_timeout": "Maximum time to wait for Actor event listeners to complete before exiting.", + "cleanup_timeout": "Maximum time to wait for cleanup tasks to finish." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 289, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor configuration to use. If not provided, a default configuration is created." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 290, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "231" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether to set up the default logging configuration." + } + ] + }, + "defaultValue": "True", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 291, + "kind": 32768, + "kindString": "Parameter", + "name": "configure_logging", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the Actor should call `sys.exit` when the context manager exits.\nDefaults to True, except in IPython, Pytest, and Scrapy environments." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 292, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_process", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code the Actor should use when exiting." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 293, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Final status message to display upon Actor termination." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 294, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum time to wait for Actor event listeners to complete before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 295, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum time to wait for cleanup tasks to finish." + } + ] + }, + "defaultValue": "timedelta(seconds=30)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 296, + "kind": 32768, + "kindString": "Parameter", + "name": "cleanup_timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the Actor context.\n\nInitializes the Actor when used in an `async with` block. This method:\n\n- Sets up local or cloud storage clients depending on whether the Actor runs locally or on the Apify platform.\n- Configures the event manager and starts periodic state persistence.\n- Initializes the charging manager for handling charging events.\n- Configures logging after all core services are registered.\n\nThis method must be called exactly once per Actor instance. Re-initializing an Actor or having multiple\nactive Actor instances is not standard usage and may lead to warnings or unexpected behavior." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 297, + "module": "_actor", + "name": "__aenter__", + "parsedDocstring": { + "text": "Enter the Actor context.\n\nInitializes the Actor when used in an `async with` block. This method:\n\n- Sets up local or cloud storage clients depending on whether the Actor runs locally or on the Apify platform.\n- Configures the event manager and starts periodic state persistence.\n- Initializes the charging manager for handling charging events.\n- Configures logging after all core services are registered.\n\nThis method must be called exactly once per Actor instance. Re-initializing an Actor or having multiple\nactive Actor instances is not standard usage and may lead to warnings or unexpected behavior." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 153 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the Actor context.\n\nInitializes the Actor when used in an `async with` block. This method:\n\n- Sets up local or cloud storage clients depending on whether the Actor runs locally or on the Apify platform.\n- Configures the event manager and starts periodic state persistence.\n- Initializes the charging manager for handling charging events.\n- Configures logging after all core services are registered.\n\nThis method must be called exactly once per Actor instance. Re-initializing an Actor or having multiple\nactive Actor instances is not standard usage and may lead to warnings or unexpected behavior." + } + ] + }, + "flags": {}, + "id": 298, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the Actor context.\n\nIf the block exits with an exception, the Actor fails with a non-zero exit code.\nOtherwise, it exits cleanly. In both cases the Actor:\n\n- Cancels periodic `PERSIST_STATE` events.\n- Sends a final `PERSIST_STATE` event.\n- Waits for all event listeners to finish.\n- Stops the event manager and the charging manager.\n- Optionally terminates the process with the selected exit code." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 299, + "module": "_actor", + "name": "__aexit__", + "parsedDocstring": { + "text": "Exit the Actor context.\n\nIf the block exits with an exception, the Actor fails with a non-zero exit code.\nOtherwise, it exits cleanly. In both cases the Actor:\n\n- Cancels periodic `PERSIST_STATE` events.\n- Sends a final `PERSIST_STATE` event.\n- Waits for all event listeners to finish.\n- Stops the event manager and the charging manager.\n- Optionally terminates the process with the selected exit code." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 210 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the Actor context.\n\nIf the block exits with an exception, the Actor fails with a non-zero exit code.\nOtherwise, it exits cleanly. In both cases the Actor:\n\n- Cancels periodic `PERSIST_STATE` events.\n- Sends a final `PERSIST_STATE` event.\n- Waits for all event listeners to finish.\n- Stops the event manager and the charging manager.\n- Optionally terminates the process with the selected exit code." + } + ] + }, + "flags": {}, + "id": 300, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 301, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 302, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 303, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 304, + "module": "_actor", + "name": "__repr__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 268 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 305, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__repr__", + "parameters": [], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Make a new Actor instance with a non-default configuration.\n\nThis is necessary due to the lazy object proxying of the global `Actor` instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 306, + "module": "_actor", + "name": "__call__", + "parsedDocstring": { + "text": "Make a new Actor instance with a non-default configuration.\n\nThis is necessary due to the lazy object proxying of the global `Actor` instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 274 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Make a new Actor instance with a non-default configuration.\n\nThis is necessary due to the lazy object proxying of the global `Actor` instance." + } + ] + }, + "flags": {}, + "id": 307, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__call__", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 308, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "231" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "True", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 309, + "kind": 32768, + "kindString": "Parameter", + "name": "configure_logging", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 310, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_process", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 311, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 312, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 313, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "timedelta(seconds=30)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 314, + "kind": 32768, + "kindString": "Parameter", + "name": "cleanup_timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Logger configured for this Actor." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 315, + "module": "_actor", + "name": "log", + "parsedDocstring": { + "text": "Logger configured for this Actor." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 300 + } + ], + "type": { + "name": "logging.Logger", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code the Actor will use when exiting." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 316, + "module": "_actor", + "name": "exit_code", + "parsedDocstring": { + "text": "The exit code the Actor will use when exiting." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 305 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "exit_code" + } + ], + "flags": {}, + "groups": [], + "id": 317, + "module": "_actor", + "name": "exit_code", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 310 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 318, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "exit_code", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 319, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the Actor will display upon termination." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 320, + "module": "_actor", + "name": "status_message", + "parsedDocstring": { + "text": "The final status message that the Actor will display upon termination." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 314 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "status_message" + } + ], + "flags": {}, + "groups": [], + "id": 321, + "module": "_actor", + "name": "status_message", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 319 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 322, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "status_message", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 323, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Asynchronous Apify client for interacting with the Apify API." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 324, + "module": "_actor", + "name": "apify_client", + "parsedDocstring": { + "text": "Asynchronous Apify client for interacting with the Apify API." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 323 + } + ], + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Actor configuration, uses the default instance if not explicitly set." + } + ] + }, + "decorations": [ + { + "name": "cached_property" + } + ], + "flags": {}, + "groups": [], + "id": 325, + "module": "_actor", + "name": "configuration", + "parsedDocstring": { + "text": "Actor configuration, uses the default instance if not explicitly set." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 330 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Actor configuration, uses the default instance if not explicitly set." + } + ] + }, + "flags": {}, + "id": 326, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "configuration", + "parameters": [], + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manages Apify platform events.\n\nIt uses `ApifyEventManager` on the Apify platform and `LocalEventManager` otherwise." + } + ] + }, + "decorations": [ + { + "name": "cached_property" + } + ], + "flags": {}, + "groups": [], + "id": 327, + "module": "_actor", + "name": "event_manager", + "parsedDocstring": { + "text": "Manages Apify platform events.\n\nIt uses `ApifyEventManager` on the Apify platform and `LocalEventManager` otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 351 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manages Apify platform events.\n\nIt uses `ApifyEventManager` on the Apify platform and `LocalEventManager` otherwise." + } + ] + }, + "flags": {}, + "id": 328, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "event_manager", + "parameters": [], + "type": { + "name": "EventManager", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aenter__()`." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 329, + "module": "_actor", + "name": "init", + "parsedDocstring": { + "text": "Initialize the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aenter__()`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 416 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aenter__()`." + } + ] + }, + "flags": {}, + "id": 330, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "init", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aexit__()`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 331, + "module": "_actor", + "name": "exit", + "parsedDocstring": { + "text": "Exit the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aexit__()`.\n", + "args": { + "exit_code": "The exit code the Actor should use when exiting.", + "status_message": "Final status message to display upon Actor termination.", + "event_listeners_timeout": "Maximum time to wait for Actor event listeners to complete before exiting.", + "cleanup_timeout": "Maximum time to wait for cleanup tasks to finish." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 423 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the Actor without using context-manager syntax.\n\nEquivalent to `await Actor.__aexit__()`.\n" + } + ] + }, + "flags": {}, + "id": 332, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "exit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code the Actor should use when exiting." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 333, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Final status message to display upon Actor termination." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 334, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum time to wait for Actor event listeners to complete before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 335, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum time to wait for cleanup tasks to finish." + } + ] + }, + "defaultValue": "timedelta(seconds=30)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 336, + "kind": 32768, + "kindString": "Parameter", + "name": "cleanup_timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fail the Actor instance without using context-manager syntax.\n\nEquivalent to setting the `self.exit_code` and `self.status_message` properties and using\n`await Actor.__aexit__()`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 337, + "module": "_actor", + "name": "fail", + "parsedDocstring": { + "text": "Fail the Actor instance without using context-manager syntax.\n\nEquivalent to setting the `self.exit_code` and `self.status_message` properties and using\n`await Actor.__aexit__()`.\n", + "args": { + "exit_code": "The exit code with which the Actor should fail (defaults to `1`).", + "exception": "The exception with which the Actor failed.", + "status_message": "The final status message that the Actor should display." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 447 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fail the Actor instance without using context-manager syntax.\n\nEquivalent to setting the `self.exit_code` and `self.status_message` properties and using\n`await Actor.__aexit__()`.\n" + } + ] + }, + "flags": {}, + "id": 338, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fail", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exit code with which the Actor should fail (defaults to `1`)." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 339, + "kind": 32768, + "kindString": "Parameter", + "name": "exit_code", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The exception with which the Actor failed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 340, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The final status message that the Actor should display." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 341, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 342, + "module": "_actor", + "name": "new_client", + "parsedDocstring": { + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n", + "args": { + "token": "The Apify API token.", + "api_url": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.", + "max_retries": "How many times to retry a failed request at most.", + "min_delay_between_retries": "How long will the client wait between retrying requests\n(increases exponentially from this value).", + "timeout": "The socket timeout of the HTTP requests sent to the Apify API." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 473 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a new instance of the Apify API client.\n\nThe `ApifyClientAsync` class is provided by the [apify-client](https://github.com/apify/apify-client-python)\npackage, and it is automatically configured using the `APIFY_API_BASE_URL` and `APIFY_TOKEN` environment\nvariables.\n\nYou can override the token via the available options. That's useful if you want to use the client\nas a different Apify user than the SDK internals are using.\n" + } + ] + }, + "flags": {}, + "id": 343, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "new_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 344, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 345, + "kind": 32768, + "kindString": "Parameter", + "name": "api_url", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many times to retry a failed request at most." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 346, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long will the client wait between retrying requests\n(increases exponentially from this value)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 347, + "kind": 32768, + "kindString": "Parameter", + "name": "min_delay_between_retries", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The socket timeout of the HTTP requests sent to the Apify API." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 348, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 349, + "module": "_actor", + "name": "open_dataset", + "parsedDocstring": { + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n", + "args": { + "id": "The ID of the dataset to open. If provided, searches for existing dataset by ID.\nMutually exclusive with name and alias.", + "name": "The name of the dataset to open (global scope, persists across runs).\nMutually exclusive with id and alias.", + "alias": "The alias of the dataset to open (run scope, creates unnamed storage).\nMutually exclusive with id and name.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `Dataset` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 512 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `Dataset` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a dataset.\n\nDatasets are used to store structured data where each object stored has the same attributes, such as online\nstore products or real estate offers. The actual data is stored either on the local filesystem or in\nthe Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 350, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_dataset", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset to open. If provided, searches for existing dataset by ID.\nMutually exclusive with name and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 351, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias of the dataset to open (run scope, creates unnamed storage).\nMutually exclusive with id and name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 352, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset to open (global scope, persists across runs).\nMutually exclusive with id and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 353, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 354, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "Dataset", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 355, + "module": "_actor", + "name": "open_key_value_store", + "parsedDocstring": { + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n", + "args": { + "id": "The ID of the KVS to open. If provided, searches for existing KVS by ID.\nMutually exclusive with name and alias.", + "name": "The name of the KVS to open (global scope, persists across runs).\nMutually exclusive with id and alias.", + "alias": "The alias of the KVS to open (run scope, creates unnamed storage).\nMutually exclusive with id and name.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `KeyValueStore` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 547 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `KeyValueStore` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a key-value store.\n\nKey-value stores are used to store records or files, along with their MIME content type. The records are stored\nand retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.\n" + } + ] + }, + "flags": {}, + "id": 356, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_key_value_store", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the KVS to open. If provided, searches for existing KVS by ID.\nMutually exclusive with name and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 357, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias of the KVS to open (run scope, creates unnamed storage).\nMutually exclusive with id and name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 358, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the KVS to open (global scope, persists across runs).\nMutually exclusive with id and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 359, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 360, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStore", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 361, + "module": "_actor", + "name": "open_request_queue", + "parsedDocstring": { + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n", + "args": { + "id": "The ID of the RQ to open. If provided, searches for existing RQ by ID.\nMutually exclusive with name and alias.", + "name": "The name of the RQ to open (global scope, persists across runs).\nMutually exclusive with id and alias.", + "alias": "The alias of the RQ to open (run scope, creates unnamed storage).\nMutually exclusive with id and name.", + "force_cloud": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + }, + "returns": "An instance of the `RequestQueue` class for the given ID or name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 581 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance of the `RequestQueue` class for the given ID or name." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open a request queue.\n\nRequest queue represents a queue of URLs to crawl, which is stored either on local filesystem or in\nthe Apify cloud. The queue is used for deep crawling of websites, where you start with several URLs and then\nrecursively follow links to other pages. The data structure supports both breadth-first and depth-first\ncrawling orders.\n" + } + ] + }, + "flags": {}, + "id": 362, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_request_queue", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the RQ to open. If provided, searches for existing RQ by ID.\nMutually exclusive with name and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 363, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias of the RQ to open (run scope, creates unnamed storage).\nMutually exclusive with id and name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 364, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the RQ to open (global scope, persists across runs).\nMutually exclusive with id and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 365, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True` then the Apify cloud storage is always used. This way it is possible\nto combine local and cloud storage.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 366, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "RequestQueue", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 367, + "module": "_actor", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n", + "args": { + "data": "The data to push to the default dataset.", + "charged_event_name": "If provided and if the Actor uses the pay-per-event pricing model,\nthe method will attempt to charge for the event for each pushed item." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 617 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or a list of objects to the default dataset of the current Actor run.\n" + } + ] + }, + "flags": {}, + "id": 368, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to push to the default dataset." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 369, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "dict | list[dict]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If provided and if the Actor uses the pay-per-event pricing model,\nthe method will attempt to charge for the event for each pushed item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 370, + "kind": 32768, + "kindString": "Parameter", + "name": "charged_event_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "541" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 371, + "module": "_actor", + "name": "get_input", + "parsedDocstring": { + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 678 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the Actor input value from the default key-value store associated with the current Actor run." + } + ] + }, + "flags": {}, + "id": 372, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_input", + "parameters": [], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the default key-value store associated with the current Actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 373, + "module": "_actor", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the default key-value store associated with the current Actor run.\n", + "args": { + "key": "The key of the record which to retrieve.", + "default_value": "Default value returned in case the record does not exist." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 693 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the default key-value store associated with the current Actor run.\n" + } + ] + }, + "flags": {}, + "id": 374, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 375, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 376, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 377, + "module": "_actor", + "name": "set_value", + "parsedDocstring": { + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n", + "args": { + "key": "The key of the record which to set.", + "value": "The value of the record which to set, or None, if the record should be deleted.", + "content_type": "The content type which should be set to the value." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 704 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set or delete a value in the default key-value store associated with the current Actor run.\n" + } + ] + }, + "flags": {}, + "id": 378, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record which to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 379, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record which to set, or None, if the record should be deleted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 380, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type which should be set to the value." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 381, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the charging manager to access granular pricing information." + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 382, + "module": "_actor", + "name": "get_charging_manager", + "parsedDocstring": { + "text": "Retrieve the charging manager to access granular pricing information." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 722 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve the charging manager to access granular pricing information." + } + ] + }, + "flags": {}, + "id": 383, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_charging_manager", + "parameters": [], + "type": { + "name": "ChargingManager", + "type": "reference", + "target": "513" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 384, + "module": "_actor", + "name": "charge", + "parsedDocstring": { + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n", + "args": { + "event_name": "Name of the event to be charged for.", + "count": "Number of events to charge for." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 727 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "flags": {}, + "id": 385, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the event to be charged for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 386, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of events to charge for." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 387, + "kind": 32768, + "kindString": "Parameter", + "name": "count", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "541" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 388, + "module": "_actor", + "name": "on", + "parsedDocstring": { + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n", + "args": { + "event_name": "The Actor event to listen for.", + "listener": "The function to be called when the event is emitted (can be async)." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 764 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 389, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 390, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Event", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 391, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 473, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 474, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 475, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 476, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 477, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 478, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 479, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 480, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 481, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 482, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 483, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 484, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 485, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 486, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 487, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add an event listener to the Actor's event manager.\n\nThe following events can be emitted:\n\n- `Event.SYSTEM_INFO`: Emitted every minute; the event data contains information about the Actor's resource\nusage.\n\n- `Event.MIGRATING`: Emitted when the Actor on the Apify platform is about to be migrated to another worker\nserver. Use this event to persist the Actor's state and gracefully stop in-progress tasks, preventing\ndisruption.\n\n- `Event.PERSIST_STATE`: Emitted regularly (default: 60 seconds) to notify the Actor to persist its state,\npreventing work repetition after a restart. This event is emitted together with the `MIGRATING` event, where\nthe `isMigrating` flag in the event data is `True`; otherwise, the flag is `False`. This event is for\nconvenience; the same effect can be achieved by setting an interval and listening for the `MIGRATING` event.\n\n- `Event.ABORTING`: Emitted when a user aborts an Actor run on the Apify platform, allowing the Actor time\nto clean up its state if the abort is graceful.\n" + } + ] + }, + "flags": {}, + "id": 488, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event to listen for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 489, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Event", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function to be called when the event is emitted (can be async)." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 490, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 392, + "module": "_actor", + "name": "off", + "parsedDocstring": { + "text": "Remove a listener, or all listeners, from an Actor event.\n", + "args": { + "event_name": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 805 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 393, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 394, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Event", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 395, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "Callable | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 491, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 492, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 493, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 494, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 495, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 496, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 497, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 498, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 499, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 500, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 501, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 502, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 503, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 504, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 505, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a listener, or all listeners, from an Actor event.\n" + } + ] + }, + "flags": {}, + "id": 506, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 507, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "Event", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 508, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 396, + "module": "_actor", + "name": "is_at_home", + "parsedDocstring": { + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 815 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)." + } + ] + }, + "flags": {}, + "id": 397, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_at_home", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 398, + "module": "_actor", + "name": "get_env", + "parsedDocstring": { + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 820 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.\n\nFor a list of all the environment variables, see the\n[Actor documentation](https://docs.apify.com/actors/development/environment-variables). If some variables\nare not defined or are invalid, the corresponding value in the resulting dictionary will be None." + } + ] + }, + "flags": {}, + "id": 399, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_env", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 400, + "module": "_actor", + "name": "start", + "parsedDocstring": { + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n", + "args": { + "actor_id": "The ID of the Actor to be run.", + "run_input": "The input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` or `RemainingTime` will set timeout of the\nother Actor to the time remaining from this Actor timeout.", + "wait_for_finish": "The maximum number of seconds the server waits for the run to finish. By default,\nit is 0, the maximum value is 300.", + "webhooks": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\n" + }, + "returns": "Info about the started Actor run" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 848 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run" + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Run an Actor on the Apify platform.\n\nUnlike `Actor.call`, this method just starts the run without waiting for finish.\n" + } + ] + }, + "flags": {}, + "id": 401, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 402, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 403, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 404, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 405, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 406, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 407, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` or `RemainingTime` will set timeout of the\nother Actor to the time remaining from this Actor timeout." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 408, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None | Literal['inherit', 'RemainingTime']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "inherit" + }, + { + "type": "literal", + "value": "RemainingTime" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. By default,\nit is 0, the maximum value is 300." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 409, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_finish", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with\nthe Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.\nIf you already have a webhook set up for the Actor or task, you do not have to add it again here.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 410, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "80" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ActorRun", + "type": "reference", + "target": "196" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 411, + "module": "_actor", + "name": "abort", + "parsedDocstring": { + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n", + "args": { + "run_id": "The ID of the Actor run to be aborted.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "status_message": "Status message of the Actor to be set on the platform.", + "gracefully": "If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\nto resurrect the run later.\n" + }, + "returns": "Info about the aborted Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 925 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the aborted Actor run." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Abort given Actor run on the Apify platform using the current user account.\n\nThe user account is determined by the `APIFY_TOKEN` environment variable.\n" + } + ] + }, + "flags": {}, + "id": 412, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor run to be aborted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 413, + "kind": 32768, + "kindString": "Parameter", + "name": "run_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 414, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Status message of the Actor to be set on the platform." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 415, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, the Actor run will abort gracefully. It will send `aborting` and `persistState`\nevents into the run and force-stop the run after 30 seconds. It is helpful in cases where you plan\nto resurrect the run later.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 416, + "kind": 32768, + "kindString": "Parameter", + "name": "gracefully", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ActorRun", + "type": "reference", + "target": "196" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 417, + "module": "_actor", + "name": "call", + "parsedDocstring": { + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n", + "args": { + "actor_id": "The ID of the Actor to be run.", + "run_input": "The input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` or `RemainingTime` will set timeout of the\nother Actor to the time remaining from this Actor timeout.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here.", + "wait": "The maximum number of seconds the server waits for the run to finish. If not provided,\nwaits indefinitely.", + "logger": "Logger used to redirect logs from the Actor run. Using \"default\" literal means that a predefined\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\nwill redirect logs to the provided logger.\n" + }, + "returns": "Info about the started Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 958 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Start an Actor on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n" + } + ] + }, + "flags": {}, + "id": 418, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 419, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 420, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 421, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 422, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 423, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 424, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` or `RemainingTime` will set timeout of the\nother Actor to the time remaining from this Actor timeout." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 425, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None | Literal['inherit', 'RemainingTime']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "inherit" + }, + { + "type": "literal", + "value": "RemainingTime" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 426, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "80" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided,\nwaits indefinitely." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 427, + "kind": 32768, + "kindString": "Parameter", + "name": "wait", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Logger used to redirect logs from the Actor run. Using \"default\" literal means that a predefined\ndefault logger will be used. Setting `None` will disable any log propagation. Passing custom logger\nwill redirect logs to the provided logger.\n" + } + ] + }, + "defaultValue": "'default'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 428, + "kind": 32768, + "kindString": "Parameter", + "name": "logger", + "type": { + "name": "logging.Logger | None | Literal['default']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "logging.Logger" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "default" + } + ] + } + ] + } + } + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "196" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 429, + "module": "_actor", + "name": "call_task", + "parsedDocstring": { + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n", + "args": { + "task_id": "The ID of the Actor to be run.", + "task_input": "Overrides the input to pass to the Actor run.", + "token": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable).", + "content_type": "The content type of the input.", + "build": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest).", + "memory_mbytes": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor.", + "timeout": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` will set timeout of the other Actor to the\ntime remaining from this Actor timeout.", + "webhooks": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here.", + "wait": "The maximum number of seconds the server waits for the run to finish. If not provided, waits\nindefinitely.\n" + }, + "returns": "Info about the started Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1041 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Info about the started Actor run." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Start an Actor task on the Apify Platform and wait for it to finish before returning.\n\nIt waits indefinitely, unless the wait argument is provided.\n\nNote that an Actor task is a saved input configuration and options for an Actor. If you want to run an Actor\ndirectly rather than an Actor task, please use the `Actor.call`\n" + } + ] + }, + "flags": {}, + "id": 430, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "call_task", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the Actor to be run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 431, + "kind": 32768, + "kindString": "Parameter", + "name": "task_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Overrides the input to pass to the Actor run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 432, + "kind": 32768, + "kindString": "Parameter", + "name": "task_input", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the Actor build to run. It can be either a build tag or build number. By default,\nthe run uses the build specified in the default run configuration for the Actor (typically latest)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 433, + "kind": 32768, + "kindString": "Parameter", + "name": "build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory limit for the run, in megabytes. By default, the run uses a memory limit specified\nin the default run configuration for the Actor." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 434, + "kind": 32768, + "kindString": "Parameter", + "name": "memory_mbytes", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for the run, in seconds. By default, the run uses timeout specified in\nthe default run configuration for the Actor. Using `inherit` will set timeout of the other Actor to the\ntime remaining from this Actor timeout." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 435, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None | Literal['inherit']", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "reference", + "name": "Literal", + "typeArguments": [ + { + "type": "literal", + "value": "inherit" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional webhooks (https://docs.apify.com/webhooks) associated with the Actor run, which can\nbe used to receive a notification, e.g. when the Actor finished or failed. If you already have\na webhook set up for the Actor, you do not have to add it again here." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 436, + "kind": 32768, + "kindString": "Parameter", + "name": "webhooks", + "type": { + "name": "list[Webhook] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Webhook", + "target": "80" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds the server waits for the run to finish. If not provided, waits\nindefinitely.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 437, + "kind": 32768, + "kindString": "Parameter", + "name": "wait", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API token to use for this request (defaults to the `APIFY_TOKEN` environment variable)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 438, + "kind": 32768, + "kindString": "Parameter", + "name": "token", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "196" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 439, + "module": "_actor", + "name": "metamorph", + "parsedDocstring": { + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n", + "args": { + "target_actor_id": "ID of the target Actor that the run should be transformed into", + "run_input": "The input to pass to the new run.", + "target_actor_build": "The build of the target Actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target Actor\n(typically the latest build).", + "content_type": "The content type of the input.", + "custom_after_sleep": "How long to sleep for after the metamorph, to wait for the container to be stopped." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1111 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Transform this Actor run to an Actor run of a different Actor.\n\nThe platform stops the current Actor container and starts a new container with the new Actor instead. All\nthe default storages are preserved, and the new input is stored under the `INPUT-METAMORPH-1` key in the same\ndefault key-value store.\n" + } + ] + }, + "flags": {}, + "id": 440, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "metamorph", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the target Actor that the run should be transformed into" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 441, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_id", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The input to pass to the new run." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 442, + "kind": 32768, + "kindString": "Parameter", + "name": "run_input", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The build of the target Actor. It can be either a build tag or build number.\nBy default, the run uses the build specified in the default run configuration for the target Actor\n(typically the latest build)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 443, + "kind": 32768, + "kindString": "Parameter", + "name": "target_actor_build", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The content type of the input." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 444, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the metamorph, to wait for the container to be stopped." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 445, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 446, + "module": "_actor", + "name": "reboot", + "parsedDocstring": { + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n", + "args": { + "event_listeners_timeout": "How long should the Actor wait for Actor event listeners to finish before exiting.", + "custom_after_sleep": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1157 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internally reboot this Actor.\n\nThe system stops the current container and starts a new one, with the same run ID and default storages.\n" + } + ] + }, + "flags": {}, + "id": 447, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reboot", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long should the Actor wait for Actor event listeners to finish before exiting." + } + ] + }, + "defaultValue": "EVENT_LISTENERS_TIMEOUT", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 448, + "kind": 32768, + "kindString": "Parameter", + "name": "event_listeners_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "How long to sleep for after the reboot, to wait for the container to be stopped." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 449, + "kind": 32768, + "kindString": "Parameter", + "name": "custom_after_sleep", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 450, + "module": "_actor", + "name": "add_webhook", + "parsedDocstring": { + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n", + "args": { + "webhook": "The webhook to be added", + "ignore_ssl_errors": "Whether the webhook should ignore SSL errors returned by request_url", + "do_not_retry": "Whether the webhook should retry sending the payload to request_url upon failure.", + "idempotency_key": "A unique identifier of a webhook. You can use it to ensure that you won't create\nthe same webhook multiple times.\n" + }, + "returns": "The created webhook." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1211 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The created webhook." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create an ad-hoc webhook for the current Actor run.\n\nThis webhook lets you receive a notification when the Actor run finished or failed.\n\nNote that webhooks are only supported for Actors running on the Apify platform. When running the Actor locally,\nthe function will print a warning and have no effect.\n\nFor more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).\n" + } + ] + }, + "flags": {}, + "id": 451, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_webhook", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The webhook to be added" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 452, + "kind": 32768, + "kindString": "Parameter", + "name": "webhook", + "type": { + "name": "Webhook", + "type": "reference", + "target": "80" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should ignore SSL errors returned by request_url" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 453, + "kind": 32768, + "kindString": "Parameter", + "name": "ignore_ssl_errors", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the webhook should retry sending the payload to request_url upon failure." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 454, + "kind": 32768, + "kindString": "Parameter", + "name": "do_not_retry", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique identifier of a webhook. You can use it to ensure that you won't create\nthe same webhook multiple times.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 455, + "kind": 32768, + "kindString": "Parameter", + "name": "idempotency_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set the status message for the current Actor run.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 456, + "module": "_actor", + "name": "set_status_message", + "parsedDocstring": { + "text": "Set the status message for the current Actor run.\n", + "args": { + "status_message": "The status message to set to the run.", + "is_terminal": "Set this flag to True if this is the final status message of the Actor run.\n" + }, + "returns": "The updated Actor run object." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1257 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The updated Actor run object." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Set the status message for the current Actor run.\n" + } + ] + }, + "flags": {}, + "id": 457, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_status_message", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The status message to set to the run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 458, + "kind": 32768, + "kindString": "Parameter", + "name": "status_message", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set this flag to True if this is the final status message of the Actor run.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 459, + "kind": 32768, + "kindString": "Parameter", + "name": "is_terminal", + "type": { + "name": "bool | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ActorRun | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ActorRun", + "target": "196" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 460, + "module": "_actor", + "name": "create_proxy_configuration", + "parsedDocstring": { + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n", + "args": { + "actor_proxy_input": "Proxy configuration field from the Actor input, if input has such input field. If you\npass this argument, all the other arguments will be inferred from it.", + "password": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available.", + "groups": "Proxy groups which the Apify Proxy should use, if provided.", + "country_code": "Country which the Apify Proxy should use, if provided.", + "proxy_urls": "Custom proxy server URLs which should be rotated through.", + "new_url_function": "Function which returns a custom proxy URL to be used.\n" + }, + "returns": "ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\non the configuration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1288 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based\non the configuration." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a ProxyConfiguration object with the passed proxy configuration.\n\nConfigures connection to a proxy server with the provided options. Proxy servers are used to prevent target\nwebsites from blocking your crawlers based on IP address rate limits or blacklists.\n\nFor more details and code examples, see the `ProxyConfiguration` class.\n" + } + ] + }, + "flags": {}, + "id": 461, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_proxy_configuration", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration field from the Actor input, if input has such input field. If you\npass this argument, all the other arguments will be inferred from it." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 462, + "kind": 32768, + "kindString": "Parameter", + "name": "actor_proxy_input", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Password for the Apify Proxy. If not provided, will use os.environ['APIFY_PROXY_PASSWORD'],\nif available." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 463, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy groups which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 464, + "kind": 32768, + "kindString": "Parameter", + "name": "groups", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Country which the Apify Proxy should use, if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 465, + "kind": 32768, + "kindString": "Parameter", + "name": "country_code", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Custom proxy server URLs which should be rotated through." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 466, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_urls", + "type": { + "name": "list[str | None] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Function which returns a custom proxy URL to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 467, + "kind": 32768, + "kindString": "Parameter", + "name": "new_url_function", + "type": { + "name": "_NewUrlFunction | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "_NewUrlFunction" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ProxyConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyConfiguration", + "target": "43" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Easily create and manage state values. All state values are automatically persisted.\n\nValues can be modified by simply using the assignment operator.\n" + } + ] + }, + "decorations": [ + { + "name": "_ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 468, + "module": "_actor", + "name": "use_state", + "parsedDocstring": { + "text": "Easily create and manage state values. All state values are automatically persisted.\n\nValues can be modified by simply using the assignment operator.\n", + "args": { + "default_value": "The default value to initialize the state if it is not already set.", + "key": "The key in the key-value store where the state is stored. If not provided, a default key is used.", + "kvs_name": "The name of the key-value store where the state is stored. If not provided, the default\nkey-value store associated with the Actor run is used.\n" + }, + "returns": "The state dictionary with automatic persistence." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1344 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The state dictionary with automatic persistence." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Easily create and manage state values. All state values are automatically persisted.\n\nValues can be modified by simply using the assignment operator.\n" + } + ] + }, + "flags": {}, + "id": 469, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "use_state", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The default value to initialize the state if it is not already set." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 470, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "dict[str, JsonSerializable] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "JsonSerializable" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key in the key-value store where the state is stored. If not provided, a default key is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 471, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the key-value store where the state is stored. If not provided, the default\nkey-value store associated with the Actor run is used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 472, + "kind": 32768, + "kindString": "Parameter", + "name": "kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "JsonSerializable" + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The core class for building Actors on the Apify platform.\n\nActors are serverless programs running in the cloud that can perform anything from simple actions\n(such as filling out a web form or sending an email) to complex operations (such as crawling an\nentire website or removing duplicates from a large dataset). They are packaged as Docker containers\nwhich accept well-defined JSON input, perform an action, and optionally produce well-defined output.\n\n### References\n\n- Apify platform documentation: https://docs.apify.com/platform/actors\n- Actor whitepaper: https://whitepaper.actor/\n\n### Usage\n\n```python\nimport asyncio\n\nimport httpx\nfrom apify import Actor\nfrom bs4 import BeautifulSoup\n\n\nasync def main() -> None:\n async with Actor:\n actor_input = await Actor.get_input()\n async with httpx.AsyncClient() as client:\n response = await client.get(actor_input['url'])\n soup = BeautifulSoup(response.content, 'html.parser')\n data = {\n 'url': actor_input['url'],\n 'title': soup.title.string if soup.title else None,\n }\n await Actor.push_data(data)\n\nif __name__ == '__main__':\n asyncio.run(main())\n```" + } + ] + }, + "decorations": [ + { + "args": "('Actor')", + "name": "docs_name" + }, + { + "args": "('Actor')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 297, + 299, + 306, + 288, + 304, + 411, + 450, + 417, + 429, + 384, + 325, + 460, + 327, + 331, + 317, + 337, + 382, + 398, + 371, + 373, + 329, + 396, + 439, + 342, + 392, + 388, + 349, + 355, + 361, + 367, + 446, + 456, + 377, + 400, + 321, + 468 + ], + "title": "Methods" + }, + { + "children": [ + 324, + 316, + 315, + 320 + ], + "title": "Properties" + } + ], + "id": 287, + "module": "_actor", + "name": "Actor", + "parsedDocstring": { + "text": "The core class for building Actors on the Apify platform.\n\nActors are serverless programs running in the cloud that can perform anything from simple actions\n(such as filling out a web form or sending an email) to complex operations (such as crawling an\nentire website or removing duplicates from a large dataset). They are packaged as Docker containers\nwhich accept well-defined JSON input, perform an action, and optionally produce well-defined output.\n\n### References\n\n- Apify platform documentation: https://docs.apify.com/platform/actors\n- Actor whitepaper: https://whitepaper.actor/\n\n### Usage\n\n```python\nimport asyncio\n\nimport httpx\nfrom apify import Actor\nfrom bs4 import BeautifulSoup\n\n\nasync def main() -> None:\n async with Actor:\n actor_input = await Actor.get_input()\n async with httpx.AsyncClient() as client:\n response = await client.get(actor_input['url'])\n soup = BeautifulSoup(response.content, 'html.parser')\n data = {\n 'url': actor_input['url'],\n 'title': soup.title.string if soup.title else None,\n }\n await Actor.push_data(data)\n\nif __name__ == '__main__':\n asyncio.run(main())\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The entry point of the SDK, through which all the Actor operations should be done." + } + ] + }, + "flags": {}, + "groups": [], + "id": 509, + "module": "_actor", + "name": "Actor", + "parsedDocstring": { + "text": "The entry point of the SDK, through which all the Actor operations should be done." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_actor.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 1400 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 510, + "module": "_charging", + "name": "run_validator", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 511, + "module": "_charging", + "name": "DEFAULT_DATASET_ITEM_EVENT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 512, + "module": "_charging", + "name": "charging_manager_ctx", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "type": { + "name": "ContextVar", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "ChargingManager", + "target": "513" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Lock to synchronize charge operations. Prevents race conditions between `charge` and `push_data` calls." + } + ] + }, + "flags": {}, + "groups": [], + "id": 514, + "module": "_charging", + "name": "charge_lock", + "parsedDocstring": { + "text": "Lock to synchronize charge operations. Prevents race conditions between `charge` and `push_data` calls." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "type": { + "name": "ReentrantLock", + "type": "reference", + "target": "617" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 515, + "module": "_charging", + "name": "charge", + "parsedDocstring": { + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n", + "args": { + "event_name": "Name of the event to be charged for.", + "count": "Number of events to charge for." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 58 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "flags": {}, + "id": 516, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the event to be charged for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 517, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of events to charge for." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 518, + "kind": 32768, + "kindString": "Parameter", + "name": "count", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "541" + }, + "overwrites": { + "name": "ChargingManager.charge", + "target": 515, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 519, + "module": "_charging", + "name": "calculate_total_charged_amount", + "parsedDocstring": { + "text": "Calculate the total amount of money charged for pay-per-event events so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "flags": {}, + "id": 520, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_total_charged_amount", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.calculate_total_charged_amount", + "target": 519, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 521, + "module": "_charging", + "name": "calculate_max_event_charge_count_within_limit", + "parsedDocstring": { + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n", + "args": { + "event_name": "Name of the inspected event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "flags": {}, + "id": 522, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_max_event_charge_count_within_limit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the inspected event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 523, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + }, + "overwrites": { + "name": "ChargingManager.calculate_max_event_charge_count_within_limit", + "target": 521, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 524, + "module": "_charging", + "name": "get_pricing_info", + "parsedDocstring": { + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "flags": {}, + "id": 525, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_pricing_info", + "parameters": [], + "type": { + "name": "ActorPricingInfo", + "type": "reference", + "target": "545" + }, + "overwrites": { + "name": "ChargingManager.get_pricing_info", + "target": 524, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of events with the given name that were charged so far.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 526, + "module": "_charging", + "name": "get_charged_event_count", + "parsedDocstring": { + "text": "Get the number of events with the given name that were charged so far.\n", + "args": { + "event_name": "Name of the inspected event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 84 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of events with the given name that were charged so far.\n" + } + ] + }, + "flags": {}, + "id": 527, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_charged_event_count", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the inspected event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 528, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.get_charged_event_count", + "target": 526, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the configured maximum total charge for this Actor run." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 529, + "module": "_charging", + "name": "get_max_total_charge_usd", + "parsedDocstring": { + "text": "Get the configured maximum total charge for this Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the configured maximum total charge for this Actor run." + } + ] + }, + "flags": {}, + "id": 530, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_max_total_charge_usd", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.get_max_total_charge_usd", + "target": 529, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 531, + "module": "_charging", + "name": "compute_push_data_limit", + "parsedDocstring": { + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n", + "args": { + "items_count": "The number of items to be pushed.", + "event_name": "The explicit event name to charge for each item.", + "is_default_dataset": "Whether the data is pushed to the default dataset.\nIf True, the synthetic event cost is included in the combined price.\n" + }, + "returns": "Max number of items that can be pushed within the budget." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 94 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Max number of items that can be pushed within the budget." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n" + } + ] + }, + "flags": {}, + "id": 532, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_push_data_limit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items to be pushed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 533, + "kind": 32768, + "kindString": "Parameter", + "name": "items_count", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The explicit event name to charge for each item." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 534, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the data is pushed to the default dataset.\nIf True, the synthetic event cost is included in the combined price.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 535, + "kind": 32768, + "kindString": "Parameter", + "name": "is_default_dataset", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.compute_push_data_limit", + "target": 531, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 536, + "module": "_charging", + "name": "is_event_charge_limit_reached", + "parsedDocstring": { + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 116 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + } + ] + }, + "flags": {}, + "id": 537, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_event_charge_limit_reached", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 538, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.is_event_charge_limit_reached", + "target": 536, + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 539, + "module": "_charging", + "name": "compute_chargeable", + "parsedDocstring": { + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 119 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + } + ] + }, + "flags": {}, + "id": 540, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_chargeable", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + }, + "overwrites": { + "name": "ChargingManager.compute_chargeable", + "target": 539, + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Provides fine-grained access to pay-per-event functionality.\n\nThe ChargingManager allows you to charge for specific events in your Actor when using\nthe pay-per-event pricing model. This enables precise cost control and transparent\nbilling for different operations within your Actor.\n\n### References\n\n- Apify platform documentation: https://docs.apify.com/platform/actors/publishing/monetize" + } + ] + }, + "decorations": [ + { + "args": "('Charging')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 521, + 519, + 515, + 539, + 531, + 526, + 529, + 524, + 536 + ], + "title": "Methods" + }, + { + "children": [ + 514 + ], + "title": "Properties" + } + ], + "id": 513, + "module": "_charging", + "name": "ChargingManager", + "parsedDocstring": { + "text": "Provides fine-grained access to pay-per-event functionality.\n\nThe ChargingManager allows you to charge for specific events in your Actor when using\nthe pay-per-event pricing model. This enables precise cost control and transparent\nbilling for different operations within your Actor.\n\n### References\n\n- Apify platform documentation: https://docs.apify.com/platform/actors/publishing/monetize" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedBy": [ + { + "name": "ChargingManagerImplementation", + "target": "550", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, no more events of this type can be charged within the limit." + } + ] + }, + "flags": {}, + "groups": [], + "id": 542, + "module": "_charging", + "name": "event_charge_limit_reached", + "parsedDocstring": { + "text": "If true, no more events of this type can be charged within the limit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 128 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Total amount of charged events - may be lower than the requested amount." + } + ] + }, + "flags": {}, + "groups": [], + "id": 543, + "module": "_charging", + "name": "charged_count", + "parsedDocstring": { + "text": "Total amount of charged events - may be lower than the requested amount." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 131 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "How many events of each known type can still be charged within the limit." + } + ] + }, + "flags": {}, + "groups": [], + "id": 544, + "module": "_charging", + "name": "chargeable_within_limit", + "parsedDocstring": { + "text": "How many events of each known type can still be charged within the limit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 134 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Result of the `ChargingManager.charge` method." + } + ] + }, + "decorations": [ + { + "args": "('Charging')", + "name": "docs_group" + }, + { + "args": "(frozen=True)", + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 544, + 543, + 542 + ], + "title": "Properties" + } + ], + "id": 541, + "module": "_charging", + "name": "ChargeResult", + "parsedDocstring": { + "text": "Result of the `ChargingManager.charge` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 125 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The currently effective pricing model." + } + ] + }, + "flags": {}, + "groups": [], + "id": 546, + "module": "_charging", + "name": "pricing_model", + "parsedDocstring": { + "text": "The currently effective pricing model." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 143 + } + ], + "type": { + "name": "PricingModel | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "PricingModel", + "target": "65" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A configured limit for the total charged amount - if you exceed it, you won't receive more money than this." + } + ] + }, + "flags": {}, + "groups": [], + "id": 547, + "module": "_charging", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "A configured limit for the total charged amount - if you exceed it, you won't receive more money than this." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 146 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A shortcut - true if the Actor runs with the pay-per-event pricing model." + } + ] + }, + "flags": {}, + "groups": [], + "id": 548, + "module": "_charging", + "name": "is_pay_per_event", + "parsedDocstring": { + "text": "A shortcut - true if the Actor runs with the pay-per-event pricing model." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 149 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Price of every known event type." + } + ] + }, + "flags": {}, + "groups": [], + "id": 549, + "module": "_charging", + "name": "per_event_prices", + "parsedDocstring": { + "text": "Price of every known event type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 152 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Decimal" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Result of the `ChargingManager.get_pricing_info` method." + } + ] + }, + "decorations": [ + { + "args": "('Charging')", + "name": "docs_group" + }, + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 548, + 547, + 549, + 546 + ], + "title": "Properties" + } + ], + "id": 545, + "module": "_charging", + "name": "ActorPricingInfo", + "parsedDocstring": { + "text": "Result of the `ChargingManager.get_pricing_info` method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 551, + "module": "_charging", + "name": "LOCAL_CHARGING_LOG_DATASET_NAME", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 159 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 552, + "module": "_charging", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 161 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 553, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 554, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 555, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "ApifyClientAsync", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 556, + "module": "_charging", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 180 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the charging manager - this is called by the `Actor` class and shouldn't be invoked manually." + } + ] + }, + "flags": {}, + "id": 557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 558, + "module": "_charging", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 238 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 559, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 560, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 561, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 562, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 563, + "module": "_charging", + "name": "charge", + "parsedDocstring": { + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n", + "args": { + "event_name": "Name of the event to be charged for.", + "count": "Number of events to charge for." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 251 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Charge for a specified number of events - sub-operations of the Actor.\n\nThis is relevant only for the pay-per-event pricing model.\n" + } + ] + }, + "flags": {}, + "id": 516, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "charge", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the event to be charged for." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 517, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of events to charge for." + } + ] + }, + "defaultValue": "1", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 518, + "kind": 32768, + "kindString": "Parameter", + "name": "count", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "ChargeResult", + "type": "reference", + "target": "541" + }, + "overwrites": { + "name": "ChargingManager.charge", + "target": 515, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.charge", + "target": 515, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 567, + "module": "_charging", + "name": "calculate_total_charged_amount", + "parsedDocstring": { + "text": "Calculate the total amount of money charged for pay-per-event events so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 340 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate the total amount of money charged for pay-per-event events so far." + } + ] + }, + "flags": {}, + "id": 520, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_total_charged_amount", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.calculate_total_charged_amount", + "target": 519, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.calculate_total_charged_amount", + "target": 519, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 569, + "module": "_charging", + "name": "calculate_max_event_charge_count_within_limit", + "parsedDocstring": { + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n", + "args": { + "event_name": "Name of the inspected event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 347 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Calculate how many instances of an event can be charged before we reach the configured limit.\n" + } + ] + }, + "flags": {}, + "id": 522, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "calculate_max_event_charge_count_within_limit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the inspected event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 523, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + }, + "overwrites": { + "name": "ChargingManager.calculate_max_event_charge_count_within_limit", + "target": 521, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.calculate_max_event_charge_count_within_limit", + "target": 521, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 572, + "module": "_charging", + "name": "get_pricing_info", + "parsedDocstring": { + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 357 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve detailed information about the effective pricing of the current Actor run.\n\nThis can be used for instance when your code needs to support multiple pricing models in transition periods." + } + ] + }, + "flags": {}, + "id": 525, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_pricing_info", + "parameters": [], + "type": { + "name": "ActorPricingInfo", + "type": "reference", + "target": "545" + }, + "overwrites": { + "name": "ChargingManager.get_pricing_info", + "target": 524, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.get_pricing_info", + "target": 524, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of events with the given name that were charged so far.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 574, + "module": "_charging", + "name": "get_charged_event_count", + "parsedDocstring": { + "text": "Get the number of events with the given name that were charged so far.\n", + "args": { + "event_name": "Name of the inspected event." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 370 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of events with the given name that were charged so far.\n" + } + ] + }, + "flags": {}, + "id": 527, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_charged_event_count", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the inspected event." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 528, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.get_charged_event_count", + "target": 526, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.get_charged_event_count", + "target": 526, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the configured maximum total charge for this Actor run." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 577, + "module": "_charging", + "name": "get_max_total_charge_usd", + "parsedDocstring": { + "text": "Get the configured maximum total charge for this Actor run." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 375 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the configured maximum total charge for this Actor run." + } + ] + }, + "flags": {}, + "id": 530, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_max_total_charge_usd", + "parameters": [], + "type": { + "name": "Decimal", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.get_max_total_charge_usd", + "target": 529, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.get_max_total_charge_usd", + "target": 529, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 579, + "module": "_charging", + "name": "compute_push_data_limit", + "parsedDocstring": { + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n", + "args": { + "items_count": "The number of items to be pushed.", + "event_name": "The explicit event name to charge for each item.", + "is_default_dataset": "Whether the data is pushed to the default dataset.\nIf True, the synthetic event cost is included in the combined price.\n" + }, + "returns": "Max number of items that can be pushed within the budget." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 379 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Max number of items that can be pushed within the budget." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Compute how many items can be pushed and charged within the current budget.\n\nAccounts for both the explicit event and the synthetic `DEFAULT_DATASET_ITEM_EVENT` event,\nso that the combined cost per item does not exceed the remaining budget.\n" + } + ] + }, + "flags": {}, + "id": 532, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_push_data_limit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items to be pushed." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 533, + "kind": 32768, + "kindString": "Parameter", + "name": "items_count", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The explicit event name to charge for each item." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 534, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the data is pushed to the default dataset.\nIf True, the synthetic event cost is included in the combined price.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 535, + "kind": 32768, + "kindString": "Parameter", + "name": "is_default_dataset", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.compute_push_data_limit", + "target": 531, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.compute_push_data_limit", + "target": 531, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 584, + "module": "_charging", + "name": "is_event_charge_limit_reached", + "parsedDocstring": { + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 398 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if the remaining budget is insufficient to charge even a single event of the given type." + } + ] + }, + "flags": {}, + "id": 537, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_event_charge_limit_reached", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 538, + "kind": 32768, + "kindString": "Parameter", + "name": "event_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + }, + "overwrites": { + "name": "ChargingManager.is_event_charge_limit_reached", + "target": 536, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.is_event_charge_limit_reached", + "target": 536, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 587, + "module": "_charging", + "name": "compute_chargeable", + "parsedDocstring": { + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 403 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Compute the maximum number of events of each type that can be charged within the current budget." + } + ] + }, + "flags": {}, + "id": 540, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "compute_chargeable", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + ] + }, + "overwrites": { + "name": "ChargingManager.compute_chargeable", + "target": 539, + "type": "reference" + } + } + ], + "overwrites": { + "name": "ChargingManager.compute_chargeable", + "target": 539, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Lock to synchronize charge operations. Prevents race conditions between `charge` and `push_data` calls." + } + ] + }, + "flags": {}, + "groups": [], + "id": 1166, + "module": "_charging", + "name": "charge_lock", + "parsedDocstring": { + "text": "Lock to synchronize charge operations. Prevents race conditions between `charge` and `push_data` calls." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "type": { + "name": "ReentrantLock", + "type": "reference", + "target": "617" + }, + "inheritedFrom": { + "name": "ChargingManager.charge_lock", + "target": 514, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Implementation of the `ChargingManager` Protocol - this is only meant to be instantiated internally." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 556, + 558, + 552, + 569, + 567, + 563, + 587, + 579, + 574, + 577, + 572, + 584 + ], + "title": "Methods" + }, + { + "children": [ + 1166, + 551 + ], + "title": "Properties" + } + ], + "id": 550, + "module": "_charging", + "name": "ChargingManagerImplementation", + "parsedDocstring": { + "text": "Implementation of the `ChargingManager` Protocol - this is only meant to be instantiated internally." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 156 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "ChargingManager", + "target": "513", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 590, + "module": "_charging", + "name": "charge_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 451 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 591, + "module": "_charging", + "name": "total_charged_amount", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 452 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 590, + 591 + ], + "title": "Properties" + } + ], + "id": 589, + "module": "_charging", + "name": "ChargingStateItem", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 450 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 593, + "module": "_charging", + "name": "price", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 457 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 594, + "module": "_charging", + "name": "title", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 458 + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "dataclass" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 593, + 594 + ], + "title": "Properties" + } + ], + "id": 592, + "module": "_charging", + "name": "PricingInfoItem", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 456 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 596, + "module": "_charging", + "name": "pricing_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 462 + } + ], + "type": { + "name": "( FreeActorPricingInfo | FlatPricePerMonthActorPricingInfo | PricePerDatasetItemActorPricingInfo | PayPerEventActorPricingInfo | None )", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "FreeActorPricingInfo", + "target": "174" + }, + { + "type": "reference", + "name": "FlatPricePerMonthActorPricingInfo", + "target": "176" + } + ] + }, + { + "type": "reference", + "name": "PricePerDatasetItemActorPricingInfo", + "target": "180" + } + ] + }, + { + "type": "reference", + "name": "PayPerEventActorPricingInfo", + "target": "192" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 597, + "module": "_charging", + "name": "charged_event_counts", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 469 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "int" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 598, + "module": "_charging", + "name": "max_total_charge_usd", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 470 + } + ], + "type": { + "name": "Decimal", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 597, + 598, + 596 + ], + "title": "Properties" + } + ], + "id": 595, + "module": "_charging", + "name": "_FetchedPricingInfoDict", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_charging.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 461 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 599, + "module": "_utils", + "name": "T", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a decorator that ensures the context manager is initialized before executing the method.\n\nThe decorator checks if the calling instance has the specified attribute and verifies that it is set to `True`.\nIf the instance is inactive, it raises a `RuntimeError`. Works for both synchronous and asynchronous methods.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 600, + "module": "_utils", + "name": "ensure_context", + "parsedDocstring": { + "text": "Create a decorator that ensures the context manager is initialized before executing the method.\n\nThe decorator checks if the calling instance has the specified attribute and verifies that it is set to `True`.\nIf the instance is inactive, it raises a `RuntimeError`. Works for both synchronous and asynchronous methods.\n", + "args": { + "attribute_name": "The name of the boolean attribute to check on the instance.\n" + }, + "returns": "A decorator that wraps methods with context checking." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A decorator that wraps methods with context checking." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a decorator that ensures the context manager is initialized before executing the method.\n\nThe decorator checks if the calling instance has the specified attribute and verifies that it is set to `True`.\nIf the instance is inactive, it raises a `RuntimeError`. Works for both synchronous and asynchronous methods.\n" + } + ] + }, + "flags": {}, + "id": 601, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "ensure_context", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the boolean attribute to check on the instance.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 602, + "kind": 32768, + "kindString": "Parameter", + "name": "attribute_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "[T]" + }, + { + "type": "reference", + "name": "T", + "target": "599" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 603, + "module": "_utils", + "name": "get_system_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 604, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_system_info", + "parameters": [], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 605, + "module": "_utils", + "name": "is_running_in_ipython", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 606, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "is_running_in_ipython", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 607, + "module": "_utils", + "name": "GroupName", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 608, + "module": "_utils", + "name": "docs_group", + "parsedDocstring": { + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n", + "args": { + "group_name": "The documentation group to which the symbol belongs.\n" + }, + "returns": "The original callable without modification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 88 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The original callable without modification." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a symbol for rendering and grouping in documentation.\n\nThis decorator is used solely for documentation purposes and does not modify the behavior\nof the decorated callable.\n" + } + ] + }, + "flags": {}, + "id": 609, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "docs_group", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The documentation group to which the symbol belongs.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 610, + "kind": 32768, + "kindString": "Parameter", + "name": "group_name", + "type": { + "name": "GroupName", + "type": "reference", + "target": "607" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 611, + "module": "_utils", + "name": "docs_name", + "parsedDocstring": { + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n", + "args": { + "symbol_name": "The name to be used in the documentation.\n" + }, + "returns": "The original callable without modification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 107 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The original callable without modification." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Rename a symbol for documentation rendering.\n\nThis decorator modifies only the displayed name of the symbol in the generated documentation\nand does not affect its runtime behavior.\n" + } + ] + }, + "flags": {}, + "id": 612, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "docs_name", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name to be used in the documentation.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 613, + "kind": 32768, + "kindString": "Parameter", + "name": "symbol_name", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Callable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extract the value of an enumeration member if it is an Enum, otherwise return the original value." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 614, + "module": "_utils", + "name": "maybe_extract_enum_member_value", + "parsedDocstring": { + "text": "Extract the value of an enumeration member if it is an Enum, otherwise return the original value." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 126 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extract the value of an enumeration member if it is an Enum, otherwise return the original value." + } + ] + }, + "flags": {}, + "id": 615, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "maybe_extract_enum_member_value", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 616, + "kind": 32768, + "kindString": "Parameter", + "name": "maybe_enum_member", + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 618, + "module": "_utils", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 136 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 619, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Acquire the lock if it's not already owned by the current task, otherwise proceed without acquiring." + } + ] + }, + "decorations": [ + { + "name": "asynccontextmanager" + } + ], + "flags": {}, + "groups": [], + "id": 620, + "module": "_utils", + "name": "__call__", + "parsedDocstring": { + "text": "Acquire the lock if it's not already owned by the current task, otherwise proceed without acquiring." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 141 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Acquire the lock if it's not already owned by the current task, otherwise proceed without acquiring." + } + ] + }, + "flags": {}, + "id": 621, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__call__", + "parameters": [], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A reentrant lock implementation for asyncio using asyncio.Lock." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 620, + 618 + ], + "title": "Methods" + } + ], + "id": 617, + "module": "_utils", + "name": "ReentrantLock", + "parsedDocstring": { + "text": "A reentrant lock implementation for asyncio using asyncio.Lock." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 133 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 623, + "module": "events._types", + "name": "mem_avg_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 624, + "module": "events._types", + "name": "mem_current_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 625, + "module": "events._types", + "name": "mem_max_bytes", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 626, + "module": "events._types", + "name": "cpu_avg_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 627, + "module": "events._types", + "name": "cpu_max_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 628, + "module": "events._types", + "name": "cpu_current_usage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 27 + } + ], + "type": { + "name": "float", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 629, + "module": "events._types", + "name": "is_cpu_overloaded", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 630, + "module": "events._types", + "name": "created_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 631, + "module": "events._types", + "name": "to_crawlee_format", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 632, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_crawlee_format", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 633, + "kind": 32768, + "kindString": "Parameter", + "name": "dedicated_cpus", + "type": { + "name": "float", + "type": "reference" + } + } + ], + "type": { + "name": "EventSystemInfoData", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 631 + ], + "title": "Methods" + }, + { + "children": [ + 626, + 628, + 627, + 630, + 629, + 623, + 624, + 625 + ], + "title": "Properties" + } + ], + "id": 622, + "module": "events._types", + "name": "SystemInfoEventData", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 635, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 636, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 50 + } + ], + "type": { + "name": "EventPersistStateData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 636, + 635 + ], + "title": "Properties" + } + ], + "id": 634, + "module": "events._types", + "name": "PersistStateEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 638, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 639, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "type": { + "name": "SystemInfoEventData", + "type": "reference", + "target": "622" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 639, + 638 + ], + "title": "Properties" + } + ], + "id": 637, + "module": "events._types", + "name": "SystemInfoEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 54 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 641, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 642, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "type": { + "name": "EventMigratingData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 642, + 641 + ], + "title": "Properties" + } + ], + "id": 640, + "module": "events._types", + "name": "MigratingEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 60 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 644, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 645, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 68 + } + ], + "type": { + "name": "EventAbortingData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 645, + 644 + ], + "title": "Properties" + } + ], + "id": 643, + "module": "events._types", + "name": "AbortingEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 66 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 647, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 648, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "type": { + "name": "EventExitData", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 648, + 647 + ], + "title": "Properties" + } + ], + "id": 646, + "module": "events._types", + "name": "ExitEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 72 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 650, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 79 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SESSION_RETIRED" + }, + { + "type": "reference", + "name": "Event.BROWSER_LAUNCHED" + }, + { + "type": "reference", + "name": "Event.BROWSER_RETIRED" + }, + { + "type": "reference", + "name": "Event.BROWSER_CLOSED" + }, + { + "type": "reference", + "name": "Event.PAGE_CREATED" + }, + { + "type": "reference", + "name": "Event.PAGE_CLOSED" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 651, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 651, + 650 + ], + "title": "Properties" + } + ], + "id": 649, + "module": "events._types", + "name": "EventWithoutData", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 653, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 92 + } + ], + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "cpuInfo" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 654, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 93 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 654, + 653 + ], + "title": "Properties" + } + ], + "id": 652, + "module": "events._types", + "name": "DeprecatedEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 656, + "module": "events._types", + "name": "name", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 98 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 657, + "module": "events._types", + "name": "data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": "('Events')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 657, + 656 + ], + "title": "Properties" + } + ], + "id": 655, + "module": "events._types", + "name": "UnknownEvent", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 97 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 658, + "module": "events._types", + "name": "EventMessage", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 102 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 659, + "module": "events._apify_event_manager", + "name": "event_data_adapter", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_apify_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 661, + "module": "events._apify_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "configuration": "The Actor configuration for the event manager.", + "**kwargs": "Additional event manager options passed to the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_apify_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 662, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor configuration for the event manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 663, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 665, + "module": "events._apify_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_apify_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 666, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 667, + "module": "events._apify_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_apify_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 88 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 668, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 669, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 670, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 671, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Event manager for the Apify platform.\n\nThis class extends Crawlee's `EventManager` to provide Apify-specific functionality, including websocket\nconnectivity to the Apify platform for receiving platform events.\n\nThe event manager handles:\n- Registration and emission of events and their listeners.\n- Websocket connection to Apify platform events.\n- Processing and validation of platform messages.\n- Automatic event forwarding from the platform to local event listeners.\n\nThis class should not be used directly. Use the `Actor.on` and `Actor.off` methods to interact\nwith the event system." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 665, + 667, + 661 + ], + "title": "Methods" + } + ], + "id": 660, + "module": "events._apify_event_manager", + "name": "ApifyEventManager", + "parsedDocstring": { + "text": "Event manager for the Apify platform.\n\nThis class extends Crawlee's `EventManager` to provide Apify-specific functionality, including websocket\nconnectivity to the Apify platform for receiving platform events.\n\nThe event manager handles:\n- Registration and emission of events and their listeners.\n- Websocket connection to Apify platform events.\n- Processing and validation of platform messages.\n- Automatic event forwarding from the platform to local event listeners.\n\nThis class should not be used directly. Use the `Actor.on` and `Actor.off` methods to interact\nwith the event system." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/events/_apify_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 672, + "module": "request_loaders._apify_request_list", + "name": "URL_NO_COMMAS_REGEX", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 674, + "module": "request_loaders._apify_request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 675, + "module": "request_loaders._apify_request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 676, + "module": "request_loaders._apify_request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 677, + "module": "request_loaders._apify_request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 676, + 674, + 675, + 677 + ], + "title": "Properties" + } + ], + "id": 673, + "module": "request_loaders._apify_request_list", + "name": "_RequestDetails", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedBy": [ + { + "name": "_RequestsFromUrlInput", + "target": "678", + "type": "reference" + }, + { + "name": "_SimpleUrlInput", + "target": "680", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 679, + "module": "request_loaders._apify_request_list", + "name": "requests_from_url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 30 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1158, + "module": "request_loaders._apify_request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.method", + "target": 674, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1159, + "module": "request_loaders._apify_request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "str", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.payload", + "target": 675, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1160, + "module": "request_loaders._apify_request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.headers", + "target": 676, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1161, + "module": "request_loaders._apify_request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.user_data", + "target": 677, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1160, + 1158, + 1159, + 679, + 1161 + ], + "title": "Properties" + } + ], + "id": 678, + "module": "request_loaders._apify_request_list", + "name": "_RequestsFromUrlInput", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "_RequestDetails", + "target": "673", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 681, + "module": "request_loaders._apify_request_list", + "name": "url", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1162, + "module": "request_loaders._apify_request_list", + "name": "method", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "HttpMethod", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.method", + "target": 674, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1163, + "module": "request_loaders._apify_request_list", + "name": "payload", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "str", + "type": "reference" + }, + "inheritedFrom": { + "name": "_RequestDetails.payload", + "target": 675, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1164, + "module": "request_loaders._apify_request_list", + "name": "headers", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.headers", + "target": 676, + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1165, + "module": "request_loaders._apify_request_list", + "name": "user_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "str" + } + ] + }, + "inheritedFrom": { + "name": "_RequestDetails.user_data", + "target": 677, + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1164, + 1162, + 1163, + 681, + 1165 + ], + "title": "Properties" + } + ], + "id": 680, + "module": "request_loaders._apify_request_list", + "name": "_SimpleUrlInput", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "_RequestDetails", + "target": "673", + "type": "reference" + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 682, + "module": "request_loaders._apify_request_list", + "name": "url_input_adapter", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 684, + "module": "request_loaders._apify_request_list", + "name": "open", + "parsedDocstring": { + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```", + "args": { + "name": "Name of the returned RequestList.", + "request_list_sources_input": "List of dicts with either url key or requestsFromUrl key.", + "http_client": "Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\n" + }, + "returns": "RequestList created from request_list_sources_input." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 48 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "RequestList created from request_list_sources_input." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance from request list source input.\n\n\n### Usage\n\n```python\nexample_input = [\n # Gather urls from response body.\n {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},\n # Directly include this url.\n {'url': 'https://crawlee.dev', 'method': 'GET'}\n]\nrequest_list = await RequestList.open(request_list_sources_input=example_input)\n```" + } + ] + }, + "flags": {}, + "id": 685, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the returned RequestList." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 686, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of dicts with either url key or requestsFromUrl key." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 687, + "kind": 32768, + "kindString": "Parameter", + "name": "request_list_sources_input", + "type": { + "name": "list[dict[str, Any]] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Client that will be used to send get request to urls defined by value of requestsFromUrl keys.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 688, + "kind": 32768, + "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpClient" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyRequestList", + "type": "reference", + "target": "683" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extends crawlee RequestList.\n\nMethod open is used to create RequestList from actor's requestListSources input." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 684 + ], + "title": "Methods" + } + ], + "id": 683, + "module": "request_loaders._apify_request_list", + "name": "ApifyRequestList", + "parsedDocstring": { + "text": "Extends crawlee RequestList.\n\nMethod open is used to create RequestList from actor's requestListSources input." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/request_loaders/_apify_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 689, + "module": "scrapy.extensions._httpcache", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 691, + "module": "scrapy.extensions._httpcache", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 692, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 693, + "kind": 32768, + "kindString": "Parameter", + "name": "settings", + "type": { + "name": "BaseSettings", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the cache storage for a spider." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 694, + "module": "scrapy.extensions._httpcache", + "name": "open_spider", + "parsedDocstring": { + "text": "Open the cache storage for a spider." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 46 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the cache storage for a spider." + } + ] + }, + "flags": {}, + "id": 695, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "open_spider", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 696, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the cache storage for a spider." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 697, + "module": "scrapy.extensions._httpcache", + "name": "close_spider", + "parsedDocstring": { + "text": "Close the cache storage for a spider." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the cache storage for a spider." + } + ] + }, + "flags": {}, + "id": 698, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close_spider", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 699, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 700, + "kind": 32768, + "kindString": "Parameter", + "name": "current_time", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a response from the cache storage." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 701, + "module": "scrapy.extensions._httpcache", + "name": "retrieve_response", + "parsedDocstring": { + "text": "Retrieve a response from the cache storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a response from the cache storage." + } + ] + }, + "flags": {}, + "id": 702, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "retrieve_response", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 703, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 704, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 705, + "kind": 32768, + "kindString": "Parameter", + "name": "current_time", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Response | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Response" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store a response in the cache storage." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 706, + "module": "scrapy.extensions._httpcache", + "name": "store_response", + "parsedDocstring": { + "text": "Store a response in the cache storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 144 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store a response in the cache storage." + } + ] + }, + "flags": {}, + "id": 707, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "store_response", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 708, + "kind": 32768, + "kindString": "Parameter", + "name": "_", + "type": { + "name": "Spider", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 709, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 710, + "kind": 32768, + "kindString": "Parameter", + "name": "response", + "type": { + "name": "Response", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\n\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\nto be installed." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 691, + 697, + 694, + 701, + 706 + ], + "title": "Methods" + } + ], + "id": 690, + "module": "scrapy.extensions._httpcache", + "name": "ApifyCacheStorage", + "parsedDocstring": { + "text": "A Scrapy cache storage that uses the Apify `KeyValueStore` to store responses.\n\nIt can be set as a storage for Scrapy's built-in `HttpCacheMiddleware`, which caches\nresponses to requests. See HTTPCache middleware settings (prefixed with `HTTPCACHE_`)\nin the Scrapy documentation for more information. Requires the asyncio Twisted reactor\nto be installed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dump a dictionary to a gzip-compressed byte stream." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 711, + "module": "scrapy.extensions._httpcache", + "name": "to_gzip", + "parsedDocstring": { + "text": "Dump a dictionary to a gzip-compressed byte stream." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 164 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dump a dictionary to a gzip-compressed byte stream." + } + ] + }, + "flags": {}, + "id": 712, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_gzip", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 713, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "dict", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 714, + "kind": 32768, + "kindString": "Parameter", + "name": "mtime", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load a dictionary from a gzip-compressed byte stream." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 715, + "module": "scrapy.extensions._httpcache", + "name": "from_gzip", + "parsedDocstring": { + "text": "Load a dictionary from a gzip-compressed byte stream." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 172 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Load a dictionary from a gzip-compressed byte stream." + } + ] + }, + "flags": {}, + "id": 716, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_gzip", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 717, + "kind": 32768, + "kindString": "Parameter", + "name": "gzip_bytes", + "type": { + "name": "bytes", + "type": "reference" + } + } + ], + "type": { + "name": "dict", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 718, + "module": "scrapy.extensions._httpcache", + "name": "read_gzip_time", + "parsedDocstring": { + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Read the modification time from a gzip-compressed byte stream without decompressing the data." + } + ] + }, + "flags": {}, + "id": 719, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "read_gzip_time", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 720, + "kind": 32768, + "kindString": "Parameter", + "name": "gzip_bytes", + "type": { + "name": "bytes", + "type": "reference" + } + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 721, + "module": "scrapy.extensions._httpcache", + "name": "get_kvs_name", + "parsedDocstring": { + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n", + "args": { + "spider_name": "Value of the Spider instance's name attribute.", + "max_length": "Maximum length of the key value store name.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/extensions/_httpcache.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 187 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the key value store name for a spider.\n\nThe key value store name is derived from the spider name by replacing all special characters\nwith hyphens and trimming leading and trailing hyphens. The resulting name is prefixed with\n'httpcache-' and truncated to the maximum length.\n\nThe documentation\n[about storages](https://docs.apify.com/platform/storage/usage#named-and-unnamed-storages)\nmentions that names can be up to 63 characters long, so the default max length is set to 60.\n\nSuch naming isn't unique per spider, but should be sufficiently unique for most use cases.\nThe name of the key value store should indicate to which spider it belongs, e.g. in\nthe listing in the Apify's console.\n\n\nReturns: Key value store name.\n" + } + ] + }, + "flags": {}, + "id": 722, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_kvs_name", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value of the Spider instance's name attribute." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 723, + "kind": 32768, + "kindString": "Parameter", + "name": "spider_name", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum length of the key value store name.\n" + } + ] + }, + "defaultValue": "60", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 724, + "kind": 32768, + "kindString": "Parameter", + "name": "max_length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 726, + "module": "scrapy.middlewares.apify_proxy", + "name": "__init__", + "parsedDocstring": { + "text": "Create a new instance.\n", + "args": { + "proxy_settings": "Dictionary containing proxy settings, provided by the Actor input.", + "auth_encoding": "Encoding for basic authentication (default is 'latin-1')." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 727, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dictionary containing proxy settings, provided by the Actor input." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 728, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_settings", + "type": { + "name": "dict", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class." + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 729, + "module": "scrapy.middlewares.apify_proxy", + "name": "from_crawler", + "parsedDocstring": { + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class.", + "args": { + "crawler": "Scrapy Crawler object.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 39 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.\n\n\nReturns: Instance of the class." + } + ] + }, + "flags": {}, + "id": 730, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_crawler", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Crawler object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 731, + "kind": 32768, + "kindString": "Parameter", + "name": "crawler", + "type": { + "name": "Crawler", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyHttpProxyMiddleware", + "type": "reference", + "target": "725" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 732, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_request", + "parsedDocstring": { + "text": "Process a Scrapy request by assigning a new proxy.\n", + "args": { + "request": "Scrapy Request object.", + "spider": "Scrapy Spider object.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process a Scrapy request by assigning a new proxy.\n" + } + ] + }, + "flags": {}, + "id": 733, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 734, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 735, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 736, + "module": "scrapy.middlewares.apify_proxy", + "name": "process_exception", + "parsedDocstring": { + "text": "Process an exception that occurs during request processing.\n", + "args": { + "request": "Scrapy Request object.", + "exception": "Exception object.", + "spider": "Scrapy Spider object.\n" + }, + "returns": "Returning None, meaning Scrapy will continue processing this exception, executing any other\nprocess_exception() methods of installed middleware, until no middleware is left and the default\nexception handling kicks in." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Returning None, meaning Scrapy will continue processing this exception, executing any other\nprocess_exception() methods of installed middleware, until no middleware is left and the default\nexception handling kicks in." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Process an exception that occurs during request processing.\n" + } + ] + }, + "flags": {}, + "id": 737, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "process_exception", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Request object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 738, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exception object." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 739, + "kind": 32768, + "kindString": "Parameter", + "name": "exception", + "type": { + "name": "Exception", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy Spider object.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 740, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 726, + 729, + 736, + 732 + ], + "title": "Methods" + } + ], + "id": 725, + "module": "scrapy.middlewares.apify_proxy", + "name": "ApifyHttpProxyMiddleware", + "parsedDocstring": { + "text": "Apify HTTP proxy middleware for Scrapy.\n\nThis middleware enhances request processing by adding a 'proxy' field to the request's meta and an authentication\nheader. It draws inspiration from the `HttpProxyMiddleware` included by default in Scrapy projects. The proxy URL\nis sourced from the settings under the `APIFY_PROXY_SETTINGS` key. The value of this key, a dictionary, should be\nprovided by the Actor input. An example of the proxy settings:\n\nproxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/middlewares/apify_proxy.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 741, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 743, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "process_item", + "parsedDocstring": { + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pushes the provided Scrapy item to the Actor's default dataset." + } + ] + }, + "flags": {}, + "id": 744, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "process_item", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 745, + "kind": 32768, + "kindString": "Parameter", + "name": "item", + "type": { + "name": "Item", + "type": "reference" + } + } + ], + "type": { + "name": "Item", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 743 + ], + "title": "Methods" + } + ], + "id": 742, + "module": "scrapy.pipelines.actor_dataset_push", + "name": "ActorDatasetPushPipeline", + "parsedDocstring": { + "text": "A Scrapy pipeline for pushing items to an Actor's default dataset.\n\nThis pipeline is designed to be enabled only when the Scrapy project is run as an Actor." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 746, + "module": "scrapy.utils", + "name": "get_basic_auth_header", + "parsedDocstring": { + "text": "Generate a basic authentication header for the given username and password." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a basic authentication header for the given username and password." + } + ] + }, + "flags": {}, + "id": 747, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_basic_auth_header", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 748, + "kind": 32768, + "kindString": "Parameter", + "name": "username", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 749, + "kind": 32768, + "kindString": "Parameter", + "name": "password", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "defaultValue": "'latin-1'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 750, + "kind": 32768, + "kindString": "Parameter", + "name": "auth_encoding", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bytes", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 751, + "module": "scrapy.utils", + "name": "apply_apify_settings", + "parsedDocstring": { + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n", + "args": { + "settings": "Scrapy project settings to be modified.", + "proxy_config": "Proxy configuration to be stored in the settings.\n" + }, + "returns": "Scrapy project settings with custom configurations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Scrapy project settings with custom configurations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Integrates Apify configuration into a Scrapy project settings.\n\nNote: The function directly modifies the passed `settings` object and also returns it.\n" + } + ] + }, + "flags": {}, + "id": 752, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "apply_apify_settings", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Scrapy project settings to be modified." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 753, + "kind": 32768, + "kindString": "Parameter", + "name": "settings", + "type": { + "name": "Settings | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Settings" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Proxy configuration to be stored in the settings.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 754, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_config", + "type": { + "name": "dict | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "Settings", + "type": "reference" + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 755, + "module": "scrapy._async_thread", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 757, + "module": "scrapy._async_thread", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 758, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 759, + "module": "scrapy._async_thread", + "name": "run_coro", + "parsedDocstring": { + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n", + "args": { + "coro": "The coroutine to run.", + "timeout": "The maximum number of seconds to wait for the coroutine to finish.\n" + }, + "returns": "The result returned by the coroutine." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The result returned by the coroutine." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Run a coroutine on an event loop running in a separate thread.\n\nThis method schedules the coroutine to run on the event loop and blocks until the coroutine completes\nor the specified timeout is reached.\n" + } + ] + }, + "flags": {}, + "id": 760, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "run_coro", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The coroutine to run." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 761, + "kind": 32768, + "kindString": "Parameter", + "name": "coro", + "type": { + "name": "Coroutine", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds to wait for the coroutine to finish.\n" + } + ] + }, + "defaultValue": "timedelta(seconds=60)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 762, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "Any", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 763, + "module": "scrapy._async_thread", + "name": "close", + "parsedDocstring": { + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n", + "args": { + "timeout": "The maximum number of seconds to wait for the event loop thread to exit." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the event loop and its thread gracefully.\n\nThis method cancels all pending tasks, stops the event loop, and waits for the thread to exit.\nIf the thread does not exit within the given timeout, a forced shutdown is attempted.\n" + } + ] + }, + "flags": {}, + "id": 764, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of seconds to wait for the event loop thread to exit." + } + ] + }, + "defaultValue": "timedelta(seconds=60)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 765, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Class for running an asyncio event loop in a separate thread.\n\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\nthat runs in its own dedicated thread." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 757, + 763, + 759 + ], + "title": "Methods" + } + ], + "id": 756, + "module": "scrapy._async_thread", + "name": "AsyncThread", + "parsedDocstring": { + "text": "Class for running an asyncio event loop in a separate thread.\n\nThis allows running asynchronous coroutines from synchronous code by executingthem on an event loop\nthat runs in its own dedicated thread." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_async_thread.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 16 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 766, + "module": "scrapy._logging_config", + "name": "initialize_logging", + "parsedDocstring": { + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_logging_config.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configure logging for Apify Actors and adjust Scrapy's logging settings." + } + ] + }, + "flags": {}, + "id": 767, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "initialize_logging", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function installs Twisted's asyncio-compatible reactor, then initiates it and runs the given asyncio\ncoroutine (typically the Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted\nevent loops, enabling the Apify and Scrapy integration to work together." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 768, + "module": "scrapy._actor_runner", + "name": "run_scrapy_actor", + "parsedDocstring": { + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function installs Twisted's asyncio-compatible reactor, then initiates it and runs the given asyncio\ncoroutine (typically the Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted\nevent loops, enabling the Apify and Scrapy integration to work together." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/_actor_runner.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 10 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start Twisted's reactor and execute the provided Actor coroutine.\n\nThis function installs Twisted's asyncio-compatible reactor, then initiates it and runs the given asyncio\ncoroutine (typically the Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted\nevent loops, enabling the Apify and Scrapy integration to work together." + } + ] + }, + "flags": {}, + "id": 769, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "run_scrapy_actor", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 770, + "kind": 32768, + "kindString": "Parameter", + "name": "coro", + "type": { + "name": "Coroutine", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 771, + "module": "scrapy.scheduler", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 773, + "module": "scrapy.scheduler", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 30 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 774, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 775, + "module": "scrapy.scheduler", + "name": "open", + "parsedDocstring": { + "text": "Open the scheduler.\n", + "args": { + "spider": "The spider that the scheduler is associated with." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 776, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The spider that the scheduler is associated with." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 777, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "Deferred[None] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Deferred", + "typeArguments": [ + { + "type": "literal", + "value": null + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 778, + "module": "scrapy.scheduler", + "name": "close", + "parsedDocstring": { + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n", + "args": { + "reason": "The reason for closing the spider." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the scheduler.\n\nShut down the event loop and its thread gracefully.\n" + } + ] + }, + "flags": {}, + "id": 779, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "close", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The reason for closing the spider." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 780, + "kind": 32768, + "kindString": "Parameter", + "name": "reason", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 781, + "module": "scrapy.scheduler", + "name": "has_pending_requests", + "parsedDocstring": { + "text": "Check if the scheduler has any pending requests.\n", + "returns": "True if the scheduler has any pending requests, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the scheduler has any pending requests, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the scheduler has any pending requests.\n" + } + ] + }, + "flags": {}, + "id": 782, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "has_pending_requests", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 783, + "module": "scrapy.scheduler", + "name": "enqueue_request", + "parsedDocstring": { + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n", + "args": { + "request": "The request to add to the scheduler.\n" + }, + "returns": "True if the request was successfully enqueued, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request was successfully enqueued, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a request to the scheduler.\n\nThis could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).\n" + } + ] + }, + "flags": {}, + "id": 784, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to add to the scheduler.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 785, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 786, + "module": "scrapy.scheduler", + "name": "next_request", + "parsedDocstring": { + "text": "Fetch the next request from the scheduler.\n", + "returns": "The next request, or None if there are no more requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 142 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request, or None if there are no more requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Fetch the next request from the scheduler.\n" + } + ] + }, + "flags": {}, + "id": 787, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 773, + 778, + 783, + 781, + 786, + 775 + ], + "title": "Methods" + } + ], + "id": 772, + "module": "scrapy.scheduler", + "name": "ApifyScheduler", + "parsedDocstring": { + "text": "A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.\n\nThis scheduler requires the asyncio Twisted reactor to be installed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/scheduler.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 788, + "module": "scrapy.requests", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 789, + "module": "scrapy.requests", + "name": "to_apify_request", + "parsedDocstring": { + "text": "Convert a Scrapy request to an Apify request.\n", + "args": { + "scrapy_request": "The Scrapy request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Apify request if the conversion was successful, otherwise None." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Apify request if the conversion was successful, otherwise None." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert a Scrapy request to an Apify request.\n" + } + ] + }, + "flags": {}, + "id": 790, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_apify_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 791, + "kind": 32768, + "kindString": "Parameter", + "name": "scrapy_request", + "type": { + "name": "ScrapyRequest", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 792, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "ApifyRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ApifyRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 793, + "module": "scrapy.requests", + "name": "to_scrapy_request", + "parsedDocstring": { + "text": "Convert an Apify request to a Scrapy request.\n", + "args": { + "apify_request": "The Apify request to be converted.", + "spider": "The Scrapy spider that the request is associated with.\n" + }, + "returns": "The converted Scrapy request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/scrapy/requests.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 96 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The converted Scrapy request." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Convert an Apify request to a Scrapy request.\n" + } + ] + }, + "flags": {}, + "id": 794, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "to_scrapy_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify request to be converted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 795, + "kind": 32768, + "kindString": "Parameter", + "name": "apify_request", + "type": { + "name": "ApifyRequest", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Scrapy spider that the request is associated with.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 796, + "kind": 32768, + "kindString": "Parameter", + "name": "spider", + "type": { + "name": "Spider", + "type": "reference" + } + } + ], + "type": { + "name": "ScrapyRequest", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 798, + "module": "storage_clients._apify._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "request_queue_access": "Defines how the request queue client behaves. Use `single` mode for a single\nconsumer. It has fewer API calls, meaning better performance and lower costs. If you need multiple\nconcurrent consumers use `shared` mode, but expect worse performance and higher costs due to\nthe additional overhead." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 66 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 799, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Defines how the request queue client behaves. Use `single` mode for a single\nconsumer. It has fewer API calls, meaning better performance and lower costs. If you need multiple\nconcurrent consumers use `shared` mode, but expect worse performance and higher costs due to\nthe additional overhead." + } + ] + }, + "defaultValue": "'single'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 800, + "kind": 32768, + "kindString": "Parameter", + "name": "request_queue_access", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "single" + }, + { + "type": "literal", + "value": "shared" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 801, + "module": "storage_clients._apify._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 802, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 803, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 804, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 805, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 806, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyDatasetClient", + "type": "reference", + "target": "1031" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 807, + "module": "storage_clients._apify._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 93 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 808, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 809, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 810, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 811, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 812, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyKeyValueStoreClient", + "type": "reference", + "target": "823" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 813, + "module": "storage_clients._apify._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 814, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 815, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 816, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 817, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 818, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyRequestQueueClient", + "type": "reference", + "target": "863" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 819, + "module": "storage_clients._apify._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 125 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 820, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 821, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration", + "type": "reference" + } + } + ], + "type": { + "name": "Hashable", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify platform implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the Apify platform. Each storage type is implemented with its own specific Apify client that stores data\nin the cloud, making it accessible from anywhere.\n\nThe communication with the Apify platform is handled via the Apify API client for Python, which is an HTTP API\nwrapper. For maximum efficiency and performance of the storage clients, various caching mechanisms are used to\nminimize the number of API calls made to the Apify platform. Data can be inspected and manipulated through\nthe Apify console web interface or via the Apify API.\n\nThe request queue client supports two access modes controlled by the `request_queue_access` parameter:\n\n### Single mode\n\nThe `single` mode is optimized for scenarios with only one consumer. It minimizes API calls, making it faster\nand more cost-efficient compared to the `shared` mode. This option is ideal when a single Actor is responsible\nfor consuming the entire request queue. Using multiple consumers simultaneously may lead to inconsistencies\nor unexpected behavior.\n\nIn this mode, multiple producers can safely add new requests, but forefront requests may not be processed\nimmediately, as the client relies on local head estimation instead of frequent forefront fetching. Requests can\nalso be added or marked as handled by other clients, but they must not be deleted or modified, since such changes\nwould not be reflected in the local cache. If a request is already fully cached locally, marking it as handled\nby another client will be ignored by this client. This does not cause errors but can occasionally result in\nreprocessing a request that was already handled elsewhere. If the request was not yet cached locally, marking\nit as handled poses no issue.\n\n### Shared mode\n\nThe `shared` mode is designed for scenarios with multiple concurrent consumers. It ensures proper synchronization\nand consistency across clients, at the cost of higher API usage and slightly worse performance. This mode is safe\nfor concurrent access from multiple processes, including Actors running in parallel on the Apify platform. It\nshould be used when multiple consumers need to process requests from the same queue simultaneously." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 798, + 801, + 807, + 813, + 819 + ], + "title": "Methods" + } + ], + "id": 797, + "module": "storage_clients._apify._storage_client", + "name": "ApifyStorageClient", + "parsedDocstring": { + "text": "Apify platform implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the Apify platform. Each storage type is implemented with its own specific Apify client that stores data\nin the cloud, making it accessible from anywhere.\n\nThe communication with the Apify platform is handled via the Apify API client for Python, which is an HTTP API\nwrapper. For maximum efficiency and performance of the storage clients, various caching mechanisms are used to\nminimize the number of API calls made to the Apify platform. Data can be inspected and manipulated through\nthe Apify console web interface or via the Apify API.\n\nThe request queue client supports two access modes controlled by the `request_queue_access` parameter:\n\n### Single mode\n\nThe `single` mode is optimized for scenarios with only one consumer. It minimizes API calls, making it faster\nand more cost-efficient compared to the `shared` mode. This option is ideal when a single Actor is responsible\nfor consuming the entire request queue. Using multiple consumers simultaneously may lead to inconsistencies\nor unexpected behavior.\n\nIn this mode, multiple producers can safely add new requests, but forefront requests may not be processed\nimmediately, as the client relies on local head estimation instead of frequent forefront fetching. Requests can\nalso be added or marked as handled by other clients, but they must not be deleted or modified, since such changes\nwould not be reflected in the local cache. If a request is already fully cached locally, marking it as handled\nby another client will be ignored by this client. This does not cause errors but can occasionally result in\nreprocessing a request that was already handled elsewhere. If the request was not yet cached locally, marking\nit as handled poses no issue.\n\n### Shared mode\n\nThe `shared` mode is designed for scenarios with multiple concurrent consumers. It ensures proper synchronization\nand consistency across clients, at the cost of higher API usage and slightly worse performance. This mode is safe\nfor concurrent access from multiple processes, including Actors running in parallel on the Apify platform. It\nshould be used when multiple consumers need to process requests from the same queue simultaneously." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 822, + "module": "storage_clients._apify._key_value_store_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyKeyValueStoreClient.open` class method to create a new instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 824, + "module": "storage_clients._apify._key_value_store_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `ApifyKeyValueStoreClient.open` class method to create a new instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyKeyValueStoreClient.open` class method to create a new instance." + } + ] + }, + "flags": {}, + "id": 825, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 826, + "kind": 32768, + "kindString": "Parameter", + "name": "api_client", + "type": { + "name": "KeyValueStoreClientAsync", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 827, + "kind": 32768, + "kindString": "Parameter", + "name": "api_public_base_url", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 828, + "kind": 32768, + "kindString": "Parameter", + "name": "lock", + "type": { + "name": "asyncio.Lock", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 829, + "module": "storage_clients._apify._key_value_store_client", + "name": "get_metadata", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 830, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "ApifyKeyValueStoreMetadata", + "type": "reference", + "target": "933" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open an Apify key-value store client.\n\nThis method creates and initializes a new instance of the Apify key-value store client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 831, + "module": "storage_clients._apify._key_value_store_client", + "name": "open", + "parsedDocstring": { + "text": "Open an Apify key-value store client.\n\nThis method creates and initializes a new instance of the Apify key-value store client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n", + "args": { + "id": "The ID of the KVS to open. If provided, searches for existing KVS by ID.\nMutually exclusive with name and alias.", + "name": "The name of the KVS to open (global scope, persists across runs).\nMutually exclusive with id and alias.", + "alias": "The alias of the KVS to open (run scope, creates unnamed storage).\nMutually exclusive with id and name.", + "configuration": "The configuration object containing API credentials and settings. Must include a valid\n`token` and `api_base_url`. May also contain a `default_key_value_store_id` for fallback when\nneither `id`, `name`, nor `alias` is provided.\n" + }, + "returns": "An instance for the opened or created storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 60 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance for the opened or created storage client." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open an Apify key-value store client.\n\nThis method creates and initializes a new instance of the Apify key-value store client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n" + } + ] + }, + "flags": {}, + "id": 832, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the KVS to open. If provided, searches for existing KVS by ID.\nMutually exclusive with name and alias." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 833, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the KVS to open (global scope, persists across runs).\nMutually exclusive with id and alias." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 834, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias of the KVS to open (run scope, creates unnamed storage).\nMutually exclusive with id and name." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 835, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration object containing API credentials and settings. Must include a valid\n`token` and `api_base_url`. May also contain a `default_key_value_store_id` for fallback when\nneither `id`, `name`, nor `alias` is provided.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 836, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "ApifyKeyValueStoreClient", + "type": "reference", + "target": "823" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 837, + "module": "storage_clients._apify._key_value_store_client", + "name": "purge", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 838, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 839, + "module": "storage_clients._apify._key_value_store_client", + "name": "drop", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 840, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 841, + "module": "storage_clients._apify._key_value_store_client", + "name": "get_value", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 118 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 842, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 843, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreRecord | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreRecord" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 844, + "module": "storage_clients._apify._key_value_store_client", + "name": "set_value", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 123 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 845, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 846, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 847, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 848, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 849, + "module": "storage_clients._apify._key_value_store_client", + "name": "delete_value", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 132 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 850, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_value", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 851, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 852, + "module": "storage_clients._apify._key_value_store_client", + "name": "iterate_keys", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 137 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 853, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 854, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 855, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata" + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 856, + "module": "storage_clients._apify._key_value_store_client", + "name": "record_exists", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 170 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 857, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "record_exists", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 858, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 859, + "module": "storage_clients._apify._key_value_store_client", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n", + "args": { + "key": "The key for which the URL should be generated.\n" + }, + "returns": "A public URL that can be used to access the value of the given key in the KVS." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 174 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A public URL that can be used to access the value of the given key in the KVS." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a URL for the given key that may be used to publicly access the value in the remote key-value store.\n" + } + ] + }, + "flags": {}, + "id": 860, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key for which the URL should be generated.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 861, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An Apify platform implementation of the key-value store client." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 824, + 849, + 839, + 829, + 859, + 841, + 852, + 831, + 837, + 856, + 844 + ], + "title": "Methods" + } + ], + "id": 823, + "module": "storage_clients._apify._key_value_store_client", + "name": "ApifyKeyValueStoreClient", + "parsedDocstring": { + "text": "An Apify platform implementation of the key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 862, + "module": "storage_clients._apify._request_queue_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyRequestQueueClient.open` class method to create a new instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 864, + "module": "storage_clients._apify._request_queue_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `ApifyRequestQueueClient.open` class method to create a new instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 38 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyRequestQueueClient.open` class method to create a new instance." + } + ] + }, + "flags": {}, + "id": 865, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 866, + "kind": 32768, + "kindString": "Parameter", + "name": "api_client", + "type": { + "name": "RequestQueueClientAsync", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 867, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata", + "type": { + "name": "RequestQueueMetadata", + "type": "reference" + } + }, + { + "defaultValue": "'single'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 868, + "kind": 32768, + "kindString": "Parameter", + "name": "access", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "single" + }, + { + "type": "literal", + "value": "shared" + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve current metadata about the request queue.\n\nThis method fetches metadata from the Apify API and merges it with local estimations to provide\nthe most up-to-date statistics. Local estimations are used to compensate for potential delays\nin API data propagation (typically a few seconds).\n" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 869, + "module": "storage_clients._apify._request_queue_client", + "name": "get_metadata", + "parsedDocstring": { + "text": "Retrieve current metadata about the request queue.\n\nThis method fetches metadata from the Apify API and merges it with local estimations to provide\nthe most up-to-date statistics. Local estimations are used to compensate for potential delays\nin API data propagation (typically a few seconds).\n", + "returns": "Request queue metadata with accurate counts and timestamps, combining API data with local estimates." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Request queue metadata with accurate counts and timestamps, combining API data with local estimates." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve current metadata about the request queue.\n\nThis method fetches metadata from the Apify API and merges it with local estimations to provide\nthe most up-to-date statistics. Local estimations are used to compensate for potential delays\nin API data propagation (typically a few seconds).\n" + } + ] + }, + "flags": {}, + "id": 870, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "ApifyRequestQueueMetadata", + "type": "reference", + "target": "970" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open an Apify request queue client.\n\nThis method creates and initializes a new request queue client instance, handling authentication,\nstorage lookup or creation, metadata retrieval, and initialization of internal caching structures.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 871, + "module": "storage_clients._apify._request_queue_client", + "name": "open", + "parsedDocstring": { + "text": "Open an Apify request queue client.\n\nThis method creates and initializes a new request queue client instance, handling authentication,\nstorage lookup or creation, metadata retrieval, and initialization of internal caching structures.\n", + "args": { + "id": "ID of an existing request queue to open. Mutually exclusive with `name` and `alias`.", + "name": "Name of the request queue to open or create (persists across Actor runs).\nMutually exclusive with `id` and `alias`.", + "alias": "Alias for the request queue (scoped to current Actor run, creates unnamed storage).\nMutually exclusive with `id` and `name`.", + "configuration": "Configuration object containing API credentials (`token`, `api_base_url`) and\noptionally a `default_request_queue_id` for fallback when no identifier is provided.", + "access": "Access mode controlling the client's behavior:\n- `single`: Optimized for single-consumer scenarios (lower API usage, better performance).\n- `shared`: Optimized for multi-consumer scenarios (more API calls, guaranteed consistency).\n" + }, + "returns": "An instance for the opened or created storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance for the opened or created storage client." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open an Apify request queue client.\n\nThis method creates and initializes a new request queue client instance, handling authentication,\nstorage lookup or creation, metadata retrieval, and initialization of internal caching structures.\n" + } + ] + }, + "flags": {}, + "id": 872, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of an existing request queue to open. Mutually exclusive with `name` and `alias`." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 873, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the request queue to open or create (persists across Actor runs).\nMutually exclusive with `id` and `alias`." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 874, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Alias for the request queue (scoped to current Actor run, creates unnamed storage).\nMutually exclusive with `id` and `name`." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 875, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object containing API credentials (`token`, `api_base_url`) and\noptionally a `default_request_queue_id` for fallback when no identifier is provided." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 876, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Access mode controlling the client's behavior:\n- `single`: Optimized for single-consumer scenarios (lower API usage, better performance).\n- `shared`: Optimized for multi-consumer scenarios (more API calls, guaranteed consistency).\n" + } + ] + }, + "defaultValue": "'single'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 877, + "kind": 32768, + "kindString": "Parameter", + "name": "access", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "single" + }, + { + "type": "literal", + "value": "shared" + } + ] + } + } + ], + "type": { + "name": "ApifyRequestQueueClient", + "type": "reference", + "target": "863" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 878, + "module": "storage_clients._apify._request_queue_client", + "name": "purge", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 879, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 880, + "module": "storage_clients._apify._request_queue_client", + "name": "drop", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 164 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 881, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 882, + "module": "storage_clients._apify._request_queue_client", + "name": "add_batch_of_requests", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 168 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 883, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_batch_of_requests", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 884, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 885, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "AddRequestsResponse", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 886, + "module": "storage_clients._apify._request_queue_client", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 177 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 887, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 888, + "module": "storage_clients._apify._request_queue_client", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 181 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 889, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 890, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 891, + "module": "storage_clients._apify._request_queue_client", + "name": "get_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 185 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 892, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 893, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 894, + "module": "storage_clients._apify._request_queue_client", + "name": "reclaim_request", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 189 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 895, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 896, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 897, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 898, + "module": "storage_clients._apify._request_queue_client", + "name": "is_empty", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 198 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 899, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue client for the Apify platform.\n\nThis client provides access to request queues stored on the Apify platform, supporting both single-consumer\nand multi-consumer scenarios. It manages local caching, request fetching, and state synchronization with the\nplatform's API." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 864, + 882, + 880, + 886, + 869, + 891, + 898, + 888, + 871, + 878, + 894 + ], + "title": "Methods" + } + ], + "id": 863, + "module": "storage_clients._apify._request_queue_client", + "name": "ApifyRequestQueueClient", + "parsedDocstring": { + "text": "Request queue client for the Apify platform.\n\nThis client provides access to request queues stored on the Apify platform, supporting both single-consumer\nand multi-consumer scenarios. It manages local caching, request fetching, and state synchronization with the\nplatform's API." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 900, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new shared request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='shared')` instead of calling this directly.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 902, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new shared request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='shared')` instead of calling this directly.\n", + "args": { + "api_client": "The Apify API client for request queue operations.", + "metadata": "Initial metadata for the request queue.", + "cache_size": "Maximum number of requests to cache locally.", + "metadata_getter": "Async function to fetch current metadata from the API." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new shared request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='shared')` instead of calling this directly.\n" + } + ] + }, + "flags": {}, + "id": 903, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API client for request queue operations." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 904, + "kind": 32768, + "kindString": "Parameter", + "name": "api_client", + "type": { + "name": "RequestQueueClientAsync", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initial metadata for the request queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 905, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata", + "type": { + "name": "RequestQueueMetadata", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of requests to cache locally." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 906, + "kind": 32768, + "kindString": "Parameter", + "name": "cache_size", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async function to fetch current metadata from the API." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 907, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata_getter", + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "[]" + }, + { + "type": "reference", + "name": "Coroutine", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + }, + { + "type": "reference", + "name": "Any" + }, + { + "type": "reference", + "name": "ApifyRequestQueueMetadata", + "target": "970" + } + ] + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 908, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "add_batch_of_requests", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 82 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 909, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_batch_of_requests", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 910, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 911, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "AddRequestsResponse", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 912, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "get_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 166 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 913, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 914, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 915, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 170 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 916, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 917, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 212 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 918, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 919, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 920, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 244 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 921, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 922, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 923, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 924, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "is_empty", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ shared access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 288 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ shared access mode." + } + ] + }, + "flags": {}, + "id": 925, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal request queue client implementation for multi-consumer scenarios on the Apify platform.\n\nThis implementation is optimized for scenarios where multiple clients concurrently fetch and process requests\nfrom the same queue. It makes more frequent API calls to ensure consistency across all consumers and uses\nrequest locking to prevent duplicate processing.\n\nThis class is used internally by `ApifyRequestQueueClient` when `access='shared'` is specified.\n\nPublic methods are not individually documented as they implement the interface defined in `RequestQueueClient`." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 902, + 908, + 915, + 912, + 924, + 917, + 920 + ], + "title": "Methods" + } + ], + "id": 901, + "module": "storage_clients._apify._request_queue_shared_client", + "name": "ApifyRequestQueueSharedClient", + "parsedDocstring": { + "text": "Internal request queue client implementation for multi-consumer scenarios on the Apify platform.\n\nThis implementation is optimized for scenarios where multiple clients concurrently fetch and process requests\nfrom the same queue. It makes more frequent API calls to ensure consistency across all consumers and uses\nrequest locking to prevent duplicate processing.\n\nThis class is used internally by `ApifyRequestQueueClient` when `access='shared'` is specified.\n\nPublic methods are not individually documented as they implement the interface defined in `RequestQueueClient`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Generate a deterministic request ID based on a unique key.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 926, + "module": "storage_clients._apify._utils", + "name": "unique_key_to_request_id", + "parsedDocstring": { + "text": "Generate a deterministic request ID based on a unique key.\n", + "args": { + "unique_key": "The unique key to convert into a request ID.", + "request_id_length": "The length of the request ID.\n" + }, + "returns": "A URL-safe, truncated request ID based on the unique key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 14 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A URL-safe, truncated request ID based on the unique key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Generate a deterministic request ID based on a unique key.\n" + } + ] + }, + "flags": {}, + "id": 927, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "unique_key_to_request_id", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique key to convert into a request ID." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 928, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The length of the request ID.\n" + } + ] + }, + "defaultValue": "15", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 929, + "kind": 32768, + "kindString": "Parameter", + "name": "request_id_length", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Hash configuration.api_public_base_url and configuration.token in deterministic way." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 930, + "module": "storage_clients._apify._utils", + "name": "hash_api_base_url_and_token", + "parsedDocstring": { + "text": "Hash configuration.api_public_base_url and configuration.token in deterministic way." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_utils.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Hash configuration.api_public_base_url and configuration.token in deterministic way." + } + ] + }, + "flags": {}, + "id": 931, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "hash_api_base_url_and_token", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 932, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The secret key used for signing URLs for secure access to key-value store records." + } + ] + }, + "flags": {}, + "groups": [], + "id": 934, + "module": "storage_clients._apify._models", + "name": "url_signing_secret_key", + "parsedDocstring": { + "text": "The secret key used for signing URLs for secure access to key-value store records." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='urlSigningSecretKey', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Extended key-value store metadata model for Apify platform.\n\nIncludes additional Apify-specific fields." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 934 + ], + "title": "Properties" + } + ], + "id": 933, + "module": "storage_clients._apify._models", + "name": "ApifyKeyValueStoreMetadata", + "parsedDocstring": { + "text": "Extended key-value store metadata model for Apify platform.\n\nIncludes additional Apify-specific fields." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 936, + "module": "storage_clients._apify._models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 937, + "module": "storage_clients._apify._models", + "name": "lock_expires_at", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Response to prolong request lock calls." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 937, + 936 + ], + "title": "Properties" + } + ], + "id": 935, + "module": "storage_clients._apify._models", + "name": "ProlongRequestLockResponse", + "parsedDocstring": { + "text": "Response to prolong request lock calls." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 939, + "module": "storage_clients._apify._models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of requests that were requested from the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 940, + "module": "storage_clients._apify._models", + "name": "limit", + "parsedDocstring": { + "text": "The maximum number of requests that were requested from the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 44 + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='limit', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 941, + "module": "storage_clients._apify._models", + "name": "had_multiple_clients", + "parsedDocstring": { + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 47 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the queue was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 942, + "module": "storage_clients._apify._models", + "name": "queue_modified_at", + "parsedDocstring": { + "text": "The timestamp when the queue was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 50 + } + ], + "type": { + "name": "datetime", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The duration for which the returned requests are locked and cannot be processed by other clients." + } + ] + }, + "flags": {}, + "groups": [], + "id": 943, + "module": "storage_clients._apify._models", + "name": "lock_time", + "parsedDocstring": { + "text": "The duration for which the returned requests are locked and cannot be processed by other clients." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "type": { + "name": "Annotated[timedelta | None, Field(alias='lockSecs', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue contains any locked requests." + } + ] + }, + "flags": {}, + "groups": [], + "id": 944, + "module": "storage_clients._apify._models", + "name": "queue_has_locked_requests", + "parsedDocstring": { + "text": "Indicates whether the queue contains any locked requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "type": { + "name": "Annotated[bool | None, Field(alias='queueHasLockedRequests', default=False)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "bool" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The list of request objects retrieved from the beginning of the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 945, + "module": "storage_clients._apify._models", + "name": "items", + "parsedDocstring": { + "text": "The list of request objects retrieved from the beginning of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for request queue head.\n\nRepresents a collection of requests retrieved from the beginning of a queue,\nincluding metadata about the queue's state and lock information for the requests." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 941, + 945, + 940, + 943, + 939, + 944, + 942 + ], + "title": "Properties" + } + ], + "id": 938, + "module": "storage_clients._apify._models", + "name": "RequestQueueHead", + "parsedDocstring": { + "text": "Model for request queue head.\n\nRepresents a collection of requests retrieved from the beginning of a queue,\nincluding metadata about the queue's state and lock information for the requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 947, + "module": "storage_clients._apify._models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 948, + "module": "storage_clients._apify._models", + "name": "key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 949, + "module": "storage_clients._apify._models", + "name": "size", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 72 + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store key info.\n\nOnly internal structure." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 948, + 947, + 949 + ], + "title": "Properties" + } + ], + "id": 946, + "module": "storage_clients._apify._models", + "name": "KeyValueStoreKeyInfo", + "parsedDocstring": { + "text": "Model for a key-value store key info.\n\nOnly internal structure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 951, + "module": "storage_clients._apify._models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 81 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 952, + "module": "storage_clients._apify._models", + "name": "count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 83 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 953, + "module": "storage_clients._apify._models", + "name": "limit", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 84 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 954, + "module": "storage_clients._apify._models", + "name": "is_truncated", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 85 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 955, + "module": "storage_clients._apify._models", + "name": "items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 86 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreKeyInfo", + "target": "946" + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 956, + "module": "storage_clients._apify._models", + "name": "exclusive_start_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='exclusiveStartKey', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 957, + "module": "storage_clients._apify._models", + "name": "next_exclusive_start_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 88 + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='nextExclusiveStartKey', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for listing keys in the key-value store.\n\nOnly internal structure." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 952, + 956, + 954, + 955, + 953, + 951, + 957 + ], + "title": "Properties" + } + ], + "id": 950, + "module": "storage_clients._apify._models", + "name": "KeyValueStoreListKeysPage", + "parsedDocstring": { + "text": "Model for listing keys in the key-value store.\n\nOnly internal structure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 75 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Id of the request." + } + ] + }, + "flags": {}, + "groups": [], + "id": 959, + "module": "storage_clients._apify._models", + "name": "id", + "parsedDocstring": { + "text": "Id of the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 97 + } + ], + "type": { + "name": "str", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Whether the request was already handled." + } + ] + }, + "flags": {}, + "groups": [], + "id": 960, + "module": "storage_clients._apify._models", + "name": "was_already_handled", + "parsedDocstring": { + "text": "Whether the request was already handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100 + } + ], + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The hydrated request object (the original one)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 961, + "module": "storage_clients._apify._models", + "name": "hydrated", + "parsedDocstring": { + "text": "The hydrated request object (the original one)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103 + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The expiration time of the lock on the request." + } + ] + }, + "flags": {}, + "groups": [], + "id": 962, + "module": "storage_clients._apify._models", + "name": "lock_expires_at", + "parsedDocstring": { + "text": "The expiration time of the lock on the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106 + } + ], + "type": { + "name": "datetime | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pydantic model for cached request information.\n\nOnly internal structure." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 961, + 959, + 962, + 960 + ], + "title": "Properties" + } + ], + "id": 958, + "module": "storage_clients._apify._models", + "name": "CachedRequest", + "parsedDocstring": { + "text": "Pydantic model for cached request information.\n\nOnly internal structure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 964, + "module": "storage_clients._apify._models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 111 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "\"The number of request queue deletes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 965, + "module": "storage_clients._apify._models", + "name": "delete_count", + "parsedDocstring": { + "text": "\"The number of request queue deletes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of request queue head reads." + } + ] + }, + "flags": {}, + "groups": [], + "id": 966, + "module": "storage_clients._apify._models", + "name": "head_item_read_count", + "parsedDocstring": { + "text": "The number of request queue head reads." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 116 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of request queue reads." + } + ] + }, + "flags": {}, + "groups": [], + "id": 967, + "module": "storage_clients._apify._models", + "name": "read_count", + "parsedDocstring": { + "text": "The number of request queue reads." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 119 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage size in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 968, + "module": "storage_clients._apify._models", + "name": "storage_bytes", + "parsedDocstring": { + "text": "Storage size in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 122 + } + ], + "type": { + "name": "int", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of request queue writes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 969, + "module": "storage_clients._apify._models", + "name": "write_count", + "parsedDocstring": { + "text": "The number of request queue writes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 125 + } + ], + "type": { + "name": "int", + "type": "reference" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 965, + 966, + 964, + 967, + 968, + 969 + ], + "title": "Properties" + } + ], + "id": 963, + "module": "storage_clients._apify._models", + "name": "RequestQueueStats", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 110 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Additional statistics about the request queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 971, + "module": "storage_clients._apify._models", + "name": "stats", + "parsedDocstring": { + "text": "Additional statistics about the request queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 130 + } + ], + "type": { + "name": "RequestQueueStats", + "type": "reference", + "target": "963" + } + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 971 + ], + "title": "Properties" + } + ], + "id": 970, + "module": "storage_clients._apify._models", + "name": "ApifyRequestQueueMetadata", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 129 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 972, + "module": "storage_clients._apify._alias_resolving", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open storage by alias, creating it if necessary.\n\nThis function resolves storage aliases to their IDs, creating new unnamed storage if needed.\nThe alias mapping is stored in the default key-value store for persistence across Actor runs.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 973, + "module": "storage_clients._apify._alias_resolving", + "name": "open_by_alias", + "parsedDocstring": { + "text": "Open storage by alias, creating it if necessary.\n\nThis function resolves storage aliases to their IDs, creating new unnamed storage if needed.\nThe alias mapping is stored in the default key-value store for persistence across Actor runs.\n", + "args": { + "alias": "The alias name for the storage (e.g., '__default__', 'my-storage').", + "storage_type": "The type of storage to open.", + "collection_client": "The Apify API collection client for the storage type.", + "get_resource_client_by_id": "A callable that takes a storage ID and returns the resource client.", + "configuration": "Configuration object containing API credentials and settings.\n" + }, + "returns": "The storage client for the opened or created storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 63 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The storage client for the opened or created storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open storage by alias, creating it if necessary.\n\nThis function resolves storage aliases to their IDs, creating new unnamed storage if needed.\nThe alias mapping is stored in the default key-value store for persistence across Actor runs.\n" + } + ] + }, + "flags": {}, + "id": 974, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open_by_alias", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias name for the storage (e.g., '__default__', 'my-storage')." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 975, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The type of storage to open." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 976, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "Dataset" + }, + { + "type": "literal", + "value": "KeyValueStore" + }, + { + "type": "literal", + "value": "RequestQueue" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API collection client for the storage type." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 977, + "kind": 32768, + "kindString": "Parameter", + "name": "collection_client", + "type": { + "name": "( KeyValueStoreCollectionClientAsync | RequestQueueCollectionClientAsync | DatasetCollectionClientAsync )", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreCollectionClientAsync" + }, + { + "type": "reference", + "name": "RequestQueueCollectionClientAsync" + } + ] + }, + { + "type": "reference", + "name": "DatasetCollectionClientAsync" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A callable that takes a storage ID and returns the resource client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 978, + "kind": 32768, + "kindString": "Parameter", + "name": "get_resource_client_by_id", + "type": { + "name": "Callable", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "[str]" + }, + { + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreClientAsync" + }, + { + "type": "reference", + "name": "RequestQueueClientAsync" + } + ] + }, + { + "type": "reference", + "name": "DatasetClientAsync" + } + ] + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object containing API credentials and settings.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 979, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "KeyValueStoreClientAsync | RequestQueueClientAsync | DatasetClientAsync", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreClientAsync" + }, + { + "type": "reference", + "name": "RequestQueueClientAsync" + } + ] + }, + { + "type": "reference", + "name": "DatasetClientAsync" + } + ] + } + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 981, + "module": "storage_clients._apify._alias_resolving", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 134 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 982, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 983, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "Dataset" + }, + { + "type": "literal", + "value": "KeyValueStore" + }, + { + "type": "literal", + "value": "RequestQueue" + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 984, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 985, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Context manager to prevent race condition in alias creation." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 986, + "module": "storage_clients._apify._alias_resolving", + "name": "__aenter__", + "parsedDocstring": { + "text": "Context manager to prevent race condition in alias creation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 144 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Context manager to prevent race condition in alias creation." + } + ] + }, + "flags": {}, + "id": 987, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "AliasResolver", + "type": "reference", + "target": "980" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 988, + "module": "storage_clients._apify._alias_resolving", + "name": "__aexit__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 150 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 989, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 990, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 991, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 992, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get id of the aliased storage.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 993, + "module": "storage_clients._apify._alias_resolving", + "name": "resolve_id", + "parsedDocstring": { + "text": "Get id of the aliased storage.\n", + "returns": "Storage id if it exists, None otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 190 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Storage id if it exists, None otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get id of the aliased storage.\n" + } + ] + }, + "flags": {}, + "id": 994, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "resolve_id", + "parameters": [], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add alias and related storage id to the mapping in default kvs and local in-memory mapping." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 995, + "module": "storage_clients._apify._alias_resolving", + "name": "store_mapping", + "parsedDocstring": { + "text": "Add alias and related storage id to the mapping in default kvs and local in-memory mapping." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 210 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add alias and related storage id to the mapping in default kvs and local in-memory mapping." + } + ] + }, + "flags": {}, + "id": 996, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "store_mapping", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 997, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_id", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Class for handling aliases.\n\nThe purpose of this is class is to ensure that alias storages are created with correct id. This is achieved by using\ndefault kvs as a storage for global mapping of aliases to storage ids. Same mapping is also kept in memory to avoid\nunnecessary calls to API and also have limited support of alias storages when not running on Apify platform. When on\nApify platform, the storages created with alias are accessible by the same alias even after migration or reboot." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 986, + 988, + 981, + 993, + 995 + ], + "title": "Methods" + } + ], + "id": 980, + "module": "storage_clients._apify._alias_resolving", + "name": "AliasResolver", + "parsedDocstring": { + "text": "Class for handling aliases.\n\nThe purpose of this is class is to ensure that alias storages are created with correct id. This is achieved by using\ndefault kvs as a storage for global mapping of aliases to storage ids. Same mapping is also kept in memory to avoid\nunnecessary calls to API and also have limited support of alias storages when not running on Apify platform. When on\nApify platform, the storages created with alias are accessible by the same alias even after migration or reboot." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_alias_resolving.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an Apify API client for a specific storage type.\n\nThis function handles the creation and initialization of Apify storage clients (KVS, RQ, or Dataset).\nIt manages authentication, storage lookup/creation, and client instantiation.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 998, + "module": "storage_clients._apify._api_client_creation", + "name": "create_storage_api_client", + "parsedDocstring": { + "text": "Get an Apify API client for a specific storage type.\n\nThis function handles the creation and initialization of Apify storage clients (KVS, RQ, or Dataset).\nIt manages authentication, storage lookup/creation, and client instantiation.\n", + "args": { + "storage_type": "The type of storage to open.", + "configuration": "Configuration object containing API credentials.", + "id": "Storage ID to open. Mutually exclusive with name and alias.", + "name": "Storage name (global scope, persists across runs). Mutually exclusive with id and alias.", + "alias": "Storage alias (run scope, creates unnamed storage). Mutually exclusive with id and name.\n" + }, + "returns": "The storage client for the opened or created storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_api_client_creation.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The storage client for the opened or created storage." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get an Apify API client for a specific storage type.\n\nThis function handles the creation and initialization of Apify storage clients (KVS, RQ, or Dataset).\nIt manages authentication, storage lookup/creation, and client instantiation.\n" + } + ] + }, + "flags": {}, + "id": 999, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_storage_api_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The type of storage to open." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1000, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "Dataset" + }, + { + "type": "literal", + "value": "KeyValueStore" + }, + { + "type": "literal", + "value": "RequestQueue" + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object containing API credentials." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1001, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage ID to open. Mutually exclusive with name and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1002, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage name (global scope, persists across runs). Mutually exclusive with id and alias." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1003, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage alias (run scope, creates unnamed storage). Mutually exclusive with id and name.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1004, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStoreClientAsync | RequestQueueClientAsync | DatasetClientAsync", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreClientAsync" + }, + { + "type": "reference", + "name": "RequestQueueClientAsync" + } + ] + }, + { + "type": "reference", + "name": "DatasetClientAsync" + } + ] + } + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1005, + "module": "storage_clients._apify._request_queue_single_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new single-consumer request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='single')` instead of calling this directly.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1007, + "module": "storage_clients._apify._request_queue_single_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new single-consumer request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='single')` instead of calling this directly.\n", + "args": { + "api_client": "The Apify API client for request queue operations.", + "metadata": "Initial metadata for the request queue.", + "cache_size": "Maximum number of requests to cache locally." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new single-consumer request queue client instance.\n\nUse `ApifyRequestQueueClient.open(access='single')` instead of calling this directly.\n" + } + ] + }, + "flags": {}, + "id": 1008, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Apify API client for request queue operations." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1009, + "kind": 32768, + "kindString": "Parameter", + "name": "api_client", + "type": { + "name": "RequestQueueClientAsync", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initial metadata for the request queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1010, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata", + "type": { + "name": "RequestQueueMetadata", + "type": "reference" + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of requests to cache locally." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1011, + "kind": 32768, + "kindString": "Parameter", + "name": "cache_size", + "type": { + "name": "int", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1012, + "module": "storage_clients._apify._request_queue_single_client", + "name": "add_batch_of_requests", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 98 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1013, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_batch_of_requests", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1014, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Request" + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1015, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "AddRequestsResponse", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1016, + "module": "storage_clients._apify._request_queue_single_client", + "name": "get_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1017, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1018, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1019, + "module": "storage_clients._apify._request_queue_single_client", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 187 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1020, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1021, + "module": "storage_clients._apify._request_queue_single_client", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 199 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1022, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1023, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1024, + "module": "storage_clients._apify._request_queue_single_client", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 229 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1025, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1026, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1027, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1028, + "module": "storage_clients._apify._request_queue_single_client", + "name": "is_empty", + "parsedDocstring": { + "text": "Specific implementation of this method for the RQ single access mode." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 272 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specific implementation of this method for the RQ single access mode." + } + ] + }, + "flags": {}, + "id": 1029, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal request queue client implementation for single-consumer scenarios on the Apify platform.\n\nThis implementation minimizes API calls and resource usage by leveraging local caching and head estimation.\nIt is designed for scenarios where only one client consumes requests from the queue at a time, though multiple\nproducers may add requests concurrently.\n\n### Usage constraints\n\nThis client must operate within the following constraints to function correctly:\n\n- **Single consumer**: Only one client should be consuming (fetching) requests from the queue at any given time.\n- **Multiple producers allowed**: Multiple clients can add requests concurrently, but their forefront requests\nmay not be prioritized immediately since this client relies on local head estimation.\n- **Append-only queue**: Requests should only be added to the queue, never deleted by other clients.\nMarking requests as handled is permitted.\n- **No external modifications**: Other producers can add new requests but should not modify existing ones,\nas modifications won't be reflected in the local cache.\n\nIf these constraints are not met, the client may exhibit unpredictable behavior.\n\nThis class is used internally by `ApifyRequestQueueClient` when `access='single'` is specified.\n\nPublic methods are not individually documented as they implement the interface defined in `RequestQueueClient`." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1007, + 1012, + 1019, + 1016, + 1028, + 1021, + 1024 + ], + "title": "Methods" + } + ], + "id": 1006, + "module": "storage_clients._apify._request_queue_single_client", + "name": "ApifyRequestQueueSingleClient", + "parsedDocstring": { + "text": "Internal request queue client implementation for single-consumer scenarios on the Apify platform.\n\nThis implementation minimizes API calls and resource usage by leveraging local caching and head estimation.\nIt is designed for scenarios where only one client consumes requests from the queue at a time, though multiple\nproducers may add requests concurrently.\n\n### Usage constraints\n\nThis client must operate within the following constraints to function correctly:\n\n- **Single consumer**: Only one client should be consuming (fetching) requests from the queue at any given time.\n- **Multiple producers allowed**: Multiple clients can add requests concurrently, but their forefront requests\nmay not be prioritized immediately since this client relies on local head estimation.\n- **Append-only queue**: Requests should only be added to the queue, never deleted by other clients.\nMarking requests as handled is permitted.\n- **No external modifications**: Other producers can add new requests but should not modify existing ones,\nas modifications won't be reflected in the local cache.\n\nIf these constraints are not met, the client may exhibit unpredictable behavior.\n\nThis class is used internally by `ApifyRequestQueueClient` when `access='single'` is specified.\n\nPublic methods are not individually documented as they implement the interface defined in `RequestQueueClient`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1030, + "module": "storage_clients._apify._dataset_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyDatasetClient.open` class method to create a new instance." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1032, + "module": "storage_clients._apify._dataset_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `ApifyDatasetClient.open` class method to create a new instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `ApifyDatasetClient.open` class method to create a new instance." + } + ] + }, + "flags": {}, + "id": 1033, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1034, + "kind": 32768, + "kindString": "Parameter", + "name": "api_client", + "type": { + "name": "DatasetClientAsync", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1035, + "kind": 32768, + "kindString": "Parameter", + "name": "api_public_base_url", + "type": { + "name": "str", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1036, + "kind": 32768, + "kindString": "Parameter", + "name": "lock", + "type": { + "name": "asyncio.Lock", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + }, + "overwrites": { + "name": "DatasetClientPpeMixin.__init__", + "target": 1156, + "type": "reference" + } + } + ], + "overwrites": { + "name": "DatasetClientPpeMixin.__init__", + "target": 1156, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1037, + "module": "storage_clients._apify._dataset_client", + "name": "get_metadata", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1038, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open an Apify dataset client.\n\nThis method creates and initializes a new instance of the Apify dataset client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 1039, + "module": "storage_clients._apify._dataset_client", + "name": "open", + "parsedDocstring": { + "text": "Open an Apify dataset client.\n\nThis method creates and initializes a new instance of the Apify dataset client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n", + "args": { + "id": "The ID of the dataset to open. If provided, searches for existing dataset by ID.\nMutually exclusive with name and alias.", + "name": "The name of the dataset to open (global scope, persists across runs).\nMutually exclusive with id and alias.", + "alias": "The alias of the dataset to open (run scope, creates unnamed storage).\nMutually exclusive with id and name.", + "configuration": "The configuration object containing API credentials and settings. Must include a valid\n`token` and `api_base_url`. May also contain a `default_dataset_id` for fallback when neither\n`id`, `name`, nor `alias` is provided.\n" + }, + "returns": "An instance for the opened or created storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 75 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An instance for the opened or created storage client." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Open an Apify dataset client.\n\nThis method creates and initializes a new instance of the Apify dataset client.\nIt handles authentication, storage lookup/creation, and metadata retrieval.\n" + } + ] + }, + "flags": {}, + "id": 1040, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the dataset to open. If provided, searches for existing dataset by ID.\nMutually exclusive with name and alias." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1041, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the dataset to open (global scope, persists across runs).\nMutually exclusive with id and alias." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1042, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The alias of the dataset to open (run scope, creates unnamed storage).\nMutually exclusive with id and name." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1043, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The configuration object containing API credentials and settings. Must include a valid\n`token` and `api_base_url`. May also contain a `default_dataset_id` for fallback when neither\n`id`, `name`, nor `alias` is provided.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1044, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "ApifyDatasetClient", + "type": "reference", + "target": "1031" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1045, + "module": "storage_clients._apify._dataset_client", + "name": "purge", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 128 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1046, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1047, + "module": "storage_clients._apify._dataset_client", + "name": "drop", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 135 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1048, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1049, + "module": "storage_clients._apify._dataset_client", + "name": "push_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1050, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1051, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[Any] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "Any" + } + ] + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1052, + "module": "storage_clients._apify._dataset_client", + "name": "get_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 156 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1053, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1054, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1055, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1056, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1057, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1058, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1059, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1060, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1061, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1062, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1063, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1064, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1065, + "module": "storage_clients._apify._dataset_client", + "name": "iterate_items", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 187 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1066, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1067, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1068, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1069, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1070, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1071, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1072, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1073, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str" + } + ] + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1074, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference" + } + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1075, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict" + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An Apify platform implementation of the dataset client." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1032, + 1047, + 1052, + 1037, + 1065, + 1039, + 1045, + 1049 + ], + "title": "Methods" + } + ], + "id": 1031, + "module": "storage_clients._apify._dataset_client", + "name": "ApifyDatasetClient", + "parsedDocstring": { + "text": "An Apify platform implementation of the dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_apify/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "DatasetClientPpeMixin", + "target": "1155", + "type": "reference" + } + ] + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 1076, + "module": "storage_clients._file_system._key_value_store_client", + "name": "logger", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1078, + "module": "storage_clients._file_system._key_value_store_client", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 27 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1079, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1080, + "kind": 32768, + "kindString": "Parameter", + "name": "metadata", + "type": { + "name": "KeyValueStoreMetadata", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1081, + "kind": 32768, + "kindString": "Parameter", + "name": "path_to_kvs", + "type": { + "name": "Path", + "type": "reference" + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1082, + "kind": 32768, + "kindString": "Parameter", + "name": "lock", + "type": { + "name": "asyncio.Lock", + "type": "reference" + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + }, + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 1083, + "module": "storage_clients._file_system._key_value_store_client", + "name": "open", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1084, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1085, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1086, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1087, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1088, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration", + "type": "reference" + } + } + ], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purges the key-value store by deleting all its contents.\n\nIt deletes all files in the key-value store directory, except for the metadata file and\nthe input related file and its metadata." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1089, + "module": "storage_clients._file_system._key_value_store_client", + "name": "purge", + "parsedDocstring": { + "text": "Purges the key-value store by deleting all its contents.\n\nIt deletes all files in the key-value store directory, except for the metadata file and\nthe input related file and its metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purges the key-value store by deleting all its contents.\n\nIt deletes all files in the key-value store directory, except for the metadata file and\nthe input related file and its metadata." + } + ] + }, + "flags": {}, + "id": 1090, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1091, + "module": "storage_clients._file_system._key_value_store_client", + "name": "get_value", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1092, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1093, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference" + } + } + ], + "type": { + "name": "KeyValueStoreRecord | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "KeyValueStoreRecord" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify-specific implementation of the `FileSystemKeyValueStoreClient`.\n\nThe only difference is that it overrides the `purge` method to delete all files in the key-value store\ndirectory, except for the metadata file and the `INPUT.json` file." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1078, + 1091, + 1083, + 1089 + ], + "title": "Methods" + } + ], + "id": 1077, + "module": "storage_clients._file_system._key_value_store_client", + "name": "ApifyFileSystemKeyValueStoreClient", + "parsedDocstring": { + "text": "Apify-specific implementation of the `FileSystemKeyValueStoreClient`.\n\nThe only difference is that it overrides the `purge` method to delete all files in the key-value store\ndirectory, except for the metadata file and the `INPUT.json` file." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1095, + "module": "storage_clients._ppe_dataset_mixin", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1157, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + }, + "overwrites": { + "name": "DatasetClientPpeMixin.__init__", + "target": 1156, + "type": "reference" + } + } + ], + "overwrites": { + "name": "DatasetClientPpeMixin.__init__", + "target": 1156, + "type": "reference" + } + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + }, + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 1099, + "module": "storage_clients._file_system._dataset_client", + "name": "open", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1100, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1101, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1102, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1103, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 1104, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "Self", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1105, + "module": "storage_clients._file_system._dataset_client", + "name": "push_data", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 50 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1106, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1107, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[dict[str, Any]] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ] + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "reference", + "name": "Any" + } + ] + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify-specific implementation of the `FileSystemDatasetClient`.\n\nIt extends the functionality of `FileSystemDatasetClient` using `DatasetClientPpeMixin` and updates `push_data` to\nlimit and charge for the synthetic `apify-default-dataset-item` event. This is necessary for consistent behavior\nwhen locally testing the `PAY_PER_EVENT` pricing model." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1095, + 1099, + 1105 + ], + "title": "Methods" + } + ], + "id": 1094, + "module": "storage_clients._file_system._dataset_client", + "name": "ApifyFileSystemDatasetClient", + "parsedDocstring": { + "text": "Apify-specific implementation of the `FileSystemDatasetClient`.\n\nIt extends the functionality of `FileSystemDatasetClient` using `DatasetClientPpeMixin` and updates `push_data` to\nlimit and charge for the synthetic `apify-default-dataset-item` event. This is necessary for consistent behavior\nwhen locally testing the `PAY_PER_EVENT` pricing model." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedTypes": [ + { + "name": "DatasetClientPpeMixin", + "target": "1155", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1109, + "module": "storage_clients._file_system._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1110, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1111, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "231" + } + } + ], + "type": { + "name": "Hashable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1112, + "module": "storage_clients._file_system._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1113, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1114, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1115, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1116, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1117, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "231" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "FileSystemKeyValueStoreClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1118, + "module": "storage_clients._file_system._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 54 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1119, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1120, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1121, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1122, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1123, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "231" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "ApifyFileSystemDatasetClient", + "type": "reference", + "target": "1094" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Apify-specific implementation of the file system storage client.\n\nThe only difference is that it uses `ApifyFileSystemKeyValueStoreClient` for key-value stores,\nwhich overrides the `purge` method to delete all files in the key-value store directory\nexcept for the metadata file and the `INPUT.json` file." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1118, + 1112, + 1109 + ], + "title": "Methods" + } + ], + "id": 1108, + "module": "storage_clients._file_system._storage_client", + "name": "ApifyFileSystemStorageClient", + "parsedDocstring": { + "text": "Apify-specific implementation of the file system storage client.\n\nThe only difference is that it uses `ApifyFileSystemKeyValueStoreClient` for key-value stores,\nwhich overrides the `purge` method to delete all files in the key-value store directory\nexcept for the metadata file and the `INPUT.json` file." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1125, + "module": "storage_clients._smart_apify._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "cloud_storage_client": "Storage client used when an Actor is running on the Apify platform, or when\nexplicitly enabled via the `force_cloud` argument. Defaults to `ApifyStorageClient`.", + "local_storage_client": "Storage client used when an Actor is not running on the Apify platform and when\n`force_cloud` flag is not set. Defaults to `FileSystemStorageClient`." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 35 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 1126, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client used when an Actor is running on the Apify platform, or when\nexplicitly enabled via the `force_cloud` argument. Defaults to `ApifyStorageClient`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1127, + "kind": 32768, + "kindString": "Parameter", + "name": "cloud_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client used when an Actor is not running on the Apify platform and when\n`force_cloud` flag is not set. Defaults to `FileSystemStorageClient`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1128, + "kind": 32768, + "kindString": "Parameter", + "name": "local_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "None", + "type": "literal", + "value": null + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1129, + "module": "storage_clients._smart_apify._storage_client", + "name": "__str__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1130, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__str__", + "parameters": [], + "type": { + "name": "str", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1131, + "module": "storage_clients._smart_apify._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1132, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 1133, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration", + "type": "reference" + } + } + ], + "type": { + "name": "Hashable", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1134, + "module": "storage_clients._smart_apify._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 68 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1135, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1136, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1137, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1138, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1139, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "DatasetClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1140, + "module": "storage_clients._smart_apify._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 81 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1141, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1142, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1143, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1144, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1145, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 1146, + "module": "storage_clients._smart_apify._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 94 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1147, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1148, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1149, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1150, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str" + }, + { + "type": "literal", + "value": null + } + ] + } + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1151, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "CrawleeConfiguration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "CrawleeConfiguration" + }, + { + "type": "literal", + "value": null + } + ] + } + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference" + } + } + ] + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a suitable storage client based on the global configuration and the value of the force_cloud flag.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1152, + "module": "storage_clients._smart_apify._storage_client", + "name": "get_suitable_storage_client", + "parsedDocstring": { + "text": "Get a suitable storage client based on the global configuration and the value of the force_cloud flag.\n", + "args": { + "force_cloud": "If True, return `cloud_storage_client`." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a suitable storage client based on the global configuration and the value of the force_cloud flag.\n" + } + ] + }, + "flags": {}, + "id": 1153, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_suitable_storage_client", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, return `cloud_storage_client`." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 1154, + "kind": 32768, + "kindString": "Parameter", + "name": "force_cloud", + "type": { + "name": "bool", + "type": "reference" + } + } + ], + "type": { + "name": "StorageClient", + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client that automatically selects cloud or local storage client based on the environment.\n\nThis storage client provides access to datasets, key-value stores, and request queues by intelligently\ndelegating to either the cloud or local storage client based on the execution environment and configuration.\n\nWhen running on the Apify platform (which is detected via environment variables), this client automatically\nuses the `cloud_storage_client` to store storage data there. When running locally, it uses the\n`local_storage_client` to store storage data there. You can also force cloud storage usage from your\nlocal machine by using the `force_cloud` argument.\n\nThis storage client is designed to work specifically in `Actor` context and provides a seamless development\nexperience where the same code works both locally and on the Apify platform without any changes." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 1125, + 1129, + 1134, + 1140, + 1146, + 1131, + 1152 + ], + "title": "Methods" + } + ], + "id": 1124, + "module": "storage_clients._smart_apify._storage_client", + "name": "SmartApifyStorageClient", + "parsedDocstring": { + "text": "Storage client that automatically selects cloud or local storage client based on the environment.\n\nThis storage client provides access to datasets, key-value stores, and request queues by intelligently\ndelegating to either the cloud or local storage client based on the execution environment and configuration.\n\nWhen running on the Apify platform (which is detected via environment variables), this client automatically\nuses the `cloud_storage_client` to store storage data there. When running locally, it uses the\n`local_storage_client` to store storage data there. You can also force cloud storage usage from your\nlocal machine by using the `force_cloud` argument.\n\nThis storage client is designed to work specifically in `Actor` context and provides a seamless development\nexperience where the same code works both locally and on the Apify platform without any changes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + } + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 1156, + "module": "storage_clients._ppe_dataset_mixin", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_ppe_dataset_mixin.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "signatures": [ + { + "flags": {}, + "id": 1157, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": null + }, + "overwrites": { + "name": "DatasetClientPpeMixin.__init__", + "target": 1156, + "type": "reference" + } + } + ] + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A mixin for dataset clients to add support for PPE pricing model and tracking synthetic events." + } + ] + }, + "flags": {}, + "groups": [ + { + "children": [ + 1156 + ], + "title": "Methods" + } + ], + "id": 1155, + "module": "storage_clients._ppe_dataset_mixin", + "name": "DatasetClientPpeMixin", + "parsedDocstring": { + "text": "A mixin for dataset clients to add support for PPE pricing model and tracking synthetic events." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/apify/storage_clients/_ppe_dataset_mixin.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "type": { + "name": "Undefined", + "type": "reference" + }, + "extendedBy": [ + { + "name": "ApifyDatasetClient", + "target": "1031", + "type": "reference" + }, + { + "name": "ApifyFileSystemDatasetClient", + "target": "1094", + "type": "reference" + } + ] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4735, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L20" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Storage#id", + "parentId": 3540 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4736, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L25" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Storage#name", + "parentId": 3540 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4737, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 4738, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "989", + "ref": { + "id": 2183, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "992", + "ref": { + "id": 2186, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "994", + "ref": { + "id": 2188, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 4737, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 3543 + } + ], + "permalink": "/python/api/class/Storage#get_metadata", + "parentId": 3540 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4739, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 4740, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4741, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4742, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4743, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4744, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 1704, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 4745, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 2249, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "3540", + "ref": { + "id": 4734, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 4739, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 3545 + } + ], + "permalink": "/python/api/class/Storage#open", + "parentId": 3540 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4746, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 4747, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 3580, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 3552 + } + ], + "permalink": "/python/api/class/Storage#drop", + "parentId": 3540 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 4748, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L61" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 4749, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 4748, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3554 + } + ], + "permalink": "/python/api/class/Storage#purge", + "parentId": 3540 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storages." + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 4746, + 4737, + 4739, + 4748 + ], + "title": "Methods" + }, + { + "children": [ + 4735, + 4736 + ], + "title": "Properties" + } + ], + "id": 4734, + "module": "storages._base", + "name": "Storage", + "parsedDocstring": { + "text": "Base class for storages." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_base.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStore", + "target": "3479", + "type": "reference", + "ref": { + "id": 4673, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStore" + } + }, + { + "name": "Dataset", + "target": "3581", + "type": "reference", + "ref": { + "id": 4775, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Dataset" + } + }, + { + "name": "RequestQueue", + "target": "3667", + "type": "reference", + "ref": { + "id": 4861, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + } + ], + "permalink": "/python/api/class/Storage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8443, + "module": "storages._dataset", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L69" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `Dataset.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 8444, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8445, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1107", + "ref": { + "id": 5968, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "parentId": 3583 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8446, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3583 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8447, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3583 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3582 + } + ], + "permalink": "/python/api/class/Dataset#__init__", + "parentId": 3581 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8448, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L87" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 3541, + "type": "reference", + "ref": { + "id": 8402, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/Dataset#id", + "parentId": 3581 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8449, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 92, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L92" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 3542, + "type": "reference", + "ref": { + "id": 8403, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/Dataset#name", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8450, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 96, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L96" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 8405, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "989", + "ref": { + "id": 5850, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "992", + "ref": { + "id": 5853, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "994", + "ref": { + "id": 5855, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 8404, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 3589 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 8404, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/Dataset#get_metadata", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8452, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 101, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L101" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 8407, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8408, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8409, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8410, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8411, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 5371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8412, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 5916, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "3540", + "ref": { + "id": 8401, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 8406, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 3591 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 8406, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/Dataset#open", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8459, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 128, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L128" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 8414, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 7247, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 3598 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 3552, + "type": "reference", + "ref": { + "id": 8413, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/Dataset#drop", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 8461, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 134, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L134" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 8416, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 8415, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3600 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 8415, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/Dataset#purge", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8463, + "module": "storages._dataset", + "name": "push_data", + "parsedDocstring": { + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n", + "args": { + "data": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 137, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L137" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Store an object or an array of objects to the dataset.\n\nThe size of the data is limited by the receiving API and therefore `push_data()` will only\nallow objects whose JSON representation is smaller than 9MB. When an array is passed,\nnone of the included objects may be larger than 9MB, but the array itself may be of any size.\n" + } + ] + }, + "flags": {}, + "id": 8464, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "push_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A JSON serializable data structure to be stored in the dataset. The JSON representation\nof each item must be smaller than 9MB." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8465, + "kind": 32768, + "kindString": "Parameter", + "name": "data", + "type": { + "name": "list[dict[str, Any]] | dict[str, Any]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + "parentId": 3603 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3602 + } + ], + "permalink": "/python/api/class/Dataset#push_data", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8466, + "module": "storages._dataset", + "name": "get_data", + "parsedDocstring": { + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.", + "flatten": "Fields to be flattened in returned items.", + "view": "Specifies the dataset view to be used.\n" + }, + "returns": "An object with filtered, sorted, and paginated dataset items plus pagination details." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L150" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "An object with filtered, sorted, and paginated dataset items plus pagination details." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a paginated list of items from a dataset based on various filtering parameters.\n\nThis method provides the flexibility to filter, sort, and modify the appearance of dataset items\nwhen listed. Each parameter modifies the result set according to its purpose. The method also\nsupports pagination through 'offset' and 'limit' parameters.\n" + } + ] + }, + "flags": {}, + "id": 8467, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_data", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8468, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8469, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8470, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8471, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8472, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8473, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8474, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8475, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8476, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to be flattened in returned items." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8477, + "kind": 32768, + "kindString": "Parameter", + "name": "flatten", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the dataset view to be used.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8478, + "kind": 32768, + "kindString": "Parameter", + "name": "view", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3606 + } + ], + "type": { + "name": "DatasetItemsListPage", + "type": "reference", + "target": "1008", + "ref": { + "id": 5869, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 110 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetItemsListPage" + } + }, + "parentId": 3605 + } + ], + "permalink": "/python/api/class/Dataset#get_data", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8479, + "module": "storages._dataset", + "name": "iterate_items", + "parsedDocstring": { + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 201, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L201" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over items in the dataset according to specified filters and sorting.\n\nThis method allows for asynchronously iterating through dataset items while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 8480, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8481, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8482, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8483, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8484, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8485, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8486, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8487, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8488, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3619 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8489, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3619 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3618 + } + ], + "permalink": "/python/api/class/Dataset#iterate_items", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8490, + "module": "storages._dataset", + "name": "list_items", + "parsedDocstring": { + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n", + "args": { + "offset": "Skips the specified number of items at the start.", + "limit": "The maximum number of items to retrieve. Unlimited if None.", + "clean": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty.", + "desc": "Set to True to sort results in descending order.", + "fields": "Fields to include in each item. Sorts fields as specified if provided.", + "omit": "Fields to exclude from each item.", + "unwind": "Unwinds items by a specified array field, turning each element into a separate item.", + "skip_empty": "Excludes empty items from the results if True.", + "skip_hidden": "Excludes fields starting with '#' if True.\n" + }, + "returns": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of dictionary objects, each representing a dataset item after applying\nthe specified filters and transformations." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a list of all items from the dataset according to specified filters and sorting.\n\nThis method collects all dataset items into a list while applying various filters such as\nskipping empty items, hiding specific fields, and sorting. It supports pagination via `offset` and `limit`\nparameters, and can modify the appearance of dataset items using `fields`, `omit`, `unwind`, `skip_empty`, and\n`skip_hidden` parameters.\n" + } + ] + }, + "flags": {}, + "id": 8491, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_items", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Skips the specified number of items at the start." + } + ] + }, + "defaultValue": "0", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8492, + "kind": 32768, + "kindString": "Parameter", + "name": "offset", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of items to retrieve. Unlimited if None." + } + ] + }, + "defaultValue": "999_999_999_999", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8493, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return only non-empty items and excludes hidden fields. Shortcut for skip_hidden and skip_empty." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8494, + "kind": 32768, + "kindString": "Parameter", + "name": "clean", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set to True to sort results in descending order." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8495, + "kind": 32768, + "kindString": "Parameter", + "name": "desc", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to include in each item. Sorts fields as specified if provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8496, + "kind": 32768, + "kindString": "Parameter", + "name": "fields", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fields to exclude from each item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8497, + "kind": 32768, + "kindString": "Parameter", + "name": "omit", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unwinds items by a specified array field, turning each element into a separate item." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8498, + "kind": 32768, + "kindString": "Parameter", + "name": "unwind", + "type": { + "name": "list[str] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes empty items from the results if True." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8499, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_empty", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3630 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Excludes fields starting with '#' if True.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 8500, + "kind": 32768, + "kindString": "Parameter", + "name": "skip_hidden", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3630 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3629 + } + ], + "permalink": "/python/api/class/Dataset#list_items", + "parentId": 3581 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 8501, + "module": "storages._dataset", + "name": "export_to", + "parsedDocstring": { + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n", + "args": { + "key": "The key under which to save the data in the key-value store.", + "content_type": "The format in which to export the data.", + "to_kvs_id": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_name": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name.", + "to_kvs_storage_client": "Storage client to use for the key-value store.", + "to_kvs_configuration": "Configuration for the key-value store.", + "kwargs": "Additional parameters for the export operation, specific to the chosen content type." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 323, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L323" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 8502, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8503, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "defaultValue": "'json'", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8504, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + }, + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8505, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8506, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8507, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 5916, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8508, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 5371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3641 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Additional parameters for the export operation, specific to the chosen content type." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8509, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3641 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3640 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 8510, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8511, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8512, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "json" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8513, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3649 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8514, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3649 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8515, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 5916, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3649 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8516, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 5371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5187, + "module": "_types", + "name": "skipkeys", + "parsedDocstring": { + "text": "If True (default: False), dict keys that are not of a basic type (str, int, float, bool, None) will be skipped\ninstead of raising a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 753, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L753" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5188, + "module": "_types", + "name": "ensure_ascii", + "parsedDocstring": { + "text": "Determines if non-ASCII characters should be escaped in the output JSON string." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 757, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L757" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5189, + "module": "_types", + "name": "check_circular", + "parsedDocstring": { + "text": "If False (default: True), skips the circular reference check for container types. A circular reference will\nresult in a `RecursionError` or worse if unchecked." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 760, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L760" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5190, + "module": "_types", + "name": "allow_nan", + "parsedDocstring": { + "text": "If False (default: True), raises a ValueError for out-of-range float values (nan, inf, -inf) to strictly comply\nwith the JSON specification. If True, uses their JavaScript equivalents (NaN, Infinity, -Infinity)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 764, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L764" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Allows specifying a custom JSON encoder." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5191, + "module": "_types", + "name": "cls", + "parsedDocstring": { + "text": "Allows specifying a custom JSON encoder." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 768, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L768" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "json.JSONEncoder", + "target": 0, + "ref": {} + } + ], + "target": "3134", + "ref": { + "id": 7995, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5192, + "module": "_types", + "name": "indent", + "parsedDocstring": { + "text": "Specifies the number of spaces for indentation in the pretty-printed JSON output." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 771, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L771" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5193, + "module": "_types", + "name": "separators", + "parsedDocstring": { + "text": "A tuple of (item_separator, key_separator). The default is (', ', ': ') if indent is None and (',', ': ')\notherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 774, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L774" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "tuple", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5194, + "module": "_types", + "name": "default", + "parsedDocstring": { + "text": "A function called for objects that can't be serialized otherwise. It should return a JSON-encodable version\nof the object or raise a `TypeError`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 778, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L778" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Callable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5195, + "module": "_types", + "name": "sort_keys", + "parsedDocstring": { + "text": "Specifies whether the output JSON object should have keys sorted alphabetically." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 782, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L782" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3649 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3640 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Export the entire dataset into a specified file stored under a key in a key-value store.\n\nThis method consolidates all entries from a specified dataset into one file, which is then saved under a\ngiven key in a key-value store. The format of the exported file is determined by the `content_type` parameter.\nEither the dataset's ID or name should be specified, and similarly, either the target key-value store's ID or\nname should be used.\n" + } + ] + }, + "flags": {}, + "id": 8518, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "export_to", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key under which to save the data in the key-value store." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8519, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The format in which to export the data." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 8520, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": "csv" + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8521, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3657 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Name of the key-value store to save the exported file.\nSpecify only one of ID or name." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8522, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3657 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Storage client to use for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8523, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 5916, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3657 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration for the key-value store." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 8524, + "kind": 32768, + "kindString": "Parameter", + "name": "to_kvs_configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 5371, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Specifies a dialect to be used in CSV parsing and writing." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5197, + "module": "_types", + "name": "dialect", + "parsedDocstring": { + "text": "Specifies a dialect to be used in CSV parsing and writing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 789, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L789" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to separate fields. Defaults to ','." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5198, + "module": "_types", + "name": "delimiter", + "parsedDocstring": { + "text": "A one-character string used to separate fields. Defaults to ','." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 792, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L792" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5199, + "module": "_types", + "name": "doublequote", + "parsedDocstring": { + "text": "Controls how instances of `quotechar` inside a field should be quoted. When True, the character is doubled;\nwhen False, the `escapechar` is used as a prefix. Defaults to True." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 795, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L795" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5200, + "module": "_types", + "name": "escapechar", + "parsedDocstring": { + "text": "A one-character string used to escape the delimiter if `quoting` is set to `QUOTE_NONE` and the `quotechar`\nif `doublequote` is False. Defaults to None, disabling escaping." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 799, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L799" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5201, + "module": "_types", + "name": "lineterminator", + "parsedDocstring": { + "text": "The string used to terminate lines produced by the writer. Defaults to '\\r\\n'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 803, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L803" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5202, + "module": "_types", + "name": "quotechar", + "parsedDocstring": { + "text": "A one-character string used to quote fields containing special characters, like the delimiter or quotechar,\nor fields containing new-line characters. Defaults to '\"'." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 806, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L806" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5203, + "module": "_types", + "name": "quoting", + "parsedDocstring": { + "text": "Controls when quotes should be generated by the writer and recognized by the reader. Can take any of\nthe `QUOTE_*` constants, with a default of `QUOTE_MINIMAL`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 810, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L810" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5204, + "module": "_types", + "name": "skipinitialspace", + "parsedDocstring": { + "text": "When True, spaces immediately following the delimiter are ignored. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 814, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L814" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "When True, raises an exception on bad CSV input. Defaults to False." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 5205, + "module": "_types", + "name": "strict", + "parsedDocstring": { + "text": "When True, raises an exception on bad CSV input. Defaults to False." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 817, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_types.py#L817" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "bool", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3657 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3640 + } + ], + "permalink": "/python/api/class/Dataset#export_to", + "parentId": 3581 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 8443, + 8459, + 8501, + 8466, + 8450, + 8479, + 8490, + 8452, + 8461, + 8463 + ], + "title": "Methods" + }, + { + "children": [ + 8448, + 8449 + ], + "title": "Properties" + } + ], + "id": 8442, + "module": "storages._dataset", + "name": "Dataset", + "parsedDocstring": { + "text": "Dataset is a storage for managing structured tabular data.\n\nThe dataset class provides a high-level interface for storing and retrieving structured data\nwith consistent schema, similar to database tables or spreadsheets. It abstracts the underlying\nstorage implementation details, offering a consistent API regardless of where the data is\nphysically stored.\n\nDataset operates in an append-only mode, allowing new records to be added but not modified\nor deleted after creation. This makes it particularly suitable for storing crawling results\nand other data that should be immutable once collected.\n\nThe class provides methods for adding data, retrieving data with various filtering options,\nand exporting data to different formats. You can create a dataset using the `open` class method,\nspecifying either a name or ID. The underlying storage implementation is determined by\nthe configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import Dataset\n\n# Open a dataset\ndataset = await Dataset.open(name='my-dataset')\n\n# Add data\nawait dataset.push_data({'title': 'Example Product', 'price': 99.99})\n\n# Retrieve filtered data\nresults = await dataset.get_data(limit=10, desc=True)\n\n# Export data\nawait dataset.export_to('results.json', content_type='json')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_dataset.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_dataset.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "3540", + "type": "reference", + "ref": { + "id": 8401, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/Dataset", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12004, + "module": "storages._key_value_store", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `KeyValueStore.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 12005, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12006, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1079", + "ref": { + "id": 9603, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "parentId": 3481 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12007, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3481 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12008, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3481 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3480 + } + ], + "permalink": "/python/api/class/KeyValueStore#__init__", + "parentId": 3479 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12009, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L99" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 3541, + "type": "reference", + "ref": { + "id": 12065, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/KeyValueStore#id", + "parentId": 3479 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12010, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L104" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 3542, + "type": "reference", + "ref": { + "id": 12066, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/KeyValueStore#name", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12011, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 12068, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "989", + "ref": { + "id": 9513, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "992", + "ref": { + "id": 9516, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "994", + "ref": { + "id": 9518, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 12067, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 3487 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 12067, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/KeyValueStore#get_metadata", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12013, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 12070, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12071, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12072, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12073, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12074, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 9034, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 12075, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 9579, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "3540", + "ref": { + "id": 12064, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 12069, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 3489 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 12069, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/KeyValueStore#open", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12020, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 12077, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 10910, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 3496 + } + ], + "overwrites": { + "name": "Storage.drop", + "target": 3552, + "type": "reference", + "ref": { + "id": 12076, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 57 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#drop" + } + }, + "permalink": "/python/api/class/KeyValueStore#drop", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 12022, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L148" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 12079, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 12078, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3498 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 12078, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/KeyValueStore#purge", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12024, + "module": "storages._key_value_store", + "name": "get_value", + "parsedDocstring": { + "text": "Get a value from the KVS.\n", + "args": { + "key": "Key of the record to retrieve.", + "default_value": "Default value returned in case the record does not exist.\n" + }, + "returns": "The value associated with the given key. `default_value` is used in case the record does not exist." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 160, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L160" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12025, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12026, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3501 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12027, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3501 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3500 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12056, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12057, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3532 + } + ], + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3500 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12058, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12059, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3534 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12060, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T", + "type": "reference", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 3534 + } + ], + "type": { + "name": "T", + "type": "reference", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + "parentId": 3500 + }, + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The value associated with the given key. `default_value` is used in case the record does not exist." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12061, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to retrieve." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12062, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3537 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Default value returned in case the record does not exist.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12063, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3537 + } + ], + "type": { + "name": "T | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "T", + "target": "82", + "ref": { + "id": 8606, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#T" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3500 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_value", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12028, + "module": "storages._key_value_store", + "name": "set_value", + "parsedDocstring": { + "text": "Set a value in the KVS.\n", + "args": { + "key": "Key of the record to set.", + "value": "Value to set.", + "content_type": "The MIME content type string." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Set a value in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12029, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "set_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12030, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3505 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to set." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12031, + "kind": 32768, + "kindString": "Parameter", + "name": "value", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3505 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME content type string." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12032, + "kind": 32768, + "kindString": "Parameter", + "name": "content_type", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3505 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3504 + } + ], + "permalink": "/python/api/class/KeyValueStore#set_value", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12033, + "module": "storages._key_value_store", + "name": "delete_value", + "parsedDocstring": { + "text": "Delete a value from the KVS.\n", + "args": { + "key": "Key of the record to delete." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 188, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L188" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Delete a value from the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12034, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "delete_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to delete." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12035, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3510 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3509 + } + ], + "permalink": "/python/api/class/KeyValueStore#delete_value", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12036, + "module": "storages._key_value_store", + "name": "iterate_keys", + "parsedDocstring": { + "text": "Iterate over the existing keys in the KVS.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return. None means no limit.\n" + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 196, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L196" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Iterate over the existing keys in the KVS.\n" + } + ] + }, + "flags": {}, + "id": 12037, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "iterate_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12038, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3513 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return. None means no limit.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12039, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3513 + } + ], + "type": { + "name": "AsyncIterator", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1000", + "ref": { + "id": 9524, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3512 + } + ], + "permalink": "/python/api/class/KeyValueStore#iterate_keys", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12040, + "module": "storages._key_value_store", + "name": "list_keys", + "parsedDocstring": { + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n", + "args": { + "exclusive_start_key": "Key to start the iteration from.", + "limit": "Maximum number of keys to return.\n" + }, + "returns": "A list of keys in the KVS." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 216, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L216" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A list of keys in the KVS." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "List all the existing keys in the KVS.\n\nIt uses client's `iterate_keys` method to get the keys.\n" + } + ] + }, + "flags": {}, + "id": 12041, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "list_keys", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key to start the iteration from." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12042, + "kind": 32768, + "kindString": "Parameter", + "name": "exclusive_start_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3517 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of keys to return.\n" + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12043, + "kind": 32768, + "kindString": "Parameter", + "name": "limit", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3517 + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "KeyValueStoreRecordMetadata", + "target": "1000", + "ref": { + "id": 9524, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3516 + } + ], + "permalink": "/python/api/class/KeyValueStore#list_keys", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12044, + "module": "storages._key_value_store", + "name": "record_exists", + "parsedDocstring": { + "text": "Check if a record with the given key exists in the key-value store.\n", + "args": { + "key": "Key of the record to check for existence.\n" + }, + "returns": "True if a record with the given key exists, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 240, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L240" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if a record with the given key exists, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if a record with the given key exists in the key-value store.\n" + } + ] + }, + "flags": {}, + "id": 12045, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "record_exists", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record to check for existence.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12046, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3521 + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3520 + } + ], + "permalink": "/python/api/class/KeyValueStore#record_exists", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12047, + "module": "storages._key_value_store", + "name": "get_public_url", + "parsedDocstring": { + "text": "Get the public URL for the given key.\n", + "args": { + "key": "Key of the record for which URL is required.\n" + }, + "returns": "The public URL for the given key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 251, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L251" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The public URL for the given key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get the public URL for the given key.\n" + } + ] + }, + "flags": {}, + "id": 12048, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_public_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record for which URL is required.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12049, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3524 + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3523 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_public_url", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12050, + "module": "storages._key_value_store", + "name": "get_auto_saved_value", + "parsedDocstring": { + "text": "Get a value from KVS that will be automatically saved on changes.\n", + "args": { + "key": "Key of the record, to store the value.", + "default_value": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + }, + "returns": "Return the value of the key." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 262, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L262" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Return the value of the key." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Get a value from KVS that will be automatically saved on changes.\n" + } + ] + }, + "flags": {}, + "id": 12051, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_auto_saved_value", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key of the record, to store the value." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 12052, + "kind": 32768, + "kindString": "Parameter", + "name": "key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3527 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Value to be used if the record does not exist yet. Should be a dictionary.\n" + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 12053, + "kind": 32768, + "kindString": "Parameter", + "name": "default_value", + "type": { + "name": "dict[str, JsonSerializable] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3527 + } + ], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "JsonSerializable", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 3526 + } + ], + "permalink": "/python/api/class/KeyValueStore#get_auto_saved_value", + "parentId": 3479 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 12054, + "module": "storages._key_value_store", + "name": "persist_autosaved_values", + "parsedDocstring": { + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 299, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L299" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Force autosaved values to be saved without waiting for an event in Event Manager." + } + ] + }, + "flags": {}, + "id": 12055, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "persist_autosaved_values", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3530 + } + ], + "permalink": "/python/api/class/KeyValueStore#persist_autosaved_values", + "parentId": 3479 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 12004, + 12033, + 12020, + 12050, + 12011, + 12047, + 12024, + 12036, + 12040, + 12013, + 12054, + 12022, + 12044, + 12028 + ], + "title": "Methods" + }, + { + "children": [ + 12009, + 12010 + ], + "title": "Properties" + } + ], + "id": 12003, + "module": "storages._key_value_store", + "name": "KeyValueStore", + "parsedDocstring": { + "text": "Key-value store is a storage for reading and writing data records with unique key identifiers.\n\nThe key-value store class acts as a high-level interface for storing, retrieving, and managing data records\nidentified by unique string keys. It abstracts away the underlying storage implementation details,\nallowing you to work with the same API regardless of whether data is stored in memory, on disk,\nor in the cloud.\n\nEach data record is associated with a specific MIME content type, allowing storage of various\ndata formats such as JSON, text, images, HTML snapshots or any binary data. This class is\ncommonly used to store inputs, outputs, and other artifacts of crawler operations.\n\nYou can instantiate a key-value store using the `open` class method, which will create a store\nwith the specified name or id. The underlying storage implementation is determined by the configured\nstorage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import KeyValueStore\n\n# Open a named key-value store\nkvs = await KeyValueStore.open(name='my-store')\n\n# Store and retrieve data\nawait kvs.set_value('product-1234.json', [{'name': 'Smartphone', 'price': 799.99}])\nproduct = await kvs.get_value('product-1234')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_key_value_store.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_key_value_store.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "3540", + "type": "reference", + "ref": { + "id": 12064, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + } + ], + "permalink": "/python/api/class/KeyValueStore", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15747, + "module": "storages._request_queue", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n", + "args": { + "client": "An instance of a storage client.", + "id": "The unique identifier of the storage.", + "name": "The name of the storage, if available." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nPreferably use the `RequestQueue.open` constructor to create a new instance.\n" + } + ] + }, + "flags": {}, + "id": 15748, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An instance of a storage client." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15749, + "kind": 32768, + "kindString": "Parameter", + "name": "client", + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1030", + "ref": { + "id": 13109, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "parentId": 3669 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15750, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3669 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage, if available." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15751, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3669 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 3668 + } + ], + "permalink": "/python/api/class/RequestQueue#__init__", + "parentId": 3667 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage ID." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15752, + "module": "storages._base", + "name": "id", + "parsedDocstring": { + "text": "Get the storage ID." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L95" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "Storage.id", + "target": 3541, + "type": "reference", + "ref": { + "id": 15620, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#id" + } + }, + "permalink": "/python/api/class/RequestQueue#id", + "parentId": 3667 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage name." + } + ] + }, + "decorations": [ + { + "name": "property" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15753, + "module": "storages._base", + "name": "name", + "parsedDocstring": { + "text": "Get the storage name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L100" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "Storage.name", + "target": 3542, + "type": "reference", + "ref": { + "id": 15621, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/Storage#name" + } + }, + "permalink": "/python/api/class/RequestQueue#name", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15754, + "module": "storages._base", + "name": "get_metadata", + "parsedDocstring": { + "text": "Get the storage metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the storage metadata." + } + ] + }, + "flags": {}, + "id": 15623, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_metadata", + "parameters": [], + "type": { + "name": "DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "DatasetMetadata", + "target": "989", + "ref": { + "id": 13068, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "type": "reference", + "name": "KeyValueStoreMetadata", + "target": "992", + "ref": { + "id": 13071, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + } + ] + }, + { + "type": "reference", + "name": "RequestQueueMetadata", + "target": "994", + "ref": { + "id": 13073, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ] + }, + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 15622, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "parentId": 3675 + } + ], + "overwrites": { + "name": "Storage.get_metadata", + "target": 3543, + "type": "reference", + "ref": { + "id": 15622, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#get_metadata" + } + }, + "permalink": "/python/api/class/RequestQueue#get_metadata", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15756, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 108, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L108" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 14400, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 3933, + "type": "reference", + "ref": { + "id": 16012, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 3677 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 3933, + "type": "reference", + "ref": { + "id": 16012, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_handled_count", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15758, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 14402, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 3934, + "type": "reference", + "ref": { + "id": 16013, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 3679 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 3934, + "type": "reference", + "ref": { + "id": 16013, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestQueue#get_total_count", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + }, + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15760, + "module": "storages._base", + "name": "open", + "parsedDocstring": { + "text": "Open a storage, either restore existing or create a new one.\n", + "args": { + "id": "The storage ID.", + "name": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\").", + "alias": "The storage alias (run scope, creates unnamed storage).", + "configuration": "Configuration object used during the storage creation or restoration process.", + "storage_client": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 119, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L119" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Open a storage, either restore existing or create a new one.\n" + } + ] + }, + "flags": {}, + "id": 15625, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "open", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage ID." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15626, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage name (global scope, persists across runs). Name can only contain letters \"a\" through \"z\",\nthe digits \"0\" through \"9\", and the hyphen (\"-\") but only in the middle of the string\n(e.g. \"my-value-1\")." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15627, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The storage alias (run scope, creates unnamed storage)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15628, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration object used during the storage creation or restoration process." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15629, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 12589, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Underlying storage client to use. If not provided, the default global storage client\nfrom the service locator will be used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15630, + "kind": 32768, + "kindString": "Parameter", + "name": "storage_client", + "type": { + "name": "StorageClient | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "StorageClient", + "target": "1055", + "ref": { + "id": 13134, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3546 + } + ], + "type": { + "name": "Storage", + "type": "reference", + "target": "3540", + "ref": { + "id": 15619, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 15624, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "parentId": 3681 + } + ], + "overwrites": { + "name": "Storage.open", + "target": 3545, + "type": "reference", + "ref": { + "id": 15624, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#open" + } + }, + "permalink": "/python/api/class/RequestQueue#open", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15767, + "module": "storages._base", + "name": "drop", + "parsedDocstring": { + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 144, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L144" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Drop the storage, removing it from the underlying storage client and clearing the cache." + } + ] + }, + "flags": {}, + "id": 15632, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 14465, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 3688 + } + ], + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 14465, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "permalink": "/python/api/class/RequestQueue#drop", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15769, + "module": "storages._base", + "name": "purge", + "parsedDocstring": { + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 152, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L152" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Purge the storage, removing all items from the underlying storage client.\n\nThis method does not remove the storage itself, e.g. don't remove the metadata,\nbut clears all items within it." + } + ] + }, + "flags": {}, + "id": 15634, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "purge", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 15633, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "parentId": 3690 + } + ], + "overwrites": { + "name": "Storage.purge", + "target": 3554, + "type": "reference", + "ref": { + "id": 15633, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 61 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/Storage#purge" + } + }, + "permalink": "/python/api/class/RequestQueue#purge", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 15771, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 156, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L156" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 14468, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 14469, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 2389 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14470, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2389 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 13094, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 2388, + "type": "reference", + "ref": { + "id": 14467, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 3692 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 2388, + "type": "reference", + "ref": { + "id": 14467, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestQueue#add_request", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15775, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 180, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L180" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 14472, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 14473, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14474, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14475, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14476, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14477, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 14478, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2393 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 2392, + "type": "reference", + "ref": { + "id": 14471, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 3696 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 2392, + "type": "reference", + "ref": { + "id": 14471, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestQueue#add_requests", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15783, + "module": "storages._request_queue", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n", + "returns": "The next request to process, or `None` if there are no more pending requests." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 230, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L230" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The next request to process, or `None` if there are no more pending requests." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Return the next request in the queue to be processed.\n\nOnce you successfully finish processing of the request, you need to call `RequestQueue.mark_request_as_handled`\nto mark the request as handled in the queue. If there was some error in processing the request, call\n`RequestQueue.reclaim_request` instead, so that the queue will give the request to some other consumer\nin another call to the `fetch_next_request` method.\n\nNote that the `None` return value does not mean the queue processing finished, it means there are currently\nno pending requests. To check whether all requests in queue were finished, use `RequestQueue.is_finished`\ninstead.\n" + } + ] + }, + "flags": {}, + "id": 15784, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 3937, + "type": "reference", + "ref": { + "id": 16016, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 3704 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 3937, + "type": "reference", + "ref": { + "id": 16016, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestQueue#fetch_next_request", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15785, + "module": "storages._request_queue", + "name": "get_request", + "parsedDocstring": { + "text": "Retrieve a specific request from the queue by its ID.\n", + "args": { + "unique_key": "Unique key of the request to retrieve.\n" + }, + "returns": "The request with the specified ID, or `None` if no such request exists." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 247, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L247" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "The request with the specified ID, or `None` if no such request exists." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Retrieve a specific request from the queue by its ID.\n" + } + ] + }, + "flags": {}, + "id": 15786, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Unique key of the request to retrieve.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15787, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3707 + } + ], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 3706 + } + ], + "permalink": "/python/api/class/RequestQueue#get_request", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15788, + "module": "storages._request_queue", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n", + "args": { + "request": "The request to mark as handled.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 258, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L258" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after successful processing.\n\nThis method should be called after a request has been successfully processed.\nOnce marked as handled, the request will be removed from the queue and will\nnot be returned in subsequent calls to `fetch_next_request` method.\n" + } + ] + }, + "flags": {}, + "id": 15789, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to mark as handled.\n" + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15790, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3710 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 13094, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 3938, + "type": "reference", + "ref": { + "id": 16017, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 3709 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 3938, + "type": "reference", + "ref": { + "id": 16017, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestQueue#mark_request_as_handled", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15791, + "module": "storages._request_queue", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n", + "args": { + "request": "The request to return to the queue.", + "forefront": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + }, + "returns": "Information about the queue operation." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 273, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L273" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the queue operation." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Reclaim a failed request back to the queue for later processing.\n\nIf a request fails during processing, this method can be used to return it to the queue.\nThe request will be returned for processing again in a subsequent call\nto `RequestQueue.fetch_next_request`.\n" + } + ] + }, + "flags": {}, + "id": 15792, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request to return to the queue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 15793, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 12484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 3713 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If true, the request will be added to the beginning of the queue.\nOtherwise, it will be added to the end.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 15794, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 3713 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 13094, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 2400, + "type": "reference", + "ref": { + "id": 14479, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 3712 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 2400, + "type": "reference", + "ref": { + "id": 14479, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestQueue#reclaim_request", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15795, + "module": "storages._request_queue", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n", + "returns": "True if the request queue is empty, False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 295, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L295" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is empty, False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is empty.\n\nAn empty queue means that there are no requests currently in the queue, either pending or being processed.\nHowever, this does not necessarily mean that the crawling operation is finished, as there still might be\ntasks that could add additional requests to the queue.\n" + } + ] + }, + "flags": {}, + "id": 15796, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 3935, + "type": "reference", + "ref": { + "id": 16014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 3716 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 3935, + "type": "reference", + "ref": { + "id": 16014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestQueue#is_empty", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 15797, + "module": "storages._request_queue", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n", + "returns": "True if the request queue is finished (empty and no pending add operations), False otherwise." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 307, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L307" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "True if the request queue is finished (empty and no pending add operations), False otherwise." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Check if the request queue is finished.\n\nA finished queue means that all requests in the queue have been processed (the queue is empty) and there\nare no more tasks that could add additional requests to the queue. This is the definitive way to check\nif a crawling operation is complete.\n" + } + ] + }, + "flags": {}, + "id": 15798, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 3936, + "type": "reference", + "ref": { + "id": 16015, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 3718 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 3936, + "type": "reference", + "ref": { + "id": 16015, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestQueue#is_finished", + "parentId": 3667 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 16021, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 14413, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 14414, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2334 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2405", + "ref": { + "id": 14484, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 14412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 3942 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 14412, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestQueue#to_tandem", + "parentId": 3667 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storages')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 15747, + 15771, + 15775, + 15767, + 15783, + 15756, + 15754, + 15785, + 15758, + 15795, + 15797, + 15788, + 15760, + 15769, + 15791, + 16021 + ], + "title": "Methods" + }, + { + "children": [ + 15752, + 15753 + ], + "title": "Properties" + } + ], + "id": 15746, + "module": "storages._request_queue", + "name": "RequestQueue", + "parsedDocstring": { + "text": "Request queue is a storage for managing HTTP requests.\n\nThe request queue class serves as a high-level interface for organizing and managing HTTP requests\nduring web crawling. It provides methods for adding, retrieving, and manipulating requests throughout\nthe crawling lifecycle, abstracting away the underlying storage implementation details.\n\nRequest queue maintains the state of each URL to be crawled, tracking whether it has been processed,\nis currently being handled, or is waiting in the queue. Each URL in the queue is uniquely identified\nby a `unique_key` property, which prevents duplicate processing unless explicitly configured otherwise.\n\nThe class supports both breadth-first and depth-first crawling strategies through its `forefront` parameter\nwhen adding requests. It also provides mechanisms for error handling and request reclamation when\nprocessing fails.\n\nYou can open a request queue using the `open` class method, specifying either a name or ID to identify\nthe queue. The underlying storage implementation is determined by the configured storage client.\n\n### Usage\n\n```python\nfrom crawlee.storages import RequestQueue\n\n# Open a request queue\nrq = await RequestQueue.open(name='my-queue')\n\n# Add a request\nawait rq.add_request('https://example.com')\n\n# Process requests\nrequest = await rq.fetch_next_request()\nif request:\n try:\n # Process the request\n # ...\n await rq.mark_request_as_handled(request)\n except Exception:\n await rq.reclaim_request(request)\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storages/_request_queue.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "Storage", + "target": "3540", + "type": "reference", + "ref": { + "id": 15619, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Storage" + } + }, + { + "name": "RequestManager", + "target": "2385", + "type": "reference", + "ref": { + "id": 14464, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestQueue", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 17048, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 173, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L173" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#model_config", + "parentId": 1026 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + } + ] + }, + "flags": {}, + "groups": [], + "id": 17049, + "module": "storage_clients.models", + "name": "processed_requests", + "parsedDocstring": { + "text": "Successfully processed requests, including information about whether they were\nalready present in the queue and whether they had been handled previously." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 175, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L175" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 17036, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#processed_requests", + "parentId": 1026 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests that could not be processed, typically due to validation errors or other issues." + } + ] + }, + "flags": {}, + "groups": [], + "id": 17050, + "module": "storage_clients.models", + "name": "unprocessed_requests", + "parsedDocstring": { + "text": "Requests that could not be processed, typically due to validation errors or other issues." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 179, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L179" + } + ], + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "UnprocessedRequest", + "target": "1021", + "ref": { + "id": 17042, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 154 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/UnprocessedRequest" + } + } + ], + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse#unprocessed_requests", + "parentId": 1026 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 17048, + 17049, + 17050 + ], + "title": "Properties" + } + ], + "id": 17047, + "module": "storage_clients.models", + "name": "AddRequestsResponse", + "parsedDocstring": { + "text": "Model for a response to add requests to a queue.\n\nContains detailed information about the processing results when adding multiple requests\nto a queue. This includes which requests were successfully processed and which ones\nencountered issues during processing." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 165, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L165" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/AddRequestsResponse", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 18059, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 113, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L113" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#model_config", + "parentId": 1008 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of objects returned on this page." + } + ] + }, + "flags": {}, + "groups": [], + "id": 18060, + "module": "storage_clients.models", + "name": "count", + "parsedDocstring": { + "text": "The number of objects returned on this page." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 115, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L115" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#count", + "parentId": 1008 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The starting position of the first object returned, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 18061, + "module": "storage_clients.models", + "name": "offset", + "parsedDocstring": { + "text": "The starting position of the first object returned, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 118, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L118" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#offset", + "parentId": 1008 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum number of objects to return, as specified in the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 18062, + "module": "storage_clients.models", + "name": "limit", + "parsedDocstring": { + "text": "The maximum number of objects to return, as specified in the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 121, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L121" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#limit", + "parentId": 1008 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of objects that match the criteria of the API call." + } + ] + }, + "flags": {}, + "groups": [], + "id": 18063, + "module": "storage_clients.models", + "name": "total", + "parsedDocstring": { + "text": "The total number of objects that match the criteria of the API call." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 124, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L124" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#total", + "parentId": 1008 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates if the returned list is in descending order." + } + ] + }, + "flags": {}, + "groups": [], + "id": 18064, + "module": "storage_clients.models", + "name": "desc", + "parsedDocstring": { + "text": "Indicates if the returned list is in descending order." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 127, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L127" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage#desc", + "parentId": 1008 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a single page of dataset items returned from a collection list method." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 18060, + 18064, + 18062, + 18059, + 18061, + 18063 + ], + "title": "Properties" + } + ], + "id": 18058, + "module": "storage_clients.models", + "name": "DatasetItemsListPage", + "parsedDocstring": { + "text": "Model for a single page of dataset items returned from a collection list method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetItemsListPage", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 19054, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 983, + "type": "reference", + "ref": { + "id": 19047, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/DatasetMetadata#model_config", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of items in the dataset." + } + ] + }, + "flags": {}, + "groups": [], + "id": 19055, + "module": "storage_clients.models", + "name": "item_count", + "parsedDocstring": { + "text": "The number of items in the dataset." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L47" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/DatasetMetadata#item_count", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 22391, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 984, + "type": "reference", + "ref": { + "id": 19048, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/DatasetMetadata#id", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 22392, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 985, + "type": "reference", + "ref": { + "id": 19049, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/DatasetMetadata#name", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 22393, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 986, + "type": "reference", + "ref": { + "id": 19050, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#accessed_at", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 22394, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 987, + "type": "reference", + "ref": { + "id": 19051, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#created_at", + "parentId": 989 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 22395, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 988, + "type": "reference", + "ref": { + "id": 19052, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/DatasetMetadata#modified_at", + "parentId": 989 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a dataset metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 22393, + 22394, + 22391, + 19055, + 19054, + 22395, + 22392 + ], + "title": "Properties" + } + ], + "id": 19053, + "module": "storage_clients.models", + "name": "DatasetMetadata", + "parsedDocstring": { + "text": "Model for a dataset metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "982", + "type": "reference", + "ref": { + "id": 19046, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/DatasetMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 23388, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 983, + "type": "reference", + "ref": { + "id": 23378, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#model_config", + "parentId": 992 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 26727, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 984, + "type": "reference", + "ref": { + "id": 23379, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#id", + "parentId": 992 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 26728, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 985, + "type": "reference", + "ref": { + "id": 23380, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#name", + "parentId": 992 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 26729, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 986, + "type": "reference", + "ref": { + "id": 23381, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#accessed_at", + "parentId": 992 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 26730, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 987, + "type": "reference", + "ref": { + "id": 23382, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#created_at", + "parentId": 992 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 26731, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 988, + "type": "reference", + "ref": { + "id": 23383, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/KeyValueStoreMetadata#modified_at", + "parentId": 992 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 26729, + 26730, + 26727, + 23388, + 26731, + 26728 + ], + "title": "Properties" + } + ], + "id": 23387, + "module": "storage_clients.models", + "name": "KeyValueStoreMetadata", + "parsedDocstring": { + "text": "Model for a key-value store metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "982", + "type": "reference", + "ref": { + "id": 23377, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 27737, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L103" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "KeyValueStoreRecordMetadata.model_config", + "target": 1001, + "type": "reference", + "ref": { + "id": 27732, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 81 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#model_config" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#model_config", + "parentId": 1005 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The value of the record." + } + ] + }, + "flags": {}, + "groups": [], + "id": 27738, + "module": "storage_clients.models", + "name": "value", + "parsedDocstring": { + "text": "The value of the record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 105, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L105" + } + ], + "type": { + "name": "KvsValueType", + "type": "reference", + "target": "981", + "ref": { + "id": 27712, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#KvsValueType" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#value", + "parentId": 1005 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31055, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "Annotated[str, Field(alias='key')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.key", + "target": 1002, + "type": "reference", + "ref": { + "id": 27733, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 83 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#key" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#key", + "parentId": 1005 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31056, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "Annotated[str, Field(alias='contentType')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.content_type", + "target": 1003, + "type": "reference", + "ref": { + "id": 27734, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#content_type" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#content_type", + "parentId": 1005 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 31057, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "KeyValueStoreRecordMetadata.size", + "target": 1004, + "type": "reference", + "ref": { + "id": 27735, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 95 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata#size" + } + }, + "permalink": "/python/api/class/KeyValueStoreRecord#size", + "parentId": 1005 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 31056, + 31055, + 27737, + 31057, + 27738 + ], + "title": "Properties" + } + ], + "id": 27736, + "module": "storage_clients.models", + "name": "KeyValueStoreRecord", + "parsedDocstring": { + "text": "Model for a key-value store record." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L100" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "KeyValueStoreRecordMetadata", + "target": "1000", + "type": "reference", + "ref": { + "id": 27731, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecordMetadata" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecord", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 32058, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 81, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L81" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#model_config", + "parentId": 1000 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32059, + "module": "storage_clients.models", + "name": "key", + "parsedDocstring": { + "text": "The key of the record.\n\nA unique identifier for the record in the key-value store." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L83" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#key", + "parentId": 1000 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32060, + "module": "storage_clients.models", + "name": "content_type", + "parsedDocstring": { + "text": "The MIME type of the record.\n\nDescribe the format and type of data stored in the record, following the MIME specification." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 89, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L89" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#content_type", + "parentId": 1000 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The size of the record in bytes." + } + ] + }, + "flags": {}, + "groups": [], + "id": 32061, + "module": "storage_clients.models", + "name": "size", + "parsedDocstring": { + "text": "The size of the record in bytes." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 95, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L95" + } + ], + "type": { + "name": "Annotated[int | None, Field(alias='size', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/KeyValueStoreRecordMetadata#size", + "parentId": 1000 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a key-value store record metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 32060, + 32059, + 32058, + 32061 + ], + "title": "Properties" + } + ], + "id": 32057, + "module": "storage_clients.models", + "name": "KeyValueStoreRecordMetadata", + "parsedDocstring": { + "text": "Model for a key-value store record metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 78, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L78" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "KeyValueStoreRecord", + "target": "1005", + "type": "reference", + "ref": { + "id": 32062, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreRecord" + } + } + ], + "permalink": "/python/api/class/KeyValueStoreRecordMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 33078, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 143, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L143" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#model_config", + "parentId": 1015 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Internal representation of the request by the storage client. Only some clients use id." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33079, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "Internal representation of the request by the storage client. Only some clients use id." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L145" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='requestId', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/ProcessedRequest#id", + "parentId": 1015 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 33080, + "module": "storage_clients.models", + "name": "unique_key", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 148, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L148" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#unique_key", + "parentId": 1015 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 33081, + "module": "storage_clients.models", + "name": "was_already_present", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 149, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L149" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_present", + "parentId": 1015 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 33082, + "module": "storage_clients.models", + "name": "was_already_handled", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 150, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L150" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest#was_already_handled", + "parentId": 1015 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a processed request." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 33079, + 33078, + 33080, + 33082, + 33081 + ], + "title": "Properties" + } + ], + "id": 33077, + "module": "storage_clients.models", + "name": "ProcessedRequest", + "parsedDocstring": { + "text": "Represents a processed request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L140" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/ProcessedRequest", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 33488, + "module": "_request", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 169, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L169" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#model_config", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33489, + "module": "_request", + "name": "unique_key", + "parsedDocstring": { + "text": "A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing\nto the same URL.\n\nIf `unique_key` is not provided, then it is automatically generated by normalizing the URL.\nFor example, the URL of `HTTP://www.EXAMPLE.com/something/` will produce the `unique_key`\nof `http://www.example.com/something`.\n\nPass an arbitrary non-empty text value to the `unique_key` property to override the default behavior\nand specify which URLs shall be considered equal." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 171, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L171" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#unique_key", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33490, + "module": "_request", + "name": "url", + "parsedDocstring": { + "text": "The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters\nand fragments." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 183, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L183" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#url", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request method." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33491, + "module": "_request", + "name": "method", + "parsedDocstring": { + "text": "HTTP request method." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 187, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L187" + } + ], + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "83", + "ref": { + "id": 33165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "permalink": "/python/api/class/Request#method", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "HTTP request payload." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33492, + "module": "_request", + "name": "payload", + "parsedDocstring": { + "text": "HTTP request payload." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 190, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L190" + } + ], + "type": { + "name": "Annotated[ HttpPayload | None, BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v), PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v), Field(frozen=True), ]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "84", + "ref": { + "id": 33166, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#payload", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Number of times the request has been retried." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33493, + "module": "_request", + "name": "retry_count", + "parsedDocstring": { + "text": "Number of times the request has been retried." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 230, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L230" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#retry_count", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will not be retried in case of failure." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33494, + "module": "_request", + "name": "no_retry", + "parsedDocstring": { + "text": "If set to `True`, the request will not be retried in case of failure." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 233, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L233" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#no_retry", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33495, + "module": "_request", + "name": "loaded_url", + "parsedDocstring": { + "text": "URL of the web page that was loaded. This can differ from the original URL in case of redirects." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 236, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L236" + } + ], + "type": { + "name": "Annotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#loaded_url", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timestamp when the request was handled." + } + ] + }, + "flags": {}, + "groups": [], + "id": 33496, + "module": "_request", + "name": "handled_at", + "parsedDocstring": { + "text": "Timestamp when the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L239" + } + ], + "type": { + "name": "Annotated[datetime | None, Field(alias='handledAt')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "datetime", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#handled_at", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 33497, + "module": "_request", + "name": "from_url", + "parsedDocstring": { + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n", + "args": { + "url": "The URL of the request.", + "method": "The HTTP method of the request.", + "headers": "The HTTP headers of the request.", + "payload": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests.", + "label": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers).", + "session_id": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised.", + "unique_key": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical.", + "keep_url_fragment": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided.", + "use_extended_unique_key": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided.", + "always_enqueue": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.", + "enqueue_strategy": "The strategy that will be used for enqueuing the request.", + "max_retries": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`.", + "**kwargs": "Additional request properties." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 243, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L243" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new `Request` instance from a URL.\n\nThis is recommended constructor for creating new `Request` instances. It generates a `Request` object from\na given URL with additional options to customize HTTP method, payload, unique key, and other request\nproperties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL,\nmethod and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags.\n" + } + ] + }, + "flags": {}, + "id": 33498, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_url", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The URL of the request." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33499, + "kind": 32768, + "kindString": "Parameter", + "name": "url", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP method of the request." + } + ] + }, + "defaultValue": "'GET'", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33500, + "kind": 32768, + "kindString": "Parameter", + "name": "method", + "type": { + "name": "HttpMethod", + "type": "reference", + "target": "83", + "ref": { + "id": 33165, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 38 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpMethod" + } + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The HTTP headers of the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33501, + "kind": 32768, + "kindString": "Parameter", + "name": "headers", + "type": { + "name": "HttpHeaders | dict[str, str] | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpHeaders", + "target": "89", + "ref": { + "id": 33171, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 60 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpHeaders" + } + }, + { + "type": "reference", + "name": "dict", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33502, + "kind": 32768, + "kindString": "Parameter", + "name": "payload", + "type": { + "name": "HttpPayload | str | None", + "type": "union", + "types": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "HttpPayload", + "target": "84", + "ref": { + "id": 33166, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#HttpPayload" + } + }, + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ] + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A custom label to differentiate between request types. This is stored in `user_data`, and it is\nused for request routing (different requests go to different handlers)." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33503, + "kind": 32768, + "kindString": "Parameter", + "name": "label", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "ID of a specific `Session` to which the request will be strictly bound.\nIf the session becomes unavailable when the request is processed, a `RequestCollisionError` will be\nraised." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33504, + "kind": 32768, + "kindString": "Parameter", + "name": "session_id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A unique key identifying the request. If not provided, it is automatically computed based on\nthe URL and other parameters. Requests with the same `unique_key` are treated as identical." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33505, + "kind": 32768, + "kindString": "Parameter", + "name": "unique_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the URL fragment (e.g., ``section``) should be included in\nthe `unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33506, + "kind": 32768, + "kindString": "Parameter", + "name": "keep_url_fragment", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether to include the HTTP method, ID Session and payload in the\n`unique_key` computation. This is only relevant when `unique_key` is not provided." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33507, + "kind": 32768, + "kindString": "Parameter", + "name": "use_extended_unique_key", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If set to `True`, the request will be enqueued even if it is already present in the queue.\nUsing this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33508, + "kind": 32768, + "kindString": "Parameter", + "name": "always_enqueue", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that will be used for enqueuing the request." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33509, + "kind": 32768, + "kindString": "Parameter", + "name": "enqueue_strategy", + "type": { + "name": "EnqueueStrategy | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EnqueueStrategy", + "target": "86", + "ref": { + "id": 33168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 44 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of retries for this request. Allows to override the global `max_request_retries`\noption of `BasicCrawler`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33510, + "kind": 32768, + "kindString": "Parameter", + "name": "max_retries", + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 416 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33511, + "kind": 32768, + "kindString": "Parameter", + "name": "kwargs", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 416 + } + ], + "type": { + "name": "Self", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 415 + } + ], + "permalink": "/python/api/class/Request#from_url", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 33512, + "module": "_request", + "name": "get_query_param_from_url", + "parsedDocstring": { + "text": "Get the value of a specific query parameter from the URL." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 344, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L344" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the value of a specific query parameter from the URL." + } + ] + }, + "flags": {}, + "id": 33513, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_query_param_from_url", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33514, + "kind": 32768, + "kindString": "Parameter", + "name": "param", + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 431 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 33515, + "kind": 32768, + "kindString": "Parameter", + "name": "default", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 431 + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 430 + } + ], + "permalink": "/python/api/class/Request#get_query_param_from_url", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A string used to differentiate between arbitrary request types." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33516, + "module": "_request", + "name": "label", + "parsedDocstring": { + "text": "A string used to differentiate between arbitrary request types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 350, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L350" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#label", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The ID of the bound session, if there is any." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33517, + "module": "_request", + "name": "session_id", + "parsedDocstring": { + "text": "The ID of the bound session, if there is any." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 355, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L355" + } + ], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_id", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific configuration stored in the `user_data`." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33518, + "module": "_request", + "name": "crawlee_data", + "parsedDocstring": { + "text": "Crawlee-specific configuration stored in the `user_data`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 360, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L360" + } + ], + "type": { + "name": "CrawleeRequestData", + "type": "reference", + "target": "354", + "ref": { + "id": 33436, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CrawleeRequestData" + } + }, + "permalink": "/python/api/class/Request#crawlee_data", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The depth of the request in the crawl tree." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33519, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "The depth of the request in the crawl tree." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 369, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L369" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "crawl_depth" + } + ], + "flags": {}, + "groups": [], + "id": 33520, + "module": "_request", + "name": "crawl_depth", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 374, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L374" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33521, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "crawl_depth", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33522, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 439 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 438 + } + ], + "permalink": "/python/api/class/Request#crawl_depth", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific request handling state." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33523, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "Crawlee-specific request handling state." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 378, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L378" + } + ], + "type": { + "name": "RequestState", + "type": "reference", + "target": "345", + "ref": { + "id": 33427, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "permalink": "/python/api/class/Request#state", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "state" + } + ], + "flags": {}, + "groups": [], + "id": 33524, + "module": "_request", + "name": "state", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 383, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L383" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33525, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "state", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33526, + "kind": 32768, + "kindString": "Parameter", + "name": "new_state", + "type": { + "name": "RequestState", + "type": "reference", + "target": "345", + "ref": { + "id": 33427, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestState" + } + }, + "parentId": 443 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 442 + } + ], + "permalink": "/python/api/class/Request#state", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific limit on the number of retries of the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33527, + "module": "_request", + "name": "max_retries", + "parsedDocstring": { + "text": "Crawlee-specific limit on the number of retries of the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 387, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L387" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#max_retries", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Crawlee-specific number of finished session rotations for the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33528, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "Crawlee-specific number of finished session rotations for the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 392, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L392" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "session_rotation_count" + } + ], + "flags": {}, + "groups": [], + "id": 33529, + "module": "_request", + "name": "session_rotation_count", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 397, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L397" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33530, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "session_rotation_count", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33531, + "kind": 32768, + "kindString": "Parameter", + "name": "new_session_rotation_count", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 448 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 447 + } + ], + "permalink": "/python/api/class/Request#session_rotation_count", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The strategy that was used for enqueuing the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33532, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "The strategy that was used for enqueuing the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 401, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L401" + } + ], + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "86", + "ref": { + "id": 33168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 44 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "enqueue_strategy" + } + ], + "flags": {}, + "groups": [], + "id": 33533, + "module": "_request", + "name": "enqueue_strategy", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 406, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L406" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33534, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "enqueue_strategy", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33535, + "kind": 32768, + "kindString": "Parameter", + "name": "new_enqueue_strategy", + "type": { + "name": "EnqueueStrategy", + "type": "reference", + "target": "86", + "ref": { + "id": 33168, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 44 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EnqueueStrategy" + } + }, + "parentId": 452 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 451 + } + ], + "permalink": "/python/api/class/Request#enqueue_strategy", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The last proxy tier used to process the request." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33536, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "The last proxy tier used to process the request." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 410, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L410" + } + ], + "type": { + "name": "int | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "last_proxy_tier" + } + ], + "flags": {}, + "groups": [], + "id": 33537, + "module": "_request", + "name": "last_proxy_tier", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 415, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L415" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33538, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "last_proxy_tier", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33539, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 456 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 455 + } + ], + "permalink": "/python/api/class/Request#last_proxy_tier", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the request should be enqueued at the front of the queue." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33540, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "Indicate whether the request should be enqueued at the front of the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 419, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L419" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#forefront", + "parentId": 405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [ + { + "args": ".setter", + "name": "forefront" + } + ], + "flags": {}, + "groups": [], + "id": 33541, + "module": "_request", + "name": "forefront", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 424, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L424" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 33542, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "forefront", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 33543, + "kind": 32768, + "kindString": "Parameter", + "name": "new_value", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 460 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 459 + } + ], + "permalink": "/python/api/class/Request#forefront", + "parentId": 405 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the request was handled." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 33544, + "module": "_request", + "name": "was_already_handled", + "parsedDocstring": { + "text": "Indicates whether the request was handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 428, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L428" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/Request#was_already_handled", + "parentId": 405 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 33520, + 33533, + 33541, + 33497, + 33512, + 33537, + 33529, + 33524 + ], + "title": "Methods" + }, + { + "children": [ + 33519, + 33518, + 33532, + 33540, + 33496, + 33516, + 33536, + 33495, + 33527, + 33491, + 33488, + 33494, + 33492, + 33493, + 33517, + 33528, + 33523, + 33489, + 33490, + 33544 + ], + "title": "Properties" + } + ], + "id": 33487, + "module": "_request", + "name": "Request", + "parsedDocstring": { + "text": "Represents a request in the Crawlee framework, containing the necessary information for crawling operations.\n\nThe `Request` class is one of the core components in Crawlee, utilized by various components such as request\nproviders, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests,\nincluding the URL, HTTP method, headers, payload, and user data. The user data allows custom information\nto be stored and persisted throughout the request lifecycle, including its retries.\n\nKey functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used\nfor request deduplication, controlling retries, handling state management, and enabling configuration for session\nrotation and proxy handling.\n\nThe recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically\ngenerates a unique key and identifier based on the URL and request parameters.\n\n### Usage\n\n```python\nfrom crawlee import Request\n\nrequest = Request.from_url('https://crawlee.dev')\n```" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/_request.py#L145" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "RequestWithLock", + "target": "463", + "type": "reference", + "ref": { + "id": 33545, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 433 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestWithLock" + } + } + ], + "permalink": "/python/api/class/Request", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 34540, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageMetadata.model_config", + "target": 983, + "type": "reference", + "ref": { + "id": 34528, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#model_config" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#model_config", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + } + ] + }, + "flags": {}, + "groups": [], + "id": 34541, + "module": "storage_clients.models", + "name": "had_multiple_clients", + "parsedDocstring": { + "text": "Indicates whether the queue has been accessed by multiple clients (consumers)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L64" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#had_multiple_clients", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that have been handled from the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 34542, + "module": "storage_clients.models", + "name": "handled_request_count", + "parsedDocstring": { + "text": "The number of requests that have been handled from the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 67, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L67" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#handled_request_count", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests that are still pending in the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 34543, + "module": "storage_clients.models", + "name": "pending_request_count", + "parsedDocstring": { + "text": "The number of requests that are still pending in the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 70, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L70" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#pending_request_count", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The total number of requests that have been added to the queue." + } + ] + }, + "flags": {}, + "groups": [], + "id": 34544, + "module": "storage_clients.models", + "name": "total_request_count", + "parsedDocstring": { + "text": "The total number of requests that have been added to the queue." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L73" + } + ], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/RequestQueueMetadata#total_request_count", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37882, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "Annotated[str, Field(alias='id')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.id", + "target": 984, + "type": "reference", + "ref": { + "id": 34529, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#id" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#id", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37883, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.name", + "target": 985, + "type": "reference", + "ref": { + "id": 34530, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#name" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#name", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37884, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='accessedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.accessed_at", + "target": 986, + "type": "reference", + "ref": { + "id": 34531, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#accessed_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#accessed_at", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37885, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='createdAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.created_at", + "target": 987, + "type": "reference", + "ref": { + "id": 34532, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#created_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#created_at", + "parentId": 994 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 37886, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "Annotated[datetime, Field(alias='modifiedAt')]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageMetadata.modified_at", + "target": 988, + "type": "reference", + "ref": { + "id": 34533, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata#modified_at" + } + }, + "permalink": "/python/api/class/RequestQueueMetadata#modified_at", + "parentId": 994 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Model for a request queue metadata." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 37884, + 37885, + 34541, + 34542, + 37882, + 34540, + 37886, + 37883, + 34543, + 34544 + ], + "title": "Properties" + } + ], + "id": 34539, + "module": "storage_clients.models", + "name": "RequestQueueMetadata", + "parsedDocstring": { + "text": "Model for a request queue metadata." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageMetadata", + "target": "982", + "type": "reference", + "ref": { + "id": 34527, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageMetadata" + } + } + ], + "permalink": "/python/api/class/RequestQueueMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 38869, + "module": "storage_clients.models", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L23" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#model_config", + "parentId": 982 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The unique identifier of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 38870, + "module": "storage_clients.models", + "name": "id", + "parsedDocstring": { + "text": "The unique identifier of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 25, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L25" + } + ], + "type": { + "name": "str", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#id", + "parentId": 982 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The name of the storage." + } + ] + }, + "flags": {}, + "groups": [], + "id": 38871, + "module": "storage_clients.models", + "name": "name", + "parsedDocstring": { + "text": "The name of the storage." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 28, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L28" + } + ], + "type": { + "name": "Annotated[str | None, Field(alias='name', default=None)]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/StorageMetadata#name", + "parentId": 982 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last accessed." + } + ] + }, + "flags": {}, + "groups": [], + "id": 38872, + "module": "storage_clients.models", + "name": "accessed_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last accessed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L31" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#accessed_at", + "parentId": 982 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was created." + } + ] + }, + "flags": {}, + "groups": [], + "id": 38873, + "module": "storage_clients.models", + "name": "created_at", + "parsedDocstring": { + "text": "The timestamp when the storage was created." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L34" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#created_at", + "parentId": 982 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "The timestamp when the storage was last modified." + } + ] + }, + "flags": {}, + "groups": [], + "id": 38874, + "module": "storage_clients.models", + "name": "modified_at", + "parsedDocstring": { + "text": "The timestamp when the storage was last modified." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L37" + } + ], + "type": { + "name": "datetime", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/StorageMetadata#modified_at", + "parentId": 982 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + } + ] + }, + "decorations": [ + { + "args": "('Storage data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 38872, + 38873, + 38870, + 38869, + 38874, + 38871 + ], + "title": "Properties" + } + ], + "id": 38868, + "module": "storage_clients.models", + "name": "StorageMetadata", + "parsedDocstring": { + "text": "Represents the base model for storage metadata.\n\nIt contains common fields shared across all specific storage types." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/models.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "DatasetMetadata", + "target": "989", + "type": "reference", + "ref": { + "id": 38875, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetMetadata" + } + }, + { + "name": "KeyValueStoreMetadata", + "target": "992", + "type": "reference", + "ref": { + "id": 38878, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreMetadata" + } + }, + { + "name": "RequestQueueMetadata", + "target": "994", + "type": "reference", + "ref": { + "id": 38880, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueMetadata" + } + } + ], + "permalink": "/python/api/class/StorageMetadata", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 41532, + "module": "events._event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n", + "args": { + "persist_state_interval": "Interval between emitted `PersistState` events to maintain state persistence.", + "close_timeout": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 63, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L63" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n" + } + ] + }, + "flags": {}, + "id": 41533, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "defaultValue": "timedelta(minutes=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 41534, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2653 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 41535, + "kind": 32768, + "kindString": "Parameter", + "name": "close_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2653 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2652 + } + ], + "permalink": "/python/api/class/EventManager#__init__", + "parentId": 2651 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 41536, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventManager#active", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 41537, + "module": "events._event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the event manager upon entering the async context.\n" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L104" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the event manager upon entering the async context.\n" + } + ] + }, + "flags": {}, + "id": 41538, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "EventManager", + "type": "reference", + "target": "2651", + "ref": { + "id": 41531, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + }, + "parentId": 2657 + } + ], + "permalink": "/python/api/class/EventManager#__aenter__", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 41539, + "module": "events._event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 117, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L117" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nThis will stop listening for the events, and it will wait for all the event listeners to finish.\n" + } + ] + }, + "flags": {}, + "id": 41540, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 41541, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3134", + "ref": { + "id": 42014, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2660 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 41542, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2660 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 41543, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2660 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2659 + } + ], + "permalink": "/python/api/class/EventManager#__aexit__", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 41544, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L157" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41545, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41546, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 41617, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2665 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41547, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2665 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41559, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41560, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2679 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41561, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventPersistStateData", + "target": "2750", + "ref": { + "id": 41630, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2679 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41562, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41563, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2682 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41564, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventSystemInfoData", + "target": "2753", + "ref": { + "id": 41633, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2682 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41565, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41566, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2685 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41567, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventMigratingData", + "target": "2757", + "ref": { + "id": 41637, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2685 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41568, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41569, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2688 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41570, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventAbortingData", + "target": "2760", + "ref": { + "id": 41640, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2688 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41571, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41572, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2691 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41573, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventExitData", + "target": "2762", + "ref": { + "id": 41642, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2691 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41574, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41575, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2694 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41576, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "EventCrawlerStatusData", + "target": "2764", + "ref": { + "id": 41644, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2694 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 41577, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41578, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 41617, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2697 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41579, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener", + "type": "reference", + "typeArguments": [ + { + "type": "literal", + "value": {} + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + "parentId": 2697 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2664 + } + ], + "permalink": "/python/api/class/EventManager#on", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 41548, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 207, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L207" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 41549, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41550, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 41617, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2669 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 41551, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "EventListener", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": "2771", + "ref": { + "id": 41651, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 112 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventListener" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2669 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2668 + } + ], + "permalink": "/python/api/class/EventManager#off", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 41552, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L239" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41553, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41554, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 41617, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2673 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41555, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2768", + "ref": { + "id": 41648, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2673 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41580, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41581, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.PERSIST_STATE", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2700 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41582, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2750", + "ref": { + "id": 41630, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2700 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41583, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41584, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.SYSTEM_INFO", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2703 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41585, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2753", + "ref": { + "id": 41633, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2703 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41586, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41587, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.MIGRATING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2706 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41588, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2757", + "ref": { + "id": 41637, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2706 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41589, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41590, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.ABORTING", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2709 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41591, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2760", + "ref": { + "id": 41640, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2709 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41592, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41593, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.EXIT", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2712 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41594, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2762", + "ref": { + "id": 41642, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2712 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41595, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41596, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "Event.CRAWLER_STATUS", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2715 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41597, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2764", + "ref": { + "id": 41644, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2715 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 41598, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41599, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 41617, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2718 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 41600, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2718 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2672 + } + ], + "permalink": "/python/api/class/EventManager#emit", + "parentId": 2651 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 41556, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 41557, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 41558, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2677 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2676 + } + ], + "permalink": "/python/api/class/EventManager#wait_for_all_listeners_to_complete", + "parentId": 2651 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 41537, + 41539, + 41532, + 41552, + 41548, + 41544, + 41556 + ], + "title": "Methods" + }, + { + "children": [ + 41536 + ], + "title": "Properties" + } + ], + "id": 41531, + "module": "events._event_manager", + "name": "EventManager", + "parsedDocstring": { + "text": "Manage events and their listeners, enabling registration, emission, and execution control.\n\nIt allows for registering event listeners, emitting events, and ensuring all listeners complete their execution.\nBuilt on top of `pyee.asyncio.AsyncIOEventEmitter`. It implements additional features such as waiting for all\nlisteners to complete and emitting `PersistState` events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L55" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "LocalEventManager", + "target": "2722", + "type": "reference", + "ref": { + "id": 41602, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + } + ], + "permalink": "/python/api/class/EventManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44737, + "module": "events._local_event_manager", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n", + "args": { + "system_info_interval": "Interval at which `SystemInfo` events are emitted.", + "event_manager_options": "Additional options for the parent class." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 34, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_local_event_manager.py#L34" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance.\n\nIn most cases, you should use the `from_config` constructor to create a new instance based on\nthe provided configuration.\n" + } + ] + }, + "flags": {}, + "id": 44738, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval at which `SystemInfo` events are emitted." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 44739, + "kind": 32768, + "kindString": "Parameter", + "name": "system_info_interval", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2724 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Interval between emitted `PersistState` events to maintain state persistence." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 44663, + "module": "events._event_manager", + "name": "persist_state_interval", + "parsedDocstring": { + "text": "Interval between emitted `PersistState` events to maintain state persistence." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 47, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L47" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2724 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + } + ] + }, + "flags": { + "keyword-only": true, + "optional": true + }, + "groups": [], + "id": 44664, + "module": "events._event_manager", + "name": "close_timeout", + "parsedDocstring": { + "text": "Optional timeout for canceling pending event listeners if they exceed this duration." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 50, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L50" + } + ], + "type": { + "name": "NotRequired", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2724 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__init__", + "target": 2652, + "type": "reference", + "ref": { + "id": 44666, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "parentId": 2723 + } + ], + "overwrites": { + "name": "EventManager.__init__", + "target": 2652, + "type": "reference", + "ref": { + "id": 44666, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 63 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__init__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__init__", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "decorations": [ + { + "name": "classmethod" + } + ], + "flags": {}, + "groups": [], + "id": 44741, + "module": "events._local_event_manager", + "name": "from_config", + "parsedDocstring": { + "text": "Initialize a new instance based on the provided `Configuration`.\n", + "args": { + "config": "The `Configuration` instance. Uses the global (default) one if not provided." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_local_event_manager.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize a new instance based on the provided `Configuration`.\n" + } + ] + }, + "flags": {}, + "id": 44742, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "from_config", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The `Configuration` instance. Uses the global (default) one if not provided." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 44743, + "kind": 32768, + "kindString": "Parameter", + "name": "config", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 42524, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2728 + } + ], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2722", + "ref": { + "id": 44736, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "parentId": 2727 + } + ], + "permalink": "/python/api/class/LocalEventManager#from_config", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44744, + "module": "events._local_event_manager", + "name": "__aenter__", + "parsedDocstring": { + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 72, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_local_event_manager.py#L72" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the local event manager upon entering the async context.\n\nIt starts emitting system info events at regular intervals." + } + ] + }, + "flags": {}, + "id": 44745, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "LocalEventManager", + "type": "reference", + "target": "2722", + "ref": { + "id": 44736, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/LocalEventManager" + } + }, + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2657, + "type": "reference", + "ref": { + "id": 44671, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "parentId": 2730 + } + ], + "overwrites": { + "name": "EventManager.__aenter__", + "target": 2657, + "type": "reference", + "ref": { + "id": 44671, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 104 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aenter__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aenter__", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 44746, + "module": "events._local_event_manager", + "name": "__aexit__", + "parsedDocstring": { + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 81, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_local_event_manager.py#L81" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the local event manager upon exiting the async context.\n\nIt stops emitting system info events and closes the event manager." + } + ] + }, + "flags": {}, + "id": 44747, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 44748, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3134", + "ref": { + "id": 45148, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2733 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 44749, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2733 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 44750, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2733 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2659, + "type": "reference", + "ref": { + "id": 44673, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 117 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "parentId": 2732 + } + ], + "overwrites": { + "name": "EventManager.__aexit__", + "target": 2659, + "type": "reference", + "ref": { + "id": 44673, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 117 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#__aexit__" + } + }, + "permalink": "/python/api/class/LocalEventManager#__aexit__", + "parentId": 2722 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Indicate whether the context is active." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 45941, + "module": "events._event_manager", + "name": "active", + "parsedDocstring": { + "text": "Indicate whether the context is active." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L100" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.active", + "target": 2656, + "type": "reference", + "ref": { + "id": 44670, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/EventManager#active" + } + }, + "permalink": "/python/api/class/LocalEventManager#active", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 45942, + "module": "events._event_manager", + "name": "on", + "parsedDocstring": { + "text": "Register an event listener for a specific event.\n", + "args": { + "event": "The event for which to listen to.", + "listener": "The function (sync or async) which is to be called when the event is emitted." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L157" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44679, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44680, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 44751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2665 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44681, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2665 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44693, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44694, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2679 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44695, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventPersistStateData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2679 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44696, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44697, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2682 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44698, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventSystemInfoData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2682 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44699, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44700, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2685 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44701, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventMigratingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2685 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44702, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44703, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2688 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44704, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventAbortingData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2688 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44705, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44706, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2691 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44707, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventExitData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2691 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44708, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44709, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2694 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44710, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[EventCrawlerStatusData]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2694 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Register an event listener for a specific event.\n" + } + ] + }, + "flags": {}, + "id": 44711, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "on", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event for which to listen to." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44712, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 44751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2697 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The function (sync or async) which is to be called when the event is emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44713, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[None]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2697 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "parentId": 3928 + } + ], + "inheritedFrom": { + "name": "EventManager.on", + "target": 2664, + "type": "reference", + "ref": { + "id": 44678, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 157 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#on" + } + }, + "permalink": "/python/api/class/LocalEventManager#on", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 45943, + "module": "events._event_manager", + "name": "off", + "parsedDocstring": { + "text": "Remove a specific listener or all listeners for an event.\n", + "args": { + "event": "The Actor event for which to remove listeners.", + "listener": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 207, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L207" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove a specific listener or all listeners for an event.\n" + } + ] + }, + "flags": {}, + "id": 44683, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "off", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The Actor event for which to remove listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44684, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 44751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2669 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The listener which is supposed to be removed. If not passed, all listeners of this event\nare removed." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 44685, + "kind": 32768, + "kindString": "Parameter", + "name": "listener", + "type": { + "name": "EventListener[Any] | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2669 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.off", + "target": 2668, + "type": "reference", + "ref": { + "id": 44682, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 207 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "parentId": 3929 + } + ], + "inheritedFrom": { + "name": "EventManager.off", + "target": 2668, + "type": "reference", + "ref": { + "id": 44682, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 207 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#off" + } + }, + "permalink": "/python/api/class/LocalEventManager#off", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 45944, + "module": "events._event_manager", + "name": "emit", + "parsedDocstring": { + "text": "Emit an event with the associated data to all registered listeners.\n", + "args": { + "event": "The event which will be emitted.", + "event_data": "The data which will be passed to the event listeners." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L239" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44687, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44688, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 44751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2673 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44689, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventData", + "type": "reference", + "target": "2768", + "ref": { + "id": 44782, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#EventData" + } + }, + "parentId": 2673 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44714, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44715, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.PERSIST_STATE]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2700 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44716, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventPersistStateData", + "type": "reference", + "target": "2750", + "ref": { + "id": 44764, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventPersistStateData" + } + }, + "parentId": 2700 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44717, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44718, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.SYSTEM_INFO]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2703 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44719, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventSystemInfoData", + "type": "reference", + "target": "2753", + "ref": { + "id": 44767, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventSystemInfoData" + } + }, + "parentId": 2703 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44720, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44721, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.MIGRATING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2706 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44722, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventMigratingData", + "type": "reference", + "target": "2757", + "ref": { + "id": 44771, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventMigratingData" + } + }, + "parentId": 2706 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44723, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44724, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.ABORTING]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2709 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44725, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventAbortingData", + "type": "reference", + "target": "2760", + "ref": { + "id": 44774, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventAbortingData" + } + }, + "parentId": 2709 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44726, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44727, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.EXIT]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2712 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44728, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventExitData", + "type": "reference", + "target": "2762", + "ref": { + "id": 44776, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 80 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventExitData" + } + }, + "parentId": 2712 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44729, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44730, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Literal[Event.CRAWLER_STATUS]", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2715 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44731, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "EventCrawlerStatusData", + "type": "reference", + "target": "2764", + "ref": { + "id": 44778, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 87 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventCrawlerStatusData" + } + }, + "parentId": 2715 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Emit an event with the associated data to all registered listeners.\n" + } + ] + }, + "flags": {}, + "id": 44732, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "emit", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The event which will be emitted." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44733, + "kind": 32768, + "kindString": "Parameter", + "name": "event", + "type": { + "name": "Event", + "type": "reference", + "target": "2737", + "ref": { + "id": 44751, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 8, + "permalink": "https://crawlee.dev/python/api/enum/Event" + } + }, + "parentId": 2718 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The data which will be passed to the event listeners." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": true + }, + "id": 44734, + "kind": 32768, + "kindString": "Parameter", + "name": "event_data", + "type": { + "name": "Any", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2718 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "parentId": 3930 + } + ], + "inheritedFrom": { + "name": "EventManager.emit", + "target": 2672, + "type": "reference", + "ref": { + "id": 44686, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 239 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#emit" + } + }, + "permalink": "/python/api/class/LocalEventManager#emit", + "parentId": 2722 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "decorations": [ + { + "name": "ensure_context" + } + ], + "flags": {}, + "groups": [], + "id": 45945, + "module": "events._event_manager", + "name": "wait_for_all_listeners_to_complete", + "parsedDocstring": { + "text": "Wait for all currently executing event listeners to complete.\n", + "args": { + "timeout": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_event_manager.py#L249" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Wait for all currently executing event listeners to complete.\n" + } + ] + }, + "flags": {}, + "id": 44691, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "wait_for_all_listeners_to_complete", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The maximum time to wait for the event listeners to finish. If they do not complete within\nthe specified timeout, they will be canceled." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 44692, + "kind": 32768, + "kindString": "Parameter", + "name": "timeout", + "type": { + "name": "timedelta | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2677 + } + ], + "type": { + "name": "None", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2676, + "type": "reference", + "ref": { + "id": 44690, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "parentId": 3931 + } + ], + "inheritedFrom": { + "name": "EventManager.wait_for_all_listeners_to_complete", + "target": 2676, + "type": "reference", + "ref": { + "id": 44690, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 249 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/EventManager#wait_for_all_listeners_to_complete" + } + }, + "permalink": "/python/api/class/LocalEventManager#wait_for_all_listeners_to_complete", + "parentId": 2722 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + } + ] + }, + "decorations": [ + { + "args": "('Event managers')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 44744, + 44746, + 44737, + 45944, + 44741, + 45943, + 45942, + 45945 + ], + "title": "Methods" + }, + { + "children": [ + 45941 + ], + "title": "Properties" + } + ], + "id": 44736, + "module": "events._local_event_manager", + "name": "LocalEventManager", + "parsedDocstring": { + "text": "Event manager for local environments.\n\nIt extends the `EventManager` to emit `SystemInfo` events at regular intervals. The `LocalEventManager`\nis intended to be used in local environments, where the system metrics are required managing the `Snapshotter`\nand `AutoscaledPool`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_local_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_local_event_manager.py#L26" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "EventManager", + "target": "2651", + "type": "reference", + "ref": { + "id": 44665, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_event_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/EventManager" + } + } + ], + "permalink": "/python/api/class/LocalEventManager", + "categories": [] + }, + { + "kind": 8, + "kindString": "Enumeration", + "children": [ + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48683, + "module": "events._types", + "name": "PERSIST_STATE", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L19" + } + ], + "type": { + "type": "literal", + "value": "'persistState'" + }, + "permalink": "/python/api/enum/Event#PERSIST_STATE", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48684, + "module": "events._types", + "name": "SYSTEM_INFO", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L20" + } + ], + "type": { + "type": "literal", + "value": "'systemInfo'" + }, + "permalink": "/python/api/enum/Event#SYSTEM_INFO", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48685, + "module": "events._types", + "name": "MIGRATING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 21, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L21" + } + ], + "type": { + "type": "literal", + "value": "'migrating'" + }, + "permalink": "/python/api/enum/Event#MIGRATING", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48686, + "module": "events._types", + "name": "ABORTING", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L22" + } + ], + "type": { + "type": "literal", + "value": "'aborting'" + }, + "permalink": "/python/api/enum/Event#ABORTING", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48687, + "module": "events._types", + "name": "EXIT", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 23, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L23" + } + ], + "type": { + "type": "literal", + "value": "'exit'" + }, + "permalink": "/python/api/enum/Event#EXIT", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48688, + "module": "events._types", + "name": "SESSION_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L26" + } + ], + "type": { + "type": "literal", + "value": "'sessionRetired'" + }, + "permalink": "/python/api/enum/Event#SESSION_RETIRED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48689, + "module": "events._types", + "name": "BROWSER_LAUNCHED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L29" + } + ], + "type": { + "type": "literal", + "value": "'browserLaunched'" + }, + "permalink": "/python/api/enum/Event#BROWSER_LAUNCHED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48690, + "module": "events._types", + "name": "BROWSER_RETIRED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 30, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L30" + } + ], + "type": { + "type": "literal", + "value": "'browserRetired'" + }, + "permalink": "/python/api/enum/Event#BROWSER_RETIRED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48691, + "module": "events._types", + "name": "BROWSER_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L31" + } + ], + "type": { + "type": "literal", + "value": "'browserClosed'" + }, + "permalink": "/python/api/enum/Event#BROWSER_CLOSED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48692, + "module": "events._types", + "name": "PAGE_CREATED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L32" + } + ], + "type": { + "type": "literal", + "value": "'pageCreated'" + }, + "permalink": "/python/api/enum/Event#PAGE_CREATED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48693, + "module": "events._types", + "name": "PAGE_CLOSED", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L33" + } + ], + "type": { + "type": "literal", + "value": "'pageClosed'" + }, + "permalink": "/python/api/enum/Event#PAGE_CLOSED", + "parentId": 2737 + }, + { + "kind": 16, + "kindString": "Enumeration Member", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 48694, + "module": "events._types", + "name": "CRAWLER_STATUS", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L36" + } + ], + "type": { + "type": "literal", + "value": "'crawlerStatus'" + }, + "permalink": "/python/api/enum/Event#CRAWLER_STATUS", + "parentId": 2737 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Names of all possible events that can be emitted using an `EventManager`." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 48686, + 48691, + 48689, + 48690, + 48694, + 48687, + 48685, + 48693, + 48692, + 48683, + 48688, + 48684 + ], + "title": "Enumeration members" + } + ], + "id": 48682, + "module": "events._types", + "name": "Event", + "parsedDocstring": { + "text": "Names of all possible events that can be emitted using an `EventManager`." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/enum/Event", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 51455, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData#model_config", + "parentId": 2760 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the aborting event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 51455 + ], + "title": "Properties" + } + ], + "id": 51454, + "module": "events._types", + "name": "EventAbortingData", + "parsedDocstring": { + "text": "Data for the aborting event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 73, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L73" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventAbortingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 54218, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 83, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L83" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData#model_config", + "parentId": 2762 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the exit event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 54218 + ], + "title": "Properties" + } + ], + "id": 54217, + "module": "events._types", + "name": "EventExitData", + "parsedDocstring": { + "text": "Data for the exit event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 80, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L80" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventExitData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56976, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 65, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L65" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData#model_config", + "parentId": 2757 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 56977, + "module": "events._types", + "name": "time_remaining", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 69, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L69" + } + ], + "type": { + "name": "Annotated[timedelta_secs | None, Field(alias='timeRemainingSecs')]", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta_secs", + "target": "2981", + "ref": { + "id": 57199, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 68 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#timedelta_secs" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "permalink": "/python/api/class/EventMigratingData#time_remaining", + "parentId": 2757 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the migrating event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 56976, + 56977 + ], + "title": "Properties" + } + ], + "id": 56975, + "module": "events._types", + "name": "EventMigratingData", + "parsedDocstring": { + "text": "Data for the migrating event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L62" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventMigratingData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 59950, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#model_config", + "parentId": 2750 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 59951, + "module": "events._types", + "name": "is_migrating", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L45" + } + ], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData#is_migrating", + "parentId": 2750 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the persist state event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 59951, + 59950 + ], + "title": "Properties" + } + ], + "id": 59949, + "module": "events._types", + "name": "EventPersistStateData", + "parsedDocstring": { + "text": "Data for the persist state event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventPersistStateData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 62705, + "module": "events._types", + "name": "model_config", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData#model_config", + "parentId": 2753 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 62706, + "module": "events._types", + "name": "cpu_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 54, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L54" + } + ], + "type": { + "name": "CpuInfo", + "type": "reference", + "target": "3072", + "ref": { + "id": 63023, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/CpuInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#cpu_info", + "parentId": 2753 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "flags": {}, + "groups": [], + "id": 62707, + "module": "events._types", + "name": "memory_info", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 55, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L55" + } + ], + "type": { + "name": "MemoryUsageInfo", + "type": "reference", + "target": "3075", + "ref": { + "id": 63026, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/system.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryUsageInfo" + } + }, + "permalink": "/python/api/class/EventSystemInfoData#memory_info", + "parentId": 2753 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Data for the system info event." + } + ] + }, + "decorations": [ + { + "args": "('Event data')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 62706, + 62707, + 62705 + ], + "title": "Properties" + } + ], + "id": 62704, + "module": "events._types", + "name": "EventSystemInfoData", + "parsedDocstring": { + "text": "Data for the system info event." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/events/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 49, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/events/_types.py#L49" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/EventSystemInfoData", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 64082, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 64083, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 64084, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "510", + "ref": { + "id": 63536, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1057 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 64082, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1056 + } + ], + "permalink": "/python/api/class/StorageClient#get_storage_client_cache_key", + "parentId": 1055 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 64085, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L42" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 64086, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64087, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64088, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64089, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64090, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 63536, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1107", + "ref": { + "id": 64133, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 64085, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1059 + } + ], + "permalink": "/python/api/class/StorageClient#create_dataset_client", + "parentId": 1055 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 64091, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 64092, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64093, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64094, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64095, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64096, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 63536, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1079", + "ref": { + "id": 64105, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 64091, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1065 + } + ], + "permalink": "/python/api/class/StorageClient#create_kvs_client", + "parentId": 1055 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 64097, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L64" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 64098, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64099, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64100, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64101, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 64102, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 63536, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1030", + "ref": { + "id": 64056, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 64097, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1071 + } + ], + "permalink": "/python/api/class/StorageClient#create_rq_client", + "parentId": 1055 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 64103, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 64104, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "int", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 1077 + } + ], + "permalink": "/python/api/class/StorageClient#get_rate_limit_errors", + "parentId": 1055 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 64085, + 64091, + 64097, + 64103, + 64082 + ], + "title": "Methods" + } + ], + "id": 64081, + "module": "storage_clients._base._storage_client", + "name": "StorageClient", + "parsedDocstring": { + "text": "Base class for storage clients.\n\nThe `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's\nstorage types: datasets, key-value stores, and request queues. It provides methods to open clients for\neach of these storage types and handles common functionality.\n\nStorage clients implementations can be provided for various backends (file system, memory, databases,\nvarious cloud providers, etc.) to support different use cases from development to production environments.\n\nEach storage client implementation is responsible for ensuring proper initialization, data persistence\n(where applicable), and consistent access patterns across all storage types it supports." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L19" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "SqlStorageClient", + "target": "710", + "type": "reference", + "ref": { + "id": 63736, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + { + "name": "MemoryStorageClient", + "target": "883", + "type": "reference", + "ref": { + "id": 63909, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/MemoryStorageClient" + } + }, + { + "name": "RedisStorageClient", + "target": "1198", + "type": "reference", + "ref": { + "id": 64224, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_redis/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RedisStorageClient" + } + }, + { + "name": "FileSystemStorageClient", + "target": "1371", + "type": "reference", + "ref": { + "id": 64397, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/FileSystemStorageClient" + } + } + ], + "permalink": "/python/api/class/StorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 65281, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_memory/_storage_client.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 65457, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65458, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65459, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65460, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65461, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 64907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1107", + "ref": { + "id": 65504, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 65456, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 884 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 65456, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_dataset_client", + "parentId": 883 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 65287, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_memory/_storage_client.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 65463, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65464, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65465, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65466, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65467, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 64907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1079", + "ref": { + "id": 65476, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 65462, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 890 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 65462, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_kvs_client", + "parentId": 883 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 65293, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 59, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_memory/_storage_client.py#L59" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 65469, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65470, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65471, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65472, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 65473, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 64907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1030", + "ref": { + "id": 65427, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 65468, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 896 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 65468, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#create_rq_client", + "parentId": 883 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 68741, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 65454, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 65455, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "510", + "ref": { + "id": 64907, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1057 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 65453, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4344 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 65453, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_storage_client_cache_key", + "parentId": 883 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 68742, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 65475, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 65474, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4345 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 65474, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/MemoryStorageClient#get_rate_limit_errors", + "parentId": 883 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 65281, + 65287, + 65293, + 68742, + 68741 + ], + "title": "Methods" + } + ], + "id": 65280, + "module": "storage_clients._memory._storage_client", + "name": "MemoryStorageClient", + "parsedDocstring": { + "text": "Memory implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that store all data\nin memory using Python data structures (lists and dictionaries). No data is persisted between process runs,\nmeaning all stored data is lost when the program terminates.\n\nThe memory implementation provides fast access to data but is limited by available memory and does not\nsupport data sharing across different processes. All storage operations happen entirely in memory with\nno disk operations.\n\nThe memory storage client is useful for testing and development environments, or short-lived crawler\noperations where persistence is not required." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_memory/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 15, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_memory/_storage_client.py#L15" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1055", + "type": "reference", + "ref": { + "id": 65452, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/MemoryStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 70114, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 38, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_file_system/_storage_client.py#L38" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 69799, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 69800, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "510", + "ref": { + "id": 69252, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1057 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 69798, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 1372 + } + ], + "overwrites": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 69798, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_storage_client_cache_key", + "parentId": 1371 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 70117, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_file_system/_storage_client.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 69802, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69803, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69804, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69805, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69806, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 69252, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1107", + "ref": { + "id": 69849, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 69801, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 1375 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 69801, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_dataset_client", + "parentId": 1371 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 70123, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 57, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_file_system/_storage_client.py#L57" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 69808, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69809, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69810, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69811, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69812, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 69252, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1079", + "ref": { + "id": 69821, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 69807, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 1381 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 69807, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_kvs_client", + "parentId": 1371 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 70129, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_file_system/_storage_client.py#L71" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 69814, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69815, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69816, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69817, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 69818, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 69252, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1030", + "ref": { + "id": 69772, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 69813, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 1387 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 69813, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#create_rq_client", + "parentId": 1371 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73090, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 69820, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 69819, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4348 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 69819, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/FileSystemStorageClient#get_rate_limit_errors", + "parentId": 1371 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 70117, + 70123, + 70129, + 73090, + 70114 + ], + "title": "Methods" + } + ], + "id": 70113, + "module": "storage_clients._file_system._storage_client", + "name": "FileSystemStorageClient", + "parsedDocstring": { + "text": "File system implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto the local file system. Each storage type is implemented with its own specific file system client\nthat stores data in a structured directory hierarchy.\n\nData is stored in JSON format in predictable file paths, making it easy to inspect and manipulate\nthe stored data outside of the Crawlee application if needed.\n\nAll data persists between program runs but is limited to access from the local machine\nwhere the files are stored.\n\nWarning: This storage client is not safe for concurrent access from multiple crawler processes.\nUse it only when running a single crawler process at a time." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_file_system/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_file_system/_storage_client.py#L20" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1055", + "type": "reference", + "ref": { + "id": 69797, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/FileSystemStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73801, + "module": "storage_clients._sql._storage_client", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the SQL storage client.\n", + "args": { + "connection_string": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory.", + "engine": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 54, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L54" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the SQL storage client.\n" + } + ] + }, + "flags": {}, + "id": 73802, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Database connection string (e.g., \"sqlite+aiosqlite:///crawlee.db\").\nIf not provided, defaults to SQLite database in the storage directory." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 73803, + "kind": 32768, + "kindString": "Parameter", + "name": "connection_string", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 712 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Pre-configured AsyncEngine instance. If provided, connection_string is ignored." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 73804, + "kind": 32768, + "kindString": "Parameter", + "name": "engine", + "type": { + "name": "AsyncEngine | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "AsyncEngine", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 712 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 711 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__init__", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73805, + "module": "storage_clients._sql._storage_client", + "name": "__aenter__", + "parsedDocstring": { + "text": "Async context manager entry." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 86, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L86" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager entry." + } + ] + }, + "flags": {}, + "id": 73806, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SqlStorageClient", + "type": "reference", + "target": "710", + "ref": { + "id": 73800, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SqlStorageClient" + } + }, + "parentId": 715 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aenter__", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73807, + "module": "storage_clients._sql._storage_client", + "name": "__aexit__", + "parsedDocstring": { + "text": "Async context manager exit." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 90, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L90" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Async context manager exit." + } + ] + }, + "flags": {}, + "id": 73808, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 73809, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3134", + "ref": { + "id": 76224, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 718 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 73810, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 718 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 73811, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 718 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 717 + } + ], + "permalink": "/python/api/class/SqlStorageClient#__aexit__", + "parentId": 710 + }, + { + "kind": 1024, + "kindString": "Property", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the SQLAlchemy AsyncEngine instance." + } + ] + }, + "decorations": [ + { + "name": "property" + } + ], + "flags": {}, + "groups": [], + "id": 73812, + "module": "storage_clients._sql._storage_client", + "name": "engine", + "parsedDocstring": { + "text": "Get the SQLAlchemy AsyncEngine instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 100, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L100" + } + ], + "type": { + "name": "AsyncEngine", + "type": "reference", + "target": 0, + "ref": {} + }, + "permalink": "/python/api/class/SqlStorageClient#engine", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73813, + "module": "storage_clients._sql._storage_client", + "name": "get_dialect_name", + "parsedDocstring": { + "text": "Get the database dialect name." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L106" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the database dialect name." + } + ] + }, + "flags": {}, + "id": 73814, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_dialect_name", + "parameters": [], + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 723 + } + ], + "permalink": "/python/api/class/SqlStorageClient#get_dialect_name", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73815, + "module": "storage_clients._sql._storage_client", + "name": "initialize", + "parsedDocstring": { + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 110, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L110" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the database schema.\n\nThis method creates all necessary tables if they don't exist.\nShould be called before using the storage client." + } + ] + }, + "flags": {}, + "id": 73816, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "initialize", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 73817, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "510", + "ref": { + "id": 73600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 726 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 725 + } + ], + "permalink": "/python/api/class/SqlStorageClient#initialize", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73818, + "module": "storage_clients._sql._storage_client", + "name": "close", + "parsedDocstring": { + "text": "Close the database connection pool." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 161, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L161" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the database connection pool." + } + ] + }, + "flags": {}, + "id": 73819, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 728 + } + ], + "permalink": "/python/api/class/SqlStorageClient#close", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 73820, + "module": "storage_clients._sql._storage_client", + "name": "create_session", + "parsedDocstring": { + "text": "Create a new database session.\n", + "returns": "A new AsyncSession instance." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 167, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L167" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "A new AsyncSession instance." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Create a new database session.\n" + } + ] + }, + "flags": {}, + "id": 73821, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "create_session", + "parameters": [], + "type": { + "name": "AsyncSession", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 730 + } + ], + "permalink": "/python/api/class/SqlStorageClient#create_session", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 73822, + "module": "storage_clients._base._storage_client", + "name": "create_dataset_client", + "parsedDocstring": { + "text": "Create a dataset client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 178, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L178" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a dataset client." + } + ] + }, + "flags": {}, + "id": 74150, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_dataset_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74151, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74152, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74153, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74154, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 73600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1060 + } + ], + "type": { + "name": "DatasetClient", + "type": "reference", + "target": "1107", + "ref": { + "id": 74197, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_dataset_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/DatasetClient" + } + }, + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 74149, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "parentId": 732 + } + ], + "overwrites": { + "name": "StorageClient.create_dataset_client", + "target": 1059, + "type": "reference", + "ref": { + "id": 74149, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_dataset_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_dataset_client", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 73828, + "module": "storage_clients._base._storage_client", + "name": "create_kvs_client", + "parsedDocstring": { + "text": "Create a key-value store client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 200, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L200" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a key-value store client." + } + ] + }, + "flags": {}, + "id": 74156, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_kvs_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74157, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74158, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74159, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74160, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 73600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1066 + } + ], + "type": { + "name": "KeyValueStoreClient", + "type": "reference", + "target": "1079", + "ref": { + "id": 74169, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_key_value_store_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 12 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/KeyValueStoreClient" + } + }, + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 74155, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "parentId": 738 + } + ], + "overwrites": { + "name": "StorageClient.create_kvs_client", + "target": 1065, + "type": "reference", + "ref": { + "id": 74155, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_kvs_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_kvs_client", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 73834, + "module": "storage_clients._base._storage_client", + "name": "create_rq_client", + "parsedDocstring": { + "text": "Create a request queue client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 222, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L222" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Create a request queue client." + } + ] + }, + "flags": {}, + "id": 74162, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "create_rq_client", + "parameters": [ + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74163, + "kind": 32768, + "kindString": "Parameter", + "name": "id", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74164, + "kind": 32768, + "kindString": "Parameter", + "name": "name", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74165, + "kind": 32768, + "kindString": "Parameter", + "name": "alias", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + }, + { + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 74166, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Configuration", + "target": "510", + "ref": { + "id": 73600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 1072 + } + ], + "type": { + "name": "RequestQueueClient", + "type": "reference", + "target": "1030", + "ref": { + "id": 74120, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_request_queue_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 13 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueueClient" + } + }, + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 74161, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "parentId": 744 + } + ], + "overwrites": { + "name": "StorageClient.create_rq_client", + "target": 1071, + "type": "reference", + "ref": { + "id": 74161, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 64 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#create_rq_client" + } + }, + "permalink": "/python/api/class/SqlStorageClient#create_rq_client", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 77432, + "module": "storage_clients._base._storage_client", + "name": "get_storage_client_cache_key", + "parsedDocstring": { + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return a cache key that can differentiate between different storages of this and other clients.\n\nCan be based on configuration or on the client itself. By default, returns a module and name of the client\nclass." + } + ] + }, + "flags": {}, + "id": 74147, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_storage_client_cache_key", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 74148, + "kind": 32768, + "kindString": "Parameter", + "name": "configuration", + "type": { + "name": "Configuration", + "type": "reference", + "target": "510", + "ref": { + "id": 73600, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 20 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Configuration" + } + }, + "parentId": 1057 + } + ], + "type": { + "name": "Hashable", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 74146, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "parentId": 4342 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_storage_client_cache_key", + "target": 1056, + "type": "reference", + "ref": { + "id": 74146, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_storage_client_cache_key" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_storage_client_cache_key", + "parentId": 710 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 77433, + "module": "storage_clients._base._storage_client", + "name": "get_rate_limit_errors", + "parsedDocstring": { + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_base/_storage_client.py#L74" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return statistics about rate limit errors encountered by the HTTP client in storage client." + } + ] + }, + "flags": {}, + "id": 74168, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "get_rate_limit_errors", + "parameters": [], + "type": { + "name": "dict[int, int]", + "type": "reference", + "target": 0, + "ref": {} + }, + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 74167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "parentId": 4343 + } + ], + "inheritedFrom": { + "name": "StorageClient.get_rate_limit_errors", + "target": 1077, + "type": "reference", + "ref": { + "id": 74167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 74 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/StorageClient#get_rate_limit_errors" + } + }, + "permalink": "/python/api/class/SqlStorageClient#get_rate_limit_errors", + "parentId": 710 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + } + ] + }, + "decorations": [ + { + "args": "('Storage clients')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 73805, + 73807, + 73801, + 73818, + 73822, + 73828, + 73834, + 73820, + 73813, + 77433, + 77432, + 73815 + ], + "title": "Methods" + }, + { + "children": [ + 73812 + ], + "title": "Properties" + } + ], + "id": 73800, + "module": "storage_clients._sql._storage_client", + "name": "SqlStorageClient", + "parsedDocstring": { + "text": "SQL implementation of the storage client.\n\nThis storage client provides access to datasets, key-value stores, and request queues that persist data\nto a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for\nrecords.\n\nThe client accepts either a database connection string or a pre-configured AsyncEngine. If neither is\nprovided, it creates a default SQLite database 'crawlee.db' in the storage directory.\n\nDatabase schema is automatically created during initialization. SQLite databases receive performance\noptimizations including WAL mode and increased cache size.\n\n\n:::warning Warning\nThis is an experimental feature. The behavior and interface may change in future versions.\n:::" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_sql/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/storage_clients/_sql/_storage_client.py#L32" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "StorageClient", + "target": "1055", + "type": "reference", + "ref": { + "id": 74145, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/_base/_storage_client.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 19 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/StorageClient" + } + } + ], + "permalink": "/python/api/class/SqlStorageClient", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79753, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 79754, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2320, + "type": "reference", + "ref": { + "id": 79753, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2320 + } + ], + "permalink": "/python/api/class/RequestLoader#get_handled_count", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79755, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 79756, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2322, + "type": "reference", + "ref": { + "id": 79755, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2322 + } + ], + "permalink": "/python/api/class/RequestLoader#get_total_count", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79757, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 79758, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2324, + "type": "reference", + "ref": { + "id": 79757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2324 + } + ], + "permalink": "/python/api/class/RequestLoader#is_empty", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79759, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 79760, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2326, + "type": "reference", + "ref": { + "id": 79759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2326 + } + ], + "permalink": "/python/api/class/RequestLoader#is_finished", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79761, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 79762, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 77838, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2328, + "type": "reference", + "ref": { + "id": 79761, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2328 + } + ], + "permalink": "/python/api/class/RequestLoader#fetch_next_request", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 79763, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 79764, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 79765, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 77838, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2331 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 78448, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2330, + "type": "reference", + "ref": { + "id": 79763, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2330 + } + ], + "permalink": "/python/api/class/RequestLoader#mark_request_as_handled", + "parentId": 2319 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 79766, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 79767, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 79768, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestManager", + "target": "2385", + "ref": { + "id": 79818, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2334 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2405", + "ref": { + "id": 79838, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "parentId": 2333 + } + ], + "permalink": "/python/api/class/RequestLoader#to_tandem", + "parentId": 2319 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 79761, + 79753, + 79755, + 79757, + 79759, + 79763, + 79766 + ], + "title": "Methods" + } + ], + "id": 79752, + "module": "request_loaders._request_loader", + "name": "RequestLoader", + "parsedDocstring": { + "text": "An abstract class defining the interface for classes that provide access to a read-only stream of requests.\n\nRequest loaders are used to manage and provide access to a storage of crawling requests.\n\nKey responsibilities:\n- Fetching the next request to be processed.\n- Marking requests as successfully handled after processing.\n- Managing state information such as the total and handled request counts." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L17" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedBy": [ + { + "name": "SitemapRequestLoader", + "target": "2348", + "type": "reference", + "ref": { + "id": 79781, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + { + "name": "RequestManager", + "target": "2385", + "type": "reference", + "ref": { + "id": 79818, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + { + "name": "RequestList", + "target": "2449", + "type": "reference", + "ref": { + "id": 79882, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_list.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 32 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestList" + } + } + ], + "permalink": "/python/api/class/RequestLoader", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 82268, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager.py#L22" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 82269, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 82268, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 2386 + } + ], + "permalink": "/python/api/class/RequestManager#drop", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 82270, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager.py#L26" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 82271, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 82272, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 2389 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82273, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2389 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 80897, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 2388, + "type": "reference", + "ref": { + "id": 82270, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2388 + } + ], + "permalink": "/python/api/class/RequestManager#add_request", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 82274, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager.py#L43" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 82275, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 82276, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82277, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82278, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82279, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82280, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82281, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2393 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 2392, + "type": "reference", + "ref": { + "id": 82274, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2392 + } + ], + "permalink": "/python/api/class/RequestManager#add_requests", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 82282, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager.py#L71" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 82283, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 82284, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2401 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 82285, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2401 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 80897, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 2400, + "type": "reference", + "ref": { + "id": 82282, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2400 + } + ], + "permalink": "/python/api/class/RequestManager#reclaim_request", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83815, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L29" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 82203, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 3933, + "type": "reference", + "ref": { + "id": 83815, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 3933 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_handled_count", + "target": 2320, + "type": "reference", + "ref": { + "id": 82202, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_handled_count", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83816, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L33" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 82205, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 3934, + "type": "reference", + "ref": { + "id": 83816, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 3934 + } + ], + "inheritedFrom": { + "name": "RequestLoader.get_total_count", + "target": 2322, + "type": "reference", + "ref": { + "id": 82204, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManager#get_total_count", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83817, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L37" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 82207, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 3935, + "type": "reference", + "ref": { + "id": 83817, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 3935 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_empty", + "target": 2324, + "type": "reference", + "ref": { + "id": 82206, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/RequestManager#is_empty", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83818, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L41" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 82209, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 3936, + "type": "reference", + "ref": { + "id": 83818, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 3936 + } + ], + "inheritedFrom": { + "name": "RequestLoader.is_finished", + "target": 2326, + "type": "reference", + "ref": { + "id": 82208, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/RequestManager#is_finished", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83819, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L45" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 82211, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 3937, + "type": "reference", + "ref": { + "id": 83819, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 3937 + } + ], + "inheritedFrom": { + "name": "RequestLoader.fetch_next_request", + "target": 2328, + "type": "reference", + "ref": { + "id": 82210, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManager#fetch_next_request", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 83820, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L53" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 82213, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 82214, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 80287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2331 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 3938, + "type": "reference", + "ref": { + "id": 83820, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 3938 + } + ], + "inheritedFrom": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2330, + "type": "reference", + "ref": { + "id": 82212, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManager#mark_request_as_handled", + "parentId": 2385 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 83821, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 82216, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 82217, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2334 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2405", + "ref": { + "id": 82287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 3939 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 82215, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManager#to_tandem", + "parentId": 2385 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 82270, + 82274, + 82268, + 83819, + 83815, + 83816, + 83817, + 83818, + 83820, + 82282, + 83821 + ], + "title": "Methods" + } + ], + "id": 82267, + "module": "request_loaders._request_manager", + "name": "RequestManager", + "parsedDocstring": { + "text": "Base class that extends `RequestLoader` with the capability to enqueue new requests and reclaim failed ones." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager.py#L18" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2319", + "type": "reference", + "ref": { + "id": 82201, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "extendedBy": [ + { + "name": "RequestManagerTandem", + "target": "2405", + "type": "reference", + "ref": { + "id": 82287, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + { + "name": "RequestQueue", + "target": "3667", + "type": "reference", + "ref": { + "id": 83549, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storages/_request_queue.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestQueue" + } + } + ], + "permalink": "/python/api/class/RequestManager", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 86227, + "module": "request_loaders._request_manager_tandem", + "name": "__init__", + "parsedDocstring": { + "text": "" + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 31, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L31" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "flags": {}, + "id": 86228, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86229, + "kind": 32768, + "kindString": "Parameter", + "name": "request_loader", + "type": { + "name": "RequestLoader", + "type": "reference", + "target": "2319", + "ref": { + "id": 86140, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + }, + "parentId": 2407 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86230, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager", + "type": "reference", + "target": "2385", + "ref": { + "id": 86206, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + }, + "parentId": 2407 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2406 + } + ], + "permalink": "/python/api/class/RequestManagerTandem#__init__", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86231, + "module": "request_loaders._request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Get the number of requests in the loader that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 36, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L36" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get the number of requests in the loader that have been handled." + } + ] + }, + "flags": {}, + "id": 86142, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 3933, + "type": "reference", + "ref": { + "id": 87754, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "parentId": 2410 + } + ], + "overwrites": { + "name": "RequestManager.get_handled_count", + "target": 3933, + "type": "reference", + "ref": { + "id": 87754, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_handled_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_handled_count", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86233, + "module": "request_loaders._request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 40, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L40" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Get an offline approximation of the total number of requests in the loader (i.e. pending + handled)." + } + ] + }, + "flags": {}, + "id": 86144, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 3934, + "type": "reference", + "ref": { + "id": 87755, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "parentId": 2412 + } + ], + "overwrites": { + "name": "RequestManager.get_total_count", + "target": 3934, + "type": "reference", + "ref": { + "id": 87755, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#get_total_count" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#get_total_count", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86235, + "module": "request_loaders._request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 44, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L44" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if there are no more requests in the loader (there might still be unfinished requests)." + } + ] + }, + "flags": {}, + "id": 86146, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_empty", + "target": 3935, + "type": "reference", + "ref": { + "id": 87756, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "parentId": 2414 + } + ], + "overwrites": { + "name": "RequestManager.is_empty", + "target": 3935, + "type": "reference", + "ref": { + "id": 87756, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_empty" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_empty", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86237, + "module": "request_loaders._request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Return True if all requests have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 48, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L48" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return True if all requests have been handled." + } + ] + }, + "flags": {}, + "id": 86148, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.is_finished", + "target": 3936, + "type": "reference", + "ref": { + "id": 87757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "parentId": 2416 + } + ], + "overwrites": { + "name": "RequestManager.is_finished", + "target": 3936, + "type": "reference", + "ref": { + "id": 87757, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#is_finished" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#is_finished", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86239, + "module": "request_loaders._request_manager", + "name": "add_request", + "parsedDocstring": { + "text": "Add a single request to the manager and store it in underlying resource client.\n", + "args": { + "request": "The request object (or its string representation) to be added to the manager.", + "forefront": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + }, + "returns": "Information about the request addition to the manager or None if the request was not added." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 52, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L52" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "blockTags": [ + { + "content": [ + { + "kind": "text", + "text": "Information about the request addition to the manager or None if the request was not added." + } + ], + "tag": "@returns" + } + ], + "summary": [ + { + "kind": "text", + "text": "Add a single request to the manager and store it in underlying resource client.\n" + } + ] + }, + "flags": {}, + "id": 86210, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_request", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The request object (or its string representation) to be added to the manager." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86211, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "str | Request", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 84226, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + }, + "parentId": 2389 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Determines whether the request should be added to the beginning (if True) or the end (if False)\nof the manager.\n" + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86212, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2389 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 84836, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.add_request", + "target": 2388, + "type": "reference", + "ref": { + "id": 86209, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "parentId": 2418 + } + ], + "overwrites": { + "name": "RequestManager.add_request", + "target": 2388, + "type": "reference", + "ref": { + "id": 86209, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_request", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 86243, + "module": "request_loaders._request_manager", + "name": "add_requests", + "parsedDocstring": { + "text": "Add requests to the manager in batches.\n", + "args": { + "requests": "Requests to enqueue.", + "forefront": "If True, add requests to the beginning of the queue.", + "batch_size": "The number of requests to add in one batch.", + "wait_time_between_batches": "Time to wait between adding batches.", + "wait_for_all_requests_to_be_added": "If True, wait for all requests to be added before returning.", + "wait_for_all_requests_to_be_added_timeout": "Timeout for waiting for all requests to be added." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Add requests to the manager in batches.\n" + } + ] + }, + "flags": {}, + "id": 86214, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "add_requests", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Requests to enqueue." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86215, + "kind": 32768, + "kindString": "Parameter", + "name": "requests", + "type": { + "name": "Sequence", + "type": "reference", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 84226, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, add requests to the beginning of the queue." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86216, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "The number of requests to add in one batch." + } + ] + }, + "defaultValue": "1000", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86217, + "kind": 32768, + "kindString": "Parameter", + "name": "batch_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Time to wait between adding batches." + } + ] + }, + "defaultValue": "timedelta(seconds=1)", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86218, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_time_between_batches", + "type": { + "name": "timedelta", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "If True, wait for all requests to be added before returning." + } + ] + }, + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86219, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2393 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Timeout for waiting for all requests to be added." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86220, + "kind": 32768, + "kindString": "Parameter", + "name": "wait_for_all_requests_to_be_added_timeout", + "type": { + "name": "timedelta | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "timedelta", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2393 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.add_requests", + "target": 2392, + "type": "reference", + "ref": { + "id": 86213, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "parentId": 2422 + } + ], + "overwrites": { + "name": "RequestManager.add_requests", + "target": 2392, + "type": "reference", + "ref": { + "id": 86213, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 43 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#add_requests" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#add_requests", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86251, + "module": "request_loaders._request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 76, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L76" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the next request to be processed, or `None` if there are no more pending requests.\n\nThe method should return `None` if and only if `is_finished` would return `True`. In other cases, the method\nshould wait until a request appears." + } + ] + }, + "flags": {}, + "id": 86150, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 3937, + "type": "reference", + "ref": { + "id": 87758, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "parentId": 2430 + } + ], + "overwrites": { + "name": "RequestManager.fetch_next_request", + "target": 3937, + "type": "reference", + "ref": { + "id": 87758, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#fetch_next_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#fetch_next_request", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86253, + "module": "request_loaders._request_manager", + "name": "reclaim_request", + "parsedDocstring": { + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 99, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L99" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Reclaims a failed request back to the source, so that it can be returned for processing later again.\n\nIt is possible to modify the request data by supplying an updated request as a parameter." + } + ] + }, + "flags": {}, + "id": 86222, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "reclaim_request", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86223, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 84226, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2401 + }, + { + "defaultValue": "False", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 86224, + "kind": 32768, + "kindString": "Parameter", + "name": "forefront", + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2401 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 84836, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 2400, + "type": "reference", + "ref": { + "id": 86221, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "parentId": 2432 + } + ], + "overwrites": { + "name": "RequestManager.reclaim_request", + "target": 2400, + "type": "reference", + "ref": { + "id": 86221, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 71 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#reclaim_request" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#reclaim_request", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86257, + "module": "request_loaders._request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 103, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L103" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as handled after a successful processing (or after giving up retrying)." + } + ] + }, + "flags": {}, + "id": 86152, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 86153, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 84226, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2331 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 3938, + "type": "reference", + "ref": { + "id": 87759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "parentId": 2436 + } + ], + "overwrites": { + "name": "RequestManager.mark_request_as_handled", + "target": 3938, + "type": "reference", + "ref": { + "id": 87759, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#mark_request_as_handled", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "decorations": [ + { + "name": "abstractmethod" + } + ], + "flags": {}, + "groups": [], + "id": 86260, + "module": "request_loaders._request_manager", + "name": "drop", + "parsedDocstring": { + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 107, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L107" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Remove persistent state either from the Apify Cloud storage or from the local database." + } + ] + }, + "flags": {}, + "id": 86208, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "drop", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 86207, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "parentId": 2439 + } + ], + "overwrites": { + "name": "RequestManager.drop", + "target": 2386, + "type": "reference", + "ref": { + "id": 86207, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 22 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestManager#drop" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#drop", + "parentId": 2405 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 87762, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 86155, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 86156, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2334 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2405", + "ref": { + "id": 86226, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 86154, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 3941 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 86154, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/RequestManagerTandem#to_tandem", + "parentId": 2405 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 86227, + 86239, + 86243, + 86260, + 86251, + 86231, + 86233, + 86235, + 86237, + 86257, + 86253, + 87762 + ], + "title": "Methods" + } + ], + "id": 86226, + "module": "request_loaders._request_manager_tandem", + "name": "RequestManagerTandem", + "parsedDocstring": { + "text": "Implements a tandem behaviour for a pair of `RequestLoader` and `RequestManager`.\n\nIn this scenario, the contents of the \"loader\" get transferred into the \"manager\", allowing processing the requests\nfrom both sources and also enqueueing new requests (not possible with plain `RequestManager`)." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_manager_tandem.py#L24" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestManager", + "target": "2385", + "type": "reference", + "ref": { + "id": 86206, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 18 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManager" + } + } + ], + "permalink": "/python/api/class/RequestManagerTandem", + "categories": [] + }, + { + "kind": 128, + "kindString": "Class", + "children": [ + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90111, + "module": "request_loaders._sitemap_request_loader", + "name": "__init__", + "parsedDocstring": { + "text": "Initialize the sitemap request loader.\n", + "args": { + "sitemap_urls": "Configuration options for the loader.", + "proxy_info": "Optional proxy to use for fetching sitemaps.", + "include": "List of glob or regex patterns to include URLs.", + "exclude": "List of glob or regex patterns to exclude URLs.", + "max_buffer_size": "Maximum number of URLs to buffer in memory.", + "http_client": "the instance of `HttpClient` to use for fetching sitemaps.", + "persist_state_key": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs.", + "transform_request_function": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 106, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L106" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Initialize the sitemap request loader.\n" + } + ] + }, + "flags": {}, + "id": 90112, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [], + "name": "__init__", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Configuration options for the loader." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90113, + "kind": 32768, + "kindString": "Parameter", + "name": "sitemap_urls", + "type": { + "name": "list", + "type": "reference", + "typeArguments": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "the instance of `HttpClient` to use for fetching sitemaps." + } + ] + }, + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90114, + "kind": 32768, + "kindString": "Parameter", + "name": "http_client", + "type": { + "name": "HttpClient", + "type": "reference", + "target": "3785", + "ref": { + "id": 91547, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/http_clients/_base.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 75 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/HttpClient" + } + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Optional proxy to use for fetching sitemaps." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90115, + "kind": 32768, + "kindString": "Parameter", + "name": "proxy_info", + "type": { + "name": "ProxyInfo | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProxyInfo", + "target": "35", + "ref": { + "id": 87797, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/proxy_configuration.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 26 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProxyInfo" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to include URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90116, + "kind": 32768, + "kindString": "Parameter", + "name": "include", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3217", + "ref": { + "id": 90979, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "List of glob or regex patterns to exclude URLs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90117, + "kind": 32768, + "kindString": "Parameter", + "name": "exclude", + "type": { + "name": "list[re.Pattern[Any] | Glob] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "list", + "typeArguments": [ + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "re.Pattern", + "typeArguments": [ + { + "type": "reference", + "name": "Any", + "target": 0, + "ref": {} + } + ], + "target": 0, + "ref": {} + }, + { + "type": "reference", + "name": "Glob", + "target": "3217", + "ref": { + "id": 90979, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/globs.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 11 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Glob" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Maximum number of URLs to buffer in memory." + } + ] + }, + "defaultValue": "200", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90118, + "kind": 32768, + "kindString": "Parameter", + "name": "max_buffer_size", + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "A key for persisting the loader's state in the KeyValueStore.\nWhen provided, allows resuming from where it left off after interruption.\nIf None, no state persistence occurs." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90119, + "kind": 32768, + "kindString": "Parameter", + "name": "persist_state_key", + "type": { + "name": "str | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "str", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + }, + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "An optional function to transform requests\ngenerated by the loader. It receives `RequestOptions` with `url` and should return either\nmodified `RequestOptions` or a `RequestTransformAction`." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": true + }, + "id": 90120, + "kind": 32768, + "kindString": "Parameter", + "name": "transform_request_function", + "type": { + "name": "Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Callable", + "typeArguments": [ + { + "type": "reference", + "name": "[RequestOptions]", + "target": 0, + "ref": {} + }, + { + "type": "union", + "types": [ + { + "type": "reference", + "name": "RequestOptions", + "target": "389", + "ref": { + "id": 88151, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 121 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestOptions" + } + }, + { + "type": "reference", + "name": "RequestTransformAction", + "target": "85", + "ref": { + "id": 87847, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_types.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 42 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api#RequestTransformAction" + } + } + ] + } + ], + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2350 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2349 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__init__", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90121, + "module": "request_loaders._sitemap_request_loader", + "name": "get_total_count", + "parsedDocstring": { + "text": "Return the total number of URLs found so far." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 288, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L288" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the total number of URLs found so far." + } + ] + }, + "flags": {}, + "id": 90122, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_total_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2322, + "type": "reference", + "ref": { + "id": 90084, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "parentId": 2359 + } + ], + "overwrites": { + "name": "RequestLoader.get_total_count", + "target": 2322, + "type": "reference", + "ref": { + "id": 90084, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 33 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_total_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_total_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90123, + "module": "request_loaders._sitemap_request_loader", + "name": "get_handled_count", + "parsedDocstring": { + "text": "Return the number of URLs that have been handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 294, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L294" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Return the number of URLs that have been handled." + } + ] + }, + "flags": {}, + "id": 90124, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "get_handled_count", + "parameters": [], + "type": { + "name": "int", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2320, + "type": "reference", + "ref": { + "id": 90082, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "parentId": 2361 + } + ], + "overwrites": { + "name": "RequestLoader.get_handled_count", + "target": 2320, + "type": "reference", + "ref": { + "id": 90082, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 29 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#get_handled_count" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#get_handled_count", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90125, + "module": "request_loaders._sitemap_request_loader", + "name": "is_empty", + "parsedDocstring": { + "text": "Check if there are no more URLs to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 300, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L300" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if there are no more URLs to process." + } + ] + }, + "flags": {}, + "id": 90126, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_empty", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2324, + "type": "reference", + "ref": { + "id": 90086, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "parentId": 2363 + } + ], + "overwrites": { + "name": "RequestLoader.is_empty", + "target": 2324, + "type": "reference", + "ref": { + "id": 90086, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 37 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_empty" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_empty", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90127, + "module": "request_loaders._sitemap_request_loader", + "name": "is_finished", + "parsedDocstring": { + "text": "Check if all URLs have been processed." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 306, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L306" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Check if all URLs have been processed." + } + ] + }, + "flags": {}, + "id": 90128, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "is_finished", + "parameters": [], + "type": { + "name": "bool", + "type": "reference", + "target": 0, + "ref": {} + }, + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2326, + "type": "reference", + "ref": { + "id": 90088, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "parentId": 2365 + } + ], + "overwrites": { + "name": "RequestLoader.is_finished", + "target": 2326, + "type": "reference", + "ref": { + "id": 90088, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 41 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#is_finished" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#is_finished", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90129, + "module": "request_loaders._sitemap_request_loader", + "name": "fetch_next_request", + "parsedDocstring": { + "text": "Fetch the next request to process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 312, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L312" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Fetch the next request to process." + } + ] + }, + "flags": {}, + "id": 90130, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "fetch_next_request", + "parameters": [], + "type": { + "name": "Request | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "Request", + "target": "405", + "ref": { + "id": 88167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2328, + "type": "reference", + "ref": { + "id": 90090, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "parentId": 2367 + } + ], + "overwrites": { + "name": "RequestLoader.fetch_next_request", + "target": 2328, + "type": "reference", + "ref": { + "id": 90090, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 45 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#fetch_next_request" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#fetch_next_request", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "decorations": [ + { + "name": "override" + } + ], + "flags": {}, + "groups": [], + "id": 90131, + "module": "request_loaders._sitemap_request_loader", + "name": "mark_request_as_handled", + "parsedDocstring": { + "text": "Mark a request as successfully handled." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 340, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L340" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Mark a request as successfully handled." + } + ] + }, + "flags": {}, + "id": 90132, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "mark_request_as_handled", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90133, + "kind": 32768, + "kindString": "Parameter", + "name": "request", + "type": { + "name": "Request", + "type": "reference", + "target": "405", + "ref": { + "id": 88167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_request.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 145 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/Request" + } + }, + "parentId": 2370 + } + ], + "type": { + "name": "ProcessedRequest | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "ProcessedRequest", + "target": "1015", + "ref": { + "id": 88777, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/storage_clients/models.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 140 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/ProcessedRequest" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2330, + "type": "reference", + "ref": { + "id": 90092, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "parentId": 2369 + } + ], + "overwrites": { + "name": "RequestLoader.mark_request_as_handled", + "target": 2330, + "type": "reference", + "ref": { + "id": 90092, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 53 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#mark_request_as_handled" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#mark_request_as_handled", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90134, + "module": "request_loaders._sitemap_request_loader", + "name": "abort_loading", + "parsedDocstring": { + "text": "Abort the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 348, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L348" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Abort the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 90135, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "abort_loading", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2372 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#abort_loading", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90136, + "module": "request_loaders._sitemap_request_loader", + "name": "start", + "parsedDocstring": { + "text": "Start the sitemap loading process." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 355, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L355" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Start the sitemap loading process." + } + ] + }, + "flags": {}, + "id": 90137, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "start", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2374 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#start", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90138, + "module": "request_loaders._sitemap_request_loader", + "name": "close", + "parsedDocstring": { + "text": "Close the request loader." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 361, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L361" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Close the request loader." + } + ] + }, + "flags": {}, + "id": 90139, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "close", + "parameters": [], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2376 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#close", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90140, + "module": "request_loaders._sitemap_request_loader", + "name": "__aenter__", + "parsedDocstring": { + "text": "Enter the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 366, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L366" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Enter the context manager." + } + ] + }, + "flags": {}, + "id": 90141, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aenter__", + "parameters": [], + "type": { + "name": "SitemapRequestLoader", + "type": "reference", + "target": "2348", + "ref": { + "id": 90110, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/SitemapRequestLoader" + } + }, + "parentId": 2378 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aenter__", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 90142, + "module": "request_loaders._sitemap_request_loader", + "name": "__aexit__", + "parsedDocstring": { + "text": "Exit the context manager." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 371, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L371" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Exit the context manager." + } + ] + }, + "flags": {}, + "id": 90143, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "__aexit__", + "parameters": [ + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90144, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_type", + "type": { + "name": "type[BaseException] | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "type", + "typeArguments": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + } + ], + "target": "3134", + "ref": { + "id": 90896, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/_utils/sitemap.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 62 + } + ], + "kind": 1024, + "permalink": "https://crawlee.dev/python/api/class/SitemapSource#type" + } + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2381 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90145, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_value", + "type": { + "name": "BaseException | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "BaseException", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2381 + }, + { + "flags": { + "isOptional": false, + "keyword-only": false + }, + "id": 90146, + "kind": 32768, + "kindString": "Parameter", + "name": "exc_traceback", + "type": { + "name": "TracebackType | None", + "type": "union", + "types": [ + { + "type": "reference", + "name": "TracebackType", + "target": 0, + "ref": {} + }, + { + "type": "literal", + "value": {} + } + ] + }, + "parentId": 2381 + } + ], + "type": { + "name": "None", + "type": "literal", + "value": {} + }, + "parentId": 2380 + } + ], + "permalink": "/python/api/class/SitemapRequestLoader#__aexit__", + "parentId": 2348 + }, + { + "kind": 2048, + "kindString": "Method", + "children": [], + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "decorations": [], + "flags": {}, + "groups": [], + "id": 91694, + "module": "request_loaders._request_loader", + "name": "to_tandem", + "parsedDocstring": { + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n", + "args": { + "request_manager": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_request_loader.py#L56" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "signatures": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Combine the loader with a request manager to support adding and reclaiming requests.\n" + } + ] + }, + "flags": {}, + "id": 90096, + "kind": 4096, + "kindString": "Call signature", + "modifiers": [ + "async" + ], + "name": "to_tandem", + "parameters": [ + { + "comment": { + "summary": [ + { + "kind": "text", + "text": "Request manager to combine the loader with.\nIf None is given, the default request queue is used." + } + ] + }, + "defaultValue": "None", + "flags": { + "isOptional": true, + "keyword-only": false + }, + "id": 90097, + "kind": 32768, + "kindString": "Parameter", + "name": "request_manager", + "type": { + "name": "RequestManager | None", + "type": "reference", + "target": 0, + "ref": {} + }, + "parentId": 2334 + } + ], + "type": { + "name": "RequestManagerTandem", + "type": "reference", + "target": "2405", + "ref": { + "id": 90167, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_manager_tandem.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 24 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestManagerTandem" + } + }, + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 90095, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "parentId": 3932 + } + ], + "inheritedFrom": { + "name": "RequestLoader.to_tandem", + "target": 2333, + "type": "reference", + "ref": { + "id": 90095, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 56 + } + ], + "kind": 2048, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader#to_tandem" + } + }, + "permalink": "/python/api/class/SitemapRequestLoader#to_tandem", + "parentId": 2348 + } + ], + "comment": { + "summary": [ + { + "kind": "text", + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + } + ] + }, + "decorations": [ + { + "args": "('Request loaders')", + "name": "docs_group" + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 90140, + 90142, + 90111, + 90134, + 90138, + 90129, + 90123, + 90121, + 90125, + 90127, + 90131, + 90136, + 91694 + ], + "title": "Methods" + } + ], + "id": 90110, + "module": "request_loaders._sitemap_request_loader", + "name": "SitemapRequestLoader", + "parsedDocstring": { + "text": "A request loader that reads URLs from sitemap(s).\n\nThe loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol\n(https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.\nNote that HTML pages containing links are not supported - those should be handled by regular crawlers\nand the `enqueue_links` functionality.\n\nThe loader fetches and parses sitemaps in the background, allowing crawling to start\nbefore all URLs are loaded. It supports filtering URLs using glob and regex patterns.\n\nThe loader supports state persistence, allowing it to resume from where it left off\nafter interruption when a `persist_state_key` is provided during initialization." + }, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_sitemap_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 91, + "url": "https://github.com/apify/crawlee-python/blob/8812d480071f1c9b9561f0c3b667c6d4622884b1//src/crawlee/request_loaders/_sitemap_request_loader.py#L91" + } + ], + "type": { + "name": "Undefined", + "type": "reference", + "target": 0, + "ref": {} + }, + "extendedTypes": [ + { + "name": "RequestLoader", + "target": "2319", + "type": "reference", + "ref": { + "id": 90081, + "sources": [ + { + "character": 1, + "fileName": "/src/crawlee/request_loaders/_request_loader.py", + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c", + "line": 17 + } + ], + "kind": 128, + "permalink": "https://crawlee.dev/python/api/class/RequestLoader" + } + } + ], + "permalink": "/python/api/class/SitemapRequestLoader", + "categories": [] + } + ], + "flags": {}, + "groups": [ + { + "children": [ + 287, + 196, + 98, + 124, + 105, + 132, + 146, + 80 + ], + "title": "Actor" + }, + { + "children": [ + 545, + 541, + 513 + ], + "title": "Charging" + }, + { + "children": [ + 231, + 43, + 40 + ], + "title": "Configuration" + }, + { + "children": [ + 622, + 51454, + 54217, + 56975, + 59949, + 62704 + ], + "title": "Event data" + }, + { + "children": [ + 660, + 41531, + 44736 + ], + "title": "Event managers" + }, + { + "children": [ + 643, + 652, + 649, + 646, + 640, + 634, + 637, + 655, + 48682 + ], + "title": "Events" + }, + { + "children": [ + 683, + 79752, + 82267, + 86226, + 90110 + ], + "title": "Request loaders" + }, + { + "children": [ + 797, + 1124, + 64081, + 65280, + 70113, + 73800 + ], + "title": "Storage clients" + }, + { + "children": [ + 933, + 935, + 938, + 17047, + 18058, + 19053, + 23387, + 27736, + 32057, + 33077, + 33487, + 34539, + 38868 + ], + "title": "Storage data" + }, + { + "title": "Storages", + "children": [ + 4734, + 8442, + 12003, + 15746 + ] + } + ], + "id": 0, + "kind": 1, + "kindString": "Project", + "name": "apify-client", + "sources": [ + { + "character": 0, + "fileName": "src/index.ts", + "line": 1, + "gitRevision": "5a370233cd012a3c9536d864c9fe1817c320594c" + } + ], + "symbolIdMap": { + "1": { + "qualifiedName": "EVENT_LISTENERS_TIMEOUT", + "sourceFileName": "/src/apify/_consts.py" + }, + "2": { + "qualifiedName": "BASE64_REGEXP", + "sourceFileName": "/src/apify/_consts.py" + }, + "3": { + "qualifiedName": "ENCRYPTED_STRING_VALUE_PREFIX", + "sourceFileName": "/src/apify/_consts.py" + }, + "4": { + "qualifiedName": "ENCRYPTED_JSON_VALUE_PREFIX", + "sourceFileName": "/src/apify/_consts.py" + }, + "5": { + "qualifiedName": "ENCRYPTED_INPUT_VALUE_REGEXP", + "sourceFileName": "/src/apify/_consts.py" + }, + "6": { + "qualifiedName": "ENCRYPTION_KEY_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "7": { + "qualifiedName": "ENCRYPTION_IV_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "8": { + "qualifiedName": "ENCRYPTION_AUTH_TAG_LENGTH", + "sourceFileName": "/src/apify/_crypto.py" + }, + "9": { + "qualifiedName": "public_encrypt", + "sourceFileName": "/src/apify/_crypto.py" + }, + "13": { + "qualifiedName": "private_decrypt", + "sourceFileName": "/src/apify/_crypto.py" + }, + "18": { + "qualifiedName": "load_private_key", + "sourceFileName": "/src/apify/_crypto.py" + }, + "22": { + "qualifiedName": "decrypt_input_secrets", + "sourceFileName": "/src/apify/_crypto.py" + }, + "26": { + "qualifiedName": "CHARSET", + "sourceFileName": "/src/apify/_crypto.py" + }, + "27": { + "qualifiedName": "encode_base62", + "sourceFileName": "/src/apify/_crypto.py" + }, + "30": { + "qualifiedName": "create_hmac_signature", + "sourceFileName": "/src/apify/_crypto.py" + }, + "34": { + "qualifiedName": "APIFY_PROXY_VALUE_REGEX", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "35": { + "qualifiedName": "COUNTRY_CODE_REGEX", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "36": { + "qualifiedName": "SESSION_ID_MAX_LENGTH", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "37": { + "qualifiedName": "is_url", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "40": { + "qualifiedName": "ProxyInfo", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "41": { + "qualifiedName": "groups", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "42": { + "qualifiedName": "country_code", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "43": { + "qualifiedName": "ProxyConfiguration", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "44": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "54": { + "qualifiedName": "initialize", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "56": { + "qualifiedName": "new_proxy_info", + "sourceFileName": "/src/apify/_proxy_configuration.py" + }, + "61": { + "qualifiedName": "logger_name", + "sourceFileName": "/src/apify/log.py" + }, + "62": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/log.py" + }, + "63": { + "qualifiedName": "ActorLogFormatter", + "sourceFileName": "/src/apify/log.py" + }, + "64": { + "qualifiedName": "__version__", + "sourceFileName": "/src/apify/__init__.py" + }, + "65": { + "qualifiedName": "PricingModel", + "sourceFileName": "/src/apify/_models.py" + }, + "66": { + "qualifiedName": "GeneralAccess", + "sourceFileName": "/src/apify/_models.py" + }, + "67": { + "qualifiedName": "WebhookCondition", + "sourceFileName": "/src/apify/_models.py" + }, + "68": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "69": { + "qualifiedName": "actor_id", + "sourceFileName": "/src/apify/_models.py" + }, + "70": { + "qualifiedName": "actor_task_id", + "sourceFileName": "/src/apify/_models.py" + }, + "71": { + "qualifiedName": "actor_run_id", + "sourceFileName": "/src/apify/_models.py" + }, + "72": { + "qualifiedName": "WebhookDispatchStatus", + "sourceFileName": "/src/apify/_models.py" + }, + "73": { + "qualifiedName": "ExampleWebhookDispatch", + "sourceFileName": "/src/apify/_models.py" + }, + "74": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "75": { + "qualifiedName": "status", + "sourceFileName": "/src/apify/_models.py" + }, + "76": { + "qualifiedName": "finished_at", + "sourceFileName": "/src/apify/_models.py" + }, + "77": { + "qualifiedName": "WebhookStats", + "sourceFileName": "/src/apify/_models.py" + }, + "78": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "79": { + "qualifiedName": "total_dispatches", + "sourceFileName": "/src/apify/_models.py" + }, + "80": { + "qualifiedName": "Webhook", + "sourceFileName": "/src/apify/_models.py" + }, + "81": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "82": { + "qualifiedName": "event_types", + "sourceFileName": "/src/apify/_models.py" + }, + "83": { + "qualifiedName": "request_url", + "sourceFileName": "/src/apify/_models.py" + }, + "84": { + "qualifiedName": "id", + "sourceFileName": "/src/apify/_models.py" + }, + "85": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "86": { + "qualifiedName": "modified_at", + "sourceFileName": "/src/apify/_models.py" + }, + "87": { + "qualifiedName": "user_id", + "sourceFileName": "/src/apify/_models.py" + }, + "88": { + "qualifiedName": "is_ad_hoc", + "sourceFileName": "/src/apify/_models.py" + }, + "89": { + "qualifiedName": "should_interpolate_strings", + "sourceFileName": "/src/apify/_models.py" + }, + "90": { + "qualifiedName": "condition", + "sourceFileName": "/src/apify/_models.py" + }, + "91": { + "qualifiedName": "ignore_ssl_errors", + "sourceFileName": "/src/apify/_models.py" + }, + "92": { + "qualifiedName": "do_not_retry", + "sourceFileName": "/src/apify/_models.py" + }, + "93": { + "qualifiedName": "payload_template", + "sourceFileName": "/src/apify/_models.py" + }, + "94": { + "qualifiedName": "headers_template", + "sourceFileName": "/src/apify/_models.py" + }, + "95": { + "qualifiedName": "description", + "sourceFileName": "/src/apify/_models.py" + }, + "96": { + "qualifiedName": "last_dispatch", + "sourceFileName": "/src/apify/_models.py" + }, + "97": { + "qualifiedName": "stats", + "sourceFileName": "/src/apify/_models.py" + }, + "98": { + "qualifiedName": "ActorRunMeta", + "sourceFileName": "/src/apify/_models.py" + }, + "99": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "100": { + "qualifiedName": "origin", + "sourceFileName": "/src/apify/_models.py" + }, + "101": { + "qualifiedName": "client_ip", + "sourceFileName": "/src/apify/_models.py" + }, + "102": { + "qualifiedName": "user_agent", + "sourceFileName": "/src/apify/_models.py" + }, + "103": { + "qualifiedName": "schedule_id", + "sourceFileName": "/src/apify/_models.py" + }, + "104": { + "qualifiedName": "scheduled_at", + "sourceFileName": "/src/apify/_models.py" + }, + "105": { + "qualifiedName": "ActorRunStats", + "sourceFileName": "/src/apify/_models.py" + }, + "106": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "107": { + "qualifiedName": "input_body_len", + "sourceFileName": "/src/apify/_models.py" + }, + "108": { + "qualifiedName": "migration_count", + "sourceFileName": "/src/apify/_models.py" + }, + "109": { + "qualifiedName": "reboot_count", + "sourceFileName": "/src/apify/_models.py" + }, + "110": { + "qualifiedName": "restart_count", + "sourceFileName": "/src/apify/_models.py" + }, + "111": { + "qualifiedName": "resurrect_count", + "sourceFileName": "/src/apify/_models.py" + }, + "112": { + "qualifiedName": "mem_avg_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "113": { + "qualifiedName": "mem_max_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "114": { + "qualifiedName": "mem_current_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "115": { + "qualifiedName": "cpu_avg_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "116": { + "qualifiedName": "cpu_max_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "117": { + "qualifiedName": "cpu_current_usage", + "sourceFileName": "/src/apify/_models.py" + }, + "118": { + "qualifiedName": "net_rx_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "119": { + "qualifiedName": "net_tx_bytes", + "sourceFileName": "/src/apify/_models.py" + }, + "120": { + "qualifiedName": "duration_millis", + "sourceFileName": "/src/apify/_models.py" + }, + "121": { + "qualifiedName": "run_time_secs", + "sourceFileName": "/src/apify/_models.py" + }, + "122": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/_models.py" + }, + "123": { + "qualifiedName": "compute_units", + "sourceFileName": "/src/apify/_models.py" + }, + "124": { + "qualifiedName": "ActorRunOptions", + "sourceFileName": "/src/apify/_models.py" + }, + "125": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "126": { + "qualifiedName": "build", + "sourceFileName": "/src/apify/_models.py" + }, + "127": { + "qualifiedName": "timeout_secs", + "sourceFileName": "/src/apify/_models.py" + }, + "128": { + "qualifiedName": "memory_mbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "129": { + "qualifiedName": "disk_mbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "130": { + "qualifiedName": "max_items", + "sourceFileName": "/src/apify/_models.py" + }, + "131": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "132": { + "qualifiedName": "ActorRunUsage", + "sourceFileName": "/src/apify/_models.py" + }, + "133": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "134": { + "qualifiedName": "actor_compute_units", + "sourceFileName": "/src/apify/_models.py" + }, + "135": { + "qualifiedName": "dataset_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "136": { + "qualifiedName": "dataset_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "137": { + "qualifiedName": "key_value_store_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "138": { + "qualifiedName": "key_value_store_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "139": { + "qualifiedName": "key_value_store_lists", + "sourceFileName": "/src/apify/_models.py" + }, + "140": { + "qualifiedName": "request_queue_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "141": { + "qualifiedName": "request_queue_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "142": { + "qualifiedName": "data_transfer_internal_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "143": { + "qualifiedName": "data_transfer_external_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "144": { + "qualifiedName": "proxy_residential_transfer_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "145": { + "qualifiedName": "proxy_serps", + "sourceFileName": "/src/apify/_models.py" + }, + "146": { + "qualifiedName": "ActorRunUsageUsd", + "sourceFileName": "/src/apify/_models.py" + }, + "147": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "148": { + "qualifiedName": "actor_compute_units", + "sourceFileName": "/src/apify/_models.py" + }, + "149": { + "qualifiedName": "dataset_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "150": { + "qualifiedName": "dataset_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "151": { + "qualifiedName": "key_value_store_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "152": { + "qualifiedName": "key_value_store_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "153": { + "qualifiedName": "key_value_store_lists", + "sourceFileName": "/src/apify/_models.py" + }, + "154": { + "qualifiedName": "request_queue_reads", + "sourceFileName": "/src/apify/_models.py" + }, + "155": { + "qualifiedName": "request_queue_writes", + "sourceFileName": "/src/apify/_models.py" + }, + "156": { + "qualifiedName": "data_transfer_internal_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "157": { + "qualifiedName": "data_transfer_external_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "158": { + "qualifiedName": "proxy_residential_transfer_gbytes", + "sourceFileName": "/src/apify/_models.py" + }, + "159": { + "qualifiedName": "proxy_serps", + "sourceFileName": "/src/apify/_models.py" + }, + "160": { + "qualifiedName": "Metamorph", + "sourceFileName": "/src/apify/_models.py" + }, + "161": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "162": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "163": { + "qualifiedName": "actor_id", + "sourceFileName": "/src/apify/_models.py" + }, + "164": { + "qualifiedName": "build_id", + "sourceFileName": "/src/apify/_models.py" + }, + "165": { + "qualifiedName": "input_key", + "sourceFileName": "/src/apify/_models.py" + }, + "166": { + "qualifiedName": "CommonActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "167": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "168": { + "qualifiedName": "apify_margin_percentage", + "sourceFileName": "/src/apify/_models.py" + }, + "169": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "170": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "171": { + "qualifiedName": "notified_about_future_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "172": { + "qualifiedName": "notified_about_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "173": { + "qualifiedName": "reason_for_change", + "sourceFileName": "/src/apify/_models.py" + }, + "174": { + "qualifiedName": "FreeActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "175": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "176": { + "qualifiedName": "FlatPricePerMonthActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "177": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "178": { + "qualifiedName": "trial_minutes", + "sourceFileName": "/src/apify/_models.py" + }, + "179": { + "qualifiedName": "price_per_unit_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "180": { + "qualifiedName": "PricePerDatasetItemActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "181": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "182": { + "qualifiedName": "unit_name", + "sourceFileName": "/src/apify/_models.py" + }, + "183": { + "qualifiedName": "price_per_unit_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "184": { + "qualifiedName": "ActorChargeEvent", + "sourceFileName": "/src/apify/_models.py" + }, + "185": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "186": { + "qualifiedName": "event_price_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "187": { + "qualifiedName": "event_title", + "sourceFileName": "/src/apify/_models.py" + }, + "188": { + "qualifiedName": "event_description", + "sourceFileName": "/src/apify/_models.py" + }, + "189": { + "qualifiedName": "PricingPerEvent", + "sourceFileName": "/src/apify/_models.py" + }, + "190": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "191": { + "qualifiedName": "actor_charge_events", + "sourceFileName": "/src/apify/_models.py" + }, + "192": { + "qualifiedName": "PayPerEventActorPricingInfo", + "sourceFileName": "/src/apify/_models.py" + }, + "193": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_models.py" + }, + "194": { + "qualifiedName": "pricing_per_event", + "sourceFileName": "/src/apify/_models.py" + }, + "195": { + "qualifiedName": "minimal_max_total_charge_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "196": { + "qualifiedName": "ActorRun", + "sourceFileName": "/src/apify/_models.py" + }, + "197": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "198": { + "qualifiedName": "id", + "sourceFileName": "/src/apify/_models.py" + }, + "199": { + "qualifiedName": "act_id", + "sourceFileName": "/src/apify/_models.py" + }, + "200": { + "qualifiedName": "user_id", + "sourceFileName": "/src/apify/_models.py" + }, + "201": { + "qualifiedName": "actor_task_id", + "sourceFileName": "/src/apify/_models.py" + }, + "202": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "203": { + "qualifiedName": "finished_at", + "sourceFileName": "/src/apify/_models.py" + }, + "204": { + "qualifiedName": "status", + "sourceFileName": "/src/apify/_models.py" + }, + "205": { + "qualifiedName": "status_message", + "sourceFileName": "/src/apify/_models.py" + }, + "206": { + "qualifiedName": "is_status_message_terminal", + "sourceFileName": "/src/apify/_models.py" + }, + "207": { + "qualifiedName": "meta", + "sourceFileName": "/src/apify/_models.py" + }, + "208": { + "qualifiedName": "stats", + "sourceFileName": "/src/apify/_models.py" + }, + "209": { + "qualifiedName": "options", + "sourceFileName": "/src/apify/_models.py" + }, + "210": { + "qualifiedName": "build_id", + "sourceFileName": "/src/apify/_models.py" + }, + "211": { + "qualifiedName": "exit_code", + "sourceFileName": "/src/apify/_models.py" + }, + "212": { + "qualifiedName": "general_access", + "sourceFileName": "/src/apify/_models.py" + }, + "213": { + "qualifiedName": "default_key_value_store_id", + "sourceFileName": "/src/apify/_models.py" + }, + "214": { + "qualifiedName": "default_dataset_id", + "sourceFileName": "/src/apify/_models.py" + }, + "215": { + "qualifiedName": "default_request_queue_id", + "sourceFileName": "/src/apify/_models.py" + }, + "216": { + "qualifiedName": "build_number", + "sourceFileName": "/src/apify/_models.py" + }, + "217": { + "qualifiedName": "container_url", + "sourceFileName": "/src/apify/_models.py" + }, + "218": { + "qualifiedName": "is_container_server_ready", + "sourceFileName": "/src/apify/_models.py" + }, + "219": { + "qualifiedName": "git_branch_name", + "sourceFileName": "/src/apify/_models.py" + }, + "220": { + "qualifiedName": "usage", + "sourceFileName": "/src/apify/_models.py" + }, + "221": { + "qualifiedName": "usage_total_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "222": { + "qualifiedName": "usage_usd", + "sourceFileName": "/src/apify/_models.py" + }, + "223": { + "qualifiedName": "pricing_info", + "sourceFileName": "/src/apify/_models.py" + }, + "224": { + "qualifiedName": "charged_event_counts", + "sourceFileName": "/src/apify/_models.py" + }, + "225": { + "qualifiedName": "metamorphs", + "sourceFileName": "/src/apify/_models.py" + }, + "226": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/_configuration.py" + }, + "227": { + "qualifiedName": "ActorStorages", + "sourceFileName": "/src/apify/_configuration.py" + }, + "228": { + "qualifiedName": "key_value_stores", + "sourceFileName": "/src/apify/_configuration.py" + }, + "229": { + "qualifiedName": "datasets", + "sourceFileName": "/src/apify/_configuration.py" + }, + "230": { + "qualifiedName": "request_queues", + "sourceFileName": "/src/apify/_configuration.py" + }, + "231": { + "qualifiedName": "Configuration", + "sourceFileName": "/src/apify/_configuration.py" + }, + "232": { + "qualifiedName": "actor_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "233": { + "qualifiedName": "actor_full_name", + "sourceFileName": "/src/apify/_configuration.py" + }, + "234": { + "qualifiedName": "actor_permission_level", + "sourceFileName": "/src/apify/_configuration.py" + }, + "235": { + "qualifiedName": "actor_run_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "236": { + "qualifiedName": "actor_build_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "237": { + "qualifiedName": "actor_build_number", + "sourceFileName": "/src/apify/_configuration.py" + }, + "238": { + "qualifiedName": "actor_build_tags", + "sourceFileName": "/src/apify/_configuration.py" + }, + "239": { + "qualifiedName": "actor_task_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "240": { + "qualifiedName": "actor_events_ws_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "241": { + "qualifiedName": "api_base_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "242": { + "qualifiedName": "api_public_base_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "243": { + "qualifiedName": "dedicated_cpus", + "sourceFileName": "/src/apify/_configuration.py" + }, + "244": { + "qualifiedName": "default_dataset_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "245": { + "qualifiedName": "default_key_value_store_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "246": { + "qualifiedName": "default_request_queue_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "247": { + "qualifiedName": "disable_outdated_warning", + "sourceFileName": "/src/apify/_configuration.py" + }, + "248": { + "qualifiedName": "fact", + "sourceFileName": "/src/apify/_configuration.py" + }, + "249": { + "qualifiedName": "input_key", + "sourceFileName": "/src/apify/_configuration.py" + }, + "250": { + "qualifiedName": "input_secrets_private_key_file", + "sourceFileName": "/src/apify/_configuration.py" + }, + "251": { + "qualifiedName": "input_secrets_private_key_passphrase", + "sourceFileName": "/src/apify/_configuration.py" + }, + "252": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/_configuration.py" + }, + "253": { + "qualifiedName": "latest_sdk_version", + "sourceFileName": "/src/apify/_configuration.py" + }, + "254": { + "qualifiedName": "log_format", + "sourceFileName": "/src/apify/_configuration.py" + }, + "255": { + "qualifiedName": "max_paid_dataset_items", + "sourceFileName": "/src/apify/_configuration.py" + }, + "256": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_configuration.py" + }, + "257": { + "qualifiedName": "test_pay_per_event", + "sourceFileName": "/src/apify/_configuration.py" + }, + "258": { + "qualifiedName": "meta_origin", + "sourceFileName": "/src/apify/_configuration.py" + }, + "259": { + "qualifiedName": "metamorph_after_sleep", + "sourceFileName": "/src/apify/_configuration.py" + }, + "260": { + "qualifiedName": "proxy_hostname", + "sourceFileName": "/src/apify/_configuration.py" + }, + "261": { + "qualifiedName": "proxy_password", + "sourceFileName": "/src/apify/_configuration.py" + }, + "262": { + "qualifiedName": "proxy_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "263": { + "qualifiedName": "proxy_status_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "264": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_configuration.py" + }, + "265": { + "qualifiedName": "timeout_at", + "sourceFileName": "/src/apify/_configuration.py" + }, + "266": { + "qualifiedName": "standby_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "267": { + "qualifiedName": "standby_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "268": { + "qualifiedName": "token", + "sourceFileName": "/src/apify/_configuration.py" + }, + "269": { + "qualifiedName": "user_id", + "sourceFileName": "/src/apify/_configuration.py" + }, + "270": { + "qualifiedName": "user_is_paying", + "sourceFileName": "/src/apify/_configuration.py" + }, + "271": { + "qualifiedName": "web_server_port", + "sourceFileName": "/src/apify/_configuration.py" + }, + "272": { + "qualifiedName": "web_server_url", + "sourceFileName": "/src/apify/_configuration.py" + }, + "273": { + "qualifiedName": "workflow_key", + "sourceFileName": "/src/apify/_configuration.py" + }, + "274": { + "qualifiedName": "actor_pricing_info", + "sourceFileName": "/src/apify/_configuration.py" + }, + "275": { + "qualifiedName": "charged_event_counts", + "sourceFileName": "/src/apify/_configuration.py" + }, + "276": { + "qualifiedName": "actor_storages", + "sourceFileName": "/src/apify/_configuration.py" + }, + "277": { + "qualifiedName": "disable_browser_sandbox_on_platform", + "sourceFileName": "/src/apify/_configuration.py" + }, + "279": { + "qualifiedName": "canonical_input_key", + "sourceFileName": "/src/apify/_configuration.py" + }, + "280": { + "qualifiedName": "input_key_candidates", + "sourceFileName": "/src/apify/_configuration.py" + }, + "281": { + "qualifiedName": "get_global_configuration", + "sourceFileName": "/src/apify/_configuration.py" + }, + "283": { + "qualifiedName": "from_configuration", + "sourceFileName": "/src/apify/_configuration.py" + }, + "286": { + "qualifiedName": "MainReturnType", + "sourceFileName": "/src/apify/_actor.py" + }, + "287": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/_actor.py" + }, + "288": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_actor.py" + }, + "297": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/_actor.py" + }, + "299": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/_actor.py" + }, + "304": { + "qualifiedName": "__repr__", + "sourceFileName": "/src/apify/_actor.py" + }, + "306": { + "qualifiedName": "__call__", + "sourceFileName": "/src/apify/_actor.py" + }, + "315": { + "qualifiedName": "log", + "sourceFileName": "/src/apify/_actor.py" + }, + "316": { + "qualifiedName": "exit_code", + "sourceFileName": "/src/apify/_actor.py" + }, + "317": { + "qualifiedName": "exit_code", + "sourceFileName": "/src/apify/_actor.py" + }, + "320": { + "qualifiedName": "status_message", + "sourceFileName": "/src/apify/_actor.py" + }, + "321": { + "qualifiedName": "status_message", + "sourceFileName": "/src/apify/_actor.py" + }, + "324": { + "qualifiedName": "apify_client", + "sourceFileName": "/src/apify/_actor.py" + }, + "325": { + "qualifiedName": "configuration", + "sourceFileName": "/src/apify/_actor.py" + }, + "327": { + "qualifiedName": "event_manager", + "sourceFileName": "/src/apify/_actor.py" + }, + "329": { + "qualifiedName": "init", + "sourceFileName": "/src/apify/_actor.py" + }, + "331": { + "qualifiedName": "exit", + "sourceFileName": "/src/apify/_actor.py" + }, + "337": { + "qualifiedName": "fail", + "sourceFileName": "/src/apify/_actor.py" + }, + "342": { + "qualifiedName": "new_client", + "sourceFileName": "/src/apify/_actor.py" + }, + "349": { + "qualifiedName": "open_dataset", + "sourceFileName": "/src/apify/_actor.py" + }, + "355": { + "qualifiedName": "open_key_value_store", + "sourceFileName": "/src/apify/_actor.py" + }, + "361": { + "qualifiedName": "open_request_queue", + "sourceFileName": "/src/apify/_actor.py" + }, + "367": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/_actor.py" + }, + "371": { + "qualifiedName": "get_input", + "sourceFileName": "/src/apify/_actor.py" + }, + "373": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/_actor.py" + }, + "377": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/_actor.py" + }, + "382": { + "qualifiedName": "get_charging_manager", + "sourceFileName": "/src/apify/_actor.py" + }, + "384": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_actor.py" + }, + "388": { + "qualifiedName": "on", + "sourceFileName": "/src/apify/_actor.py" + }, + "392": { + "qualifiedName": "off", + "sourceFileName": "/src/apify/_actor.py" + }, + "396": { + "qualifiedName": "is_at_home", + "sourceFileName": "/src/apify/_actor.py" + }, + "398": { + "qualifiedName": "get_env", + "sourceFileName": "/src/apify/_actor.py" + }, + "400": { + "qualifiedName": "start", + "sourceFileName": "/src/apify/_actor.py" + }, + "411": { + "qualifiedName": "abort", + "sourceFileName": "/src/apify/_actor.py" + }, + "417": { + "qualifiedName": "call", + "sourceFileName": "/src/apify/_actor.py" + }, + "429": { + "qualifiedName": "call_task", + "sourceFileName": "/src/apify/_actor.py" + }, + "439": { + "qualifiedName": "metamorph", + "sourceFileName": "/src/apify/_actor.py" + }, + "446": { + "qualifiedName": "reboot", + "sourceFileName": "/src/apify/_actor.py" + }, + "450": { + "qualifiedName": "add_webhook", + "sourceFileName": "/src/apify/_actor.py" + }, + "456": { + "qualifiedName": "set_status_message", + "sourceFileName": "/src/apify/_actor.py" + }, + "460": { + "qualifiedName": "create_proxy_configuration", + "sourceFileName": "/src/apify/_actor.py" + }, + "468": { + "qualifiedName": "use_state", + "sourceFileName": "/src/apify/_actor.py" + }, + "509": { + "qualifiedName": "Actor", + "sourceFileName": "/src/apify/_actor.py" + }, + "510": { + "qualifiedName": "run_validator", + "sourceFileName": "/src/apify/_charging.py" + }, + "511": { + "qualifiedName": "DEFAULT_DATASET_ITEM_EVENT", + "sourceFileName": "/src/apify/_charging.py" + }, + "512": { + "qualifiedName": "charging_manager_ctx", + "sourceFileName": "/src/apify/_charging.py" + }, + "513": { + "qualifiedName": "ChargingManager", + "sourceFileName": "/src/apify/_charging.py" + }, + "514": { + "qualifiedName": "charge_lock", + "sourceFileName": "/src/apify/_charging.py" + }, + "515": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_charging.py" + }, + "519": { + "qualifiedName": "calculate_total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "521": { + "qualifiedName": "calculate_max_event_charge_count_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "524": { + "qualifiedName": "get_pricing_info", + "sourceFileName": "/src/apify/_charging.py" + }, + "526": { + "qualifiedName": "get_charged_event_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "529": { + "qualifiedName": "get_max_total_charge_usd", + "sourceFileName": "/src/apify/_charging.py" + }, + "531": { + "qualifiedName": "compute_push_data_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "536": { + "qualifiedName": "is_event_charge_limit_reached", + "sourceFileName": "/src/apify/_charging.py" + }, + "539": { + "qualifiedName": "compute_chargeable", + "sourceFileName": "/src/apify/_charging.py" + }, + "541": { + "qualifiedName": "ChargeResult", + "sourceFileName": "/src/apify/_charging.py" + }, + "542": { + "qualifiedName": "event_charge_limit_reached", + "sourceFileName": "/src/apify/_charging.py" + }, + "543": { + "qualifiedName": "charged_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "544": { + "qualifiedName": "chargeable_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "545": { + "qualifiedName": "ActorPricingInfo", + "sourceFileName": "/src/apify/_charging.py" + }, + "546": { + "qualifiedName": "pricing_model", + "sourceFileName": "/src/apify/_charging.py" + }, + "547": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_charging.py" + }, + "548": { + "qualifiedName": "is_pay_per_event", + "sourceFileName": "/src/apify/_charging.py" + }, + "549": { + "qualifiedName": "per_event_prices", + "sourceFileName": "/src/apify/_charging.py" + }, + "550": { + "qualifiedName": "ChargingManagerImplementation", + "sourceFileName": "/src/apify/_charging.py" + }, + "551": { + "qualifiedName": "LOCAL_CHARGING_LOG_DATASET_NAME", + "sourceFileName": "/src/apify/_charging.py" + }, + "552": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_charging.py" + }, + "556": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/_charging.py" + }, + "558": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/_charging.py" + }, + "563": { + "qualifiedName": "charge", + "sourceFileName": "/src/apify/_charging.py" + }, + "567": { + "qualifiedName": "calculate_total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "569": { + "qualifiedName": "calculate_max_event_charge_count_within_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "572": { + "qualifiedName": "get_pricing_info", + "sourceFileName": "/src/apify/_charging.py" + }, + "574": { + "qualifiedName": "get_charged_event_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "577": { + "qualifiedName": "get_max_total_charge_usd", + "sourceFileName": "/src/apify/_charging.py" + }, + "579": { + "qualifiedName": "compute_push_data_limit", + "sourceFileName": "/src/apify/_charging.py" + }, + "584": { + "qualifiedName": "is_event_charge_limit_reached", + "sourceFileName": "/src/apify/_charging.py" + }, + "587": { + "qualifiedName": "compute_chargeable", + "sourceFileName": "/src/apify/_charging.py" + }, + "589": { + "qualifiedName": "ChargingStateItem", + "sourceFileName": "/src/apify/_charging.py" + }, + "590": { + "qualifiedName": "charge_count", + "sourceFileName": "/src/apify/_charging.py" + }, + "591": { + "qualifiedName": "total_charged_amount", + "sourceFileName": "/src/apify/_charging.py" + }, + "592": { + "qualifiedName": "PricingInfoItem", + "sourceFileName": "/src/apify/_charging.py" + }, + "593": { + "qualifiedName": "price", + "sourceFileName": "/src/apify/_charging.py" + }, + "594": { + "qualifiedName": "title", + "sourceFileName": "/src/apify/_charging.py" + }, + "595": { + "qualifiedName": "_FetchedPricingInfoDict", + "sourceFileName": "/src/apify/_charging.py" + }, + "596": { + "qualifiedName": "pricing_info", + "sourceFileName": "/src/apify/_charging.py" + }, + "597": { + "qualifiedName": "charged_event_counts", + "sourceFileName": "/src/apify/_charging.py" + }, + "598": { + "qualifiedName": "max_total_charge_usd", + "sourceFileName": "/src/apify/_charging.py" + }, + "599": { + "qualifiedName": "T", + "sourceFileName": "/src/apify/_utils.py" + }, + "600": { + "qualifiedName": "ensure_context", + "sourceFileName": "/src/apify/_utils.py" + }, + "603": { + "qualifiedName": "get_system_info", + "sourceFileName": "/src/apify/_utils.py" + }, + "605": { + "qualifiedName": "is_running_in_ipython", + "sourceFileName": "/src/apify/_utils.py" + }, + "607": { + "qualifiedName": "GroupName", + "sourceFileName": "/src/apify/_utils.py" + }, + "608": { + "qualifiedName": "docs_group", + "sourceFileName": "/src/apify/_utils.py" + }, + "611": { + "qualifiedName": "docs_name", + "sourceFileName": "/src/apify/_utils.py" + }, + "614": { + "qualifiedName": "maybe_extract_enum_member_value", + "sourceFileName": "/src/apify/_utils.py" + }, + "617": { + "qualifiedName": "ReentrantLock", + "sourceFileName": "/src/apify/_utils.py" + }, + "618": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/_utils.py" + }, + "620": { + "qualifiedName": "__call__", + "sourceFileName": "/src/apify/_utils.py" + }, + "622": { + "qualifiedName": "SystemInfoEventData", + "sourceFileName": "/src/apify/events/_types.py" + }, + "623": { + "qualifiedName": "mem_avg_bytes", + "sourceFileName": "/src/apify/events/_types.py" + }, + "624": { + "qualifiedName": "mem_current_bytes", + "sourceFileName": "/src/apify/events/_types.py" + }, + "625": { + "qualifiedName": "mem_max_bytes", + "sourceFileName": "/src/apify/events/_types.py" + }, + "626": { + "qualifiedName": "cpu_avg_usage", + "sourceFileName": "/src/apify/events/_types.py" + }, + "627": { + "qualifiedName": "cpu_max_usage", + "sourceFileName": "/src/apify/events/_types.py" + }, + "628": { + "qualifiedName": "cpu_current_usage", + "sourceFileName": "/src/apify/events/_types.py" + }, + "629": { + "qualifiedName": "is_cpu_overloaded", + "sourceFileName": "/src/apify/events/_types.py" + }, + "630": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/events/_types.py" + }, + "631": { + "qualifiedName": "to_crawlee_format", + "sourceFileName": "/src/apify/events/_types.py" + }, + "634": { + "qualifiedName": "PersistStateEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "635": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "636": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "637": { + "qualifiedName": "SystemInfoEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "638": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "639": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "640": { + "qualifiedName": "MigratingEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "641": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "642": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "643": { + "qualifiedName": "AbortingEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "644": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "645": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "646": { + "qualifiedName": "ExitEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "647": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "648": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "649": { + "qualifiedName": "EventWithoutData", + "sourceFileName": "/src/apify/events/_types.py" + }, + "650": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "651": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "652": { + "qualifiedName": "DeprecatedEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "653": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "654": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "655": { + "qualifiedName": "UnknownEvent", + "sourceFileName": "/src/apify/events/_types.py" + }, + "656": { + "qualifiedName": "name", + "sourceFileName": "/src/apify/events/_types.py" + }, + "657": { + "qualifiedName": "data", + "sourceFileName": "/src/apify/events/_types.py" + }, + "658": { + "qualifiedName": "EventMessage", + "sourceFileName": "/src/apify/events/_types.py" + }, + "659": { + "qualifiedName": "event_data_adapter", + "sourceFileName": "/src/apify/events/_apify_event_manager.py" + }, + "660": { + "qualifiedName": "ApifyEventManager", + "sourceFileName": "/src/apify/events/_apify_event_manager.py" + }, + "661": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/events/_apify_event_manager.py" + }, + "665": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/events/_apify_event_manager.py" + }, + "667": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/events/_apify_event_manager.py" + }, + "672": { + "qualifiedName": "URL_NO_COMMAS_REGEX", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "673": { + "qualifiedName": "_RequestDetails", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "674": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "675": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "676": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "677": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "678": { + "qualifiedName": "_RequestsFromUrlInput", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "679": { + "qualifiedName": "requests_from_url", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "680": { + "qualifiedName": "_SimpleUrlInput", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "681": { + "qualifiedName": "url", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "682": { + "qualifiedName": "url_input_adapter", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "683": { + "qualifiedName": "ApifyRequestList", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "684": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "689": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "690": { + "qualifiedName": "ApifyCacheStorage", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "691": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "694": { + "qualifiedName": "open_spider", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "697": { + "qualifiedName": "close_spider", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "701": { + "qualifiedName": "retrieve_response", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "706": { + "qualifiedName": "store_response", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "711": { + "qualifiedName": "to_gzip", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "715": { + "qualifiedName": "from_gzip", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "718": { + "qualifiedName": "read_gzip_time", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "721": { + "qualifiedName": "get_kvs_name", + "sourceFileName": "/src/apify/scrapy/extensions/_httpcache.py" + }, + "725": { + "qualifiedName": "ApifyHttpProxyMiddleware", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "726": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "729": { + "qualifiedName": "from_crawler", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "732": { + "qualifiedName": "process_request", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "736": { + "qualifiedName": "process_exception", + "sourceFileName": "/src/apify/scrapy/middlewares/apify_proxy.py" + }, + "741": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "742": { + "qualifiedName": "ActorDatasetPushPipeline", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "743": { + "qualifiedName": "process_item", + "sourceFileName": "/src/apify/scrapy/pipelines/actor_dataset_push.py" + }, + "746": { + "qualifiedName": "get_basic_auth_header", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "751": { + "qualifiedName": "apply_apify_settings", + "sourceFileName": "/src/apify/scrapy/utils.py" + }, + "755": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "756": { + "qualifiedName": "AsyncThread", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "757": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "759": { + "qualifiedName": "run_coro", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "763": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/scrapy/_async_thread.py" + }, + "766": { + "qualifiedName": "initialize_logging", + "sourceFileName": "/src/apify/scrapy/_logging_config.py" + }, + "768": { + "qualifiedName": "run_scrapy_actor", + "sourceFileName": "/src/apify/scrapy/_actor_runner.py" + }, + "771": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "772": { + "qualifiedName": "ApifyScheduler", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "773": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "775": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "778": { + "qualifiedName": "close", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "781": { + "qualifiedName": "has_pending_requests", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "783": { + "qualifiedName": "enqueue_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "786": { + "qualifiedName": "next_request", + "sourceFileName": "/src/apify/scrapy/scheduler.py" + }, + "788": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "789": { + "qualifiedName": "to_apify_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "793": { + "qualifiedName": "to_scrapy_request", + "sourceFileName": "/src/apify/scrapy/requests.py" + }, + "797": { + "qualifiedName": "ApifyStorageClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "798": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "801": { + "qualifiedName": "create_dataset_client", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "807": { + "qualifiedName": "create_kvs_client", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "813": { + "qualifiedName": "create_rq_client", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "819": { + "qualifiedName": "get_storage_client_cache_key", + "sourceFileName": "/src/apify/storage_clients/_apify/_storage_client.py" + }, + "822": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "823": { + "qualifiedName": "ApifyKeyValueStoreClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "824": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "829": { + "qualifiedName": "get_metadata", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "831": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "837": { + "qualifiedName": "purge", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "839": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "841": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "844": { + "qualifiedName": "set_value", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "849": { + "qualifiedName": "delete_value", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "852": { + "qualifiedName": "iterate_keys", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "856": { + "qualifiedName": "record_exists", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "859": { + "qualifiedName": "get_public_url", + "sourceFileName": "/src/apify/storage_clients/_apify/_key_value_store_client.py" + }, + "862": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "863": { + "qualifiedName": "ApifyRequestQueueClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "864": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "869": { + "qualifiedName": "get_metadata", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "871": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "878": { + "qualifiedName": "purge", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "880": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "882": { + "qualifiedName": "add_batch_of_requests", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "886": { + "qualifiedName": "fetch_next_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "888": { + "qualifiedName": "mark_request_as_handled", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "891": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "894": { + "qualifiedName": "reclaim_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "898": { + "qualifiedName": "is_empty", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_client.py" + }, + "900": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "901": { + "qualifiedName": "ApifyRequestQueueSharedClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "902": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "908": { + "qualifiedName": "add_batch_of_requests", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "912": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "915": { + "qualifiedName": "fetch_next_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "917": { + "qualifiedName": "mark_request_as_handled", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "920": { + "qualifiedName": "reclaim_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "924": { + "qualifiedName": "is_empty", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_shared_client.py" + }, + "926": { + "qualifiedName": "unique_key_to_request_id", + "sourceFileName": "/src/apify/storage_clients/_apify/_utils.py" + }, + "930": { + "qualifiedName": "hash_api_base_url_and_token", + "sourceFileName": "/src/apify/storage_clients/_apify/_utils.py" + }, + "933": { + "qualifiedName": "ApifyKeyValueStoreMetadata", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "934": { + "qualifiedName": "url_signing_secret_key", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "935": { + "qualifiedName": "ProlongRequestLockResponse", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "936": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "937": { + "qualifiedName": "lock_expires_at", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "938": { + "qualifiedName": "RequestQueueHead", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "939": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "940": { + "qualifiedName": "limit", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "941": { + "qualifiedName": "had_multiple_clients", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "942": { + "qualifiedName": "queue_modified_at", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "943": { + "qualifiedName": "lock_time", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "944": { + "qualifiedName": "queue_has_locked_requests", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "945": { + "qualifiedName": "items", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "946": { + "qualifiedName": "KeyValueStoreKeyInfo", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "947": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "948": { + "qualifiedName": "key", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "949": { + "qualifiedName": "size", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "950": { + "qualifiedName": "KeyValueStoreListKeysPage", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "951": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "952": { + "qualifiedName": "count", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "953": { + "qualifiedName": "limit", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "954": { + "qualifiedName": "is_truncated", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "955": { + "qualifiedName": "items", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "956": { + "qualifiedName": "exclusive_start_key", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "957": { + "qualifiedName": "next_exclusive_start_key", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "958": { + "qualifiedName": "CachedRequest", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "959": { + "qualifiedName": "id", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "960": { + "qualifiedName": "was_already_handled", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "961": { + "qualifiedName": "hydrated", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "962": { + "qualifiedName": "lock_expires_at", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "963": { + "qualifiedName": "RequestQueueStats", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "964": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "965": { + "qualifiedName": "delete_count", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "966": { + "qualifiedName": "head_item_read_count", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "967": { + "qualifiedName": "read_count", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "968": { + "qualifiedName": "storage_bytes", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "969": { + "qualifiedName": "write_count", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "970": { + "qualifiedName": "ApifyRequestQueueMetadata", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "971": { + "qualifiedName": "stats", + "sourceFileName": "/src/apify/storage_clients/_apify/_models.py" + }, + "972": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "973": { + "qualifiedName": "open_by_alias", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "980": { + "qualifiedName": "AliasResolver", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "981": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "986": { + "qualifiedName": "__aenter__", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "988": { + "qualifiedName": "__aexit__", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "993": { + "qualifiedName": "resolve_id", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "995": { + "qualifiedName": "store_mapping", + "sourceFileName": "/src/apify/storage_clients/_apify/_alias_resolving.py" + }, + "998": { + "qualifiedName": "create_storage_api_client", + "sourceFileName": "/src/apify/storage_clients/_apify/_api_client_creation.py" + }, + "1005": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1006": { + "qualifiedName": "ApifyRequestQueueSingleClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1007": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1012": { + "qualifiedName": "add_batch_of_requests", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1016": { + "qualifiedName": "get_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1019": { + "qualifiedName": "fetch_next_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1021": { + "qualifiedName": "mark_request_as_handled", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1024": { + "qualifiedName": "reclaim_request", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1028": { + "qualifiedName": "is_empty", + "sourceFileName": "/src/apify/storage_clients/_apify/_request_queue_single_client.py" + }, + "1030": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1031": { + "qualifiedName": "ApifyDatasetClient", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1032": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1037": { + "qualifiedName": "get_metadata", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1039": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1045": { + "qualifiedName": "purge", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1047": { + "qualifiedName": "drop", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1049": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1052": { + "qualifiedName": "get_data", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1065": { + "qualifiedName": "iterate_items", + "sourceFileName": "/src/apify/storage_clients/_apify/_dataset_client.py" + }, + "1076": { + "qualifiedName": "logger", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1077": { + "qualifiedName": "ApifyFileSystemKeyValueStoreClient", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1078": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1083": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1089": { + "qualifiedName": "purge", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1091": { + "qualifiedName": "get_value", + "sourceFileName": "/src/apify/storage_clients/_file_system/_key_value_store_client.py" + }, + "1094": { + "qualifiedName": "ApifyFileSystemDatasetClient", + "sourceFileName": "/src/apify/storage_clients/_file_system/_dataset_client.py" + }, + "1095": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_file_system/_dataset_client.py" + }, + "1099": { + "qualifiedName": "open", + "sourceFileName": "/src/apify/storage_clients/_file_system/_dataset_client.py" + }, + "1105": { + "qualifiedName": "push_data", + "sourceFileName": "/src/apify/storage_clients/_file_system/_dataset_client.py" + }, + "1108": { + "qualifiedName": "ApifyFileSystemStorageClient", + "sourceFileName": "/src/apify/storage_clients/_file_system/_storage_client.py" + }, + "1109": { + "qualifiedName": "get_storage_client_cache_key", + "sourceFileName": "/src/apify/storage_clients/_file_system/_storage_client.py" + }, + "1112": { + "qualifiedName": "create_kvs_client", + "sourceFileName": "/src/apify/storage_clients/_file_system/_storage_client.py" + }, + "1118": { + "qualifiedName": "create_dataset_client", + "sourceFileName": "/src/apify/storage_clients/_file_system/_storage_client.py" + }, + "1124": { + "qualifiedName": "SmartApifyStorageClient", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1125": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1129": { + "qualifiedName": "__str__", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1131": { + "qualifiedName": "get_storage_client_cache_key", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1134": { + "qualifiedName": "create_dataset_client", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1140": { + "qualifiedName": "create_kvs_client", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1146": { + "qualifiedName": "create_rq_client", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1152": { + "qualifiedName": "get_suitable_storage_client", + "sourceFileName": "/src/apify/storage_clients/_smart_apify/_storage_client.py" + }, + "1155": { + "qualifiedName": "DatasetClientPpeMixin", + "sourceFileName": "/src/apify/storage_clients/_ppe_dataset_mixin.py" + }, + "1156": { + "qualifiedName": "__init__", + "sourceFileName": "/src/apify/storage_clients/_ppe_dataset_mixin.py" + }, + "1158": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1159": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1160": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1161": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1162": { + "qualifiedName": "method", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1163": { + "qualifiedName": "payload", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1164": { + "qualifiedName": "headers", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1165": { + "qualifiedName": "user_data", + "sourceFileName": "/src/apify/request_loaders/_apify_request_list.py" + }, + "1166": { + "qualifiedName": "charge_lock", + "sourceFileName": "/src/apify/_charging.py" + }, + "1167": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "1168": { + "qualifiedName": "apify_margin_percentage", + "sourceFileName": "/src/apify/_models.py" + }, + "1169": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1170": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1171": { + "qualifiedName": "notified_about_future_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1172": { + "qualifiedName": "notified_about_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1173": { + "qualifiedName": "reason_for_change", + "sourceFileName": "/src/apify/_models.py" + }, + "1174": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "1175": { + "qualifiedName": "apify_margin_percentage", + "sourceFileName": "/src/apify/_models.py" + }, + "1176": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1177": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1178": { + "qualifiedName": "notified_about_future_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1179": { + "qualifiedName": "notified_about_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1180": { + "qualifiedName": "reason_for_change", + "sourceFileName": "/src/apify/_models.py" + }, + "1181": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "1182": { + "qualifiedName": "apify_margin_percentage", + "sourceFileName": "/src/apify/_models.py" + }, + "1183": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1184": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1185": { + "qualifiedName": "notified_about_future_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1186": { + "qualifiedName": "notified_about_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1187": { + "qualifiedName": "reason_for_change", + "sourceFileName": "/src/apify/_models.py" + }, + "1188": { + "qualifiedName": "model_config", + "sourceFileName": "/src/apify/_models.py" + }, + "1189": { + "qualifiedName": "apify_margin_percentage", + "sourceFileName": "/src/apify/_models.py" + }, + "1190": { + "qualifiedName": "created_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1191": { + "qualifiedName": "started_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1192": { + "qualifiedName": "notified_about_future_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1193": { + "qualifiedName": "notified_about_change_at", + "sourceFileName": "/src/apify/_models.py" + }, + "1194": { + "qualifiedName": "reason_for_change", + "sourceFileName": "/src/apify/_models.py" + } + }, + "overloads": [ + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 33 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 34 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['Dataset']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 35 + }, + "name": "collection_client", + "type": "KEYWORD_ONLY", + "datatype": "DatasetCollectionClientAsync" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 36 + }, + "name": "get_resource_client_by_id", + "type": "KEYWORD_ONLY", + "datatype": "Callable[[str], DatasetClientAsync]" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 37 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + } + ], + "return_type": "DatasetClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 30 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 31 + }, + "name": "open_by_alias", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 44 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 45 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['KeyValueStore']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 46 + }, + "name": "collection_client", + "type": "KEYWORD_ONLY", + "datatype": "KeyValueStoreCollectionClientAsync" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 47 + }, + "name": "get_resource_client_by_id", + "type": "KEYWORD_ONLY", + "datatype": "Callable[[str], KeyValueStoreClientAsync]" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 48 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + } + ], + "return_type": "KeyValueStoreClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 41 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 42 + }, + "name": "open_by_alias", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 55 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 56 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['RequestQueue']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 57 + }, + "name": "collection_client", + "type": "KEYWORD_ONLY", + "datatype": "RequestQueueCollectionClientAsync" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 58 + }, + "name": "get_resource_client_by_id", + "type": "KEYWORD_ONLY", + "datatype": "Callable[[str], RequestQueueClientAsync]" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 59 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + } + ], + "return_type": "RequestQueueClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 52 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_alias_resolving.py", + "lineno": 53 + }, + "name": "open_by_alias", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 19 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['Dataset']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 20 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 21 + }, + "name": "id", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 22 + }, + "name": "name", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 23 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + } + ], + "return_type": "DatasetClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 16 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 17 + }, + "name": "create_storage_api_client", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 30 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['KeyValueStore']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 31 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 32 + }, + "name": "id", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 33 + }, + "name": "name", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 34 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + } + ], + "return_type": "KeyValueStoreClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 27 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 28 + }, + "name": "create_storage_api_client", + "type": "function", + "parsedDocstring": { + "text": "" + } + }, + { + "modifiers": [ + "async" + ], + "args": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 41 + }, + "name": "storage_type", + "type": "KEYWORD_ONLY", + "datatype": "Literal['RequestQueue']" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 42 + }, + "name": "configuration", + "type": "KEYWORD_ONLY", + "datatype": "Configuration" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 43 + }, + "name": "id", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 44 + }, + "name": "name", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + }, + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 45 + }, + "name": "alias", + "type": "KEYWORD_ONLY", + "datatype": "str | None", + "default_value": "None" + } + ], + "return_type": "RequestQueueClientAsync", + "decorations": [ + { + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 38 + }, + "name": "overload" + } + ], + "location": { + "filename": "REPO_ROOT_PLACEHOLDER/src/apify/storage_clients/_apify/_api_client_creation.py", + "lineno": 39 + }, + "name": "create_storage_api_client", + "type": "function", + "parsedDocstring": { + "text": "" + } + } + ] +} \ No newline at end of file diff --git a/website/versioned_sidebars/version-1.7-sidebars.json b/website/versioned_sidebars/version-1.7-sidebars.json new file mode 100644 index 00000000..1385cb4b --- /dev/null +++ b/website/versioned_sidebars/version-1.7-sidebars.json @@ -0,0 +1,38 @@ +{ + "sidebar": [ + { + "type": "doc", + "id": "introduction/introduction" + }, + { + "type": "doc", + "id": "introduction/quick-start" + }, + { + "type": "category", + "label": "Concepts", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "03-concepts" + } + ] + }, + { + "type": "category", + "label": "Guides", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "02-guides" + } + ] + }, + { + "type": "doc", + "id": "changelog" + } + ] +} diff --git a/website/versioned_sidebars/version-2.7-sidebars.json b/website/versioned_sidebars/version-2.7-sidebars.json new file mode 100644 index 00000000..26e77864 --- /dev/null +++ b/website/versioned_sidebars/version-2.7-sidebars.json @@ -0,0 +1,49 @@ +{ + "sidebar": [ + { + "type": "doc", + "id": "introduction/introduction" + }, + { + "type": "doc", + "id": "introduction/quick-start" + }, + { + "type": "category", + "label": "Concepts", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "03_concepts" + } + ] + }, + { + "type": "category", + "label": "Guides", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "02_guides" + } + ] + }, + { + "type": "category", + "label": "Upgrading", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "04_upgrading" + } + ] + }, + { + "type": "doc", + "id": "changelog" + } + ] +} diff --git a/website/versioned_sidebars/version-3.3-sidebars.json b/website/versioned_sidebars/version-3.3-sidebars.json new file mode 100644 index 00000000..d784fb49 --- /dev/null +++ b/website/versioned_sidebars/version-3.3-sidebars.json @@ -0,0 +1,49 @@ +{ + "sidebar": [ + { + "type": "doc", + "id": "introduction/introduction" + }, + { + "type": "doc", + "id": "introduction/quick-start" + }, + { + "type": "category", + "label": "Concepts", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "02_concepts" + } + ] + }, + { + "type": "category", + "label": "Guides", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "03_guides" + } + ] + }, + { + "type": "category", + "label": "Upgrading", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "04_upgrading" + } + ] + }, + { + "type": "doc", + "id": "changelog" + } + ] +} diff --git a/website/versions.json b/website/versions.json new file mode 100644 index 00000000..8e097de6 --- /dev/null +++ b/website/versions.json @@ -0,0 +1,5 @@ +[ + "3.3", + "2.7", + "1.7" +]