diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 240ffca..820c29d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.13' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d0ce2d..b056a24 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -42,13 +42,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.11'] + python-version: ['3.12'] # Keep in sync with .readthedocs.yml tox-job: ["mypy", "docs"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5420cd..c2700c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.13.1 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.19.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.3.0 + - black==24.10.0 diff --git a/.readthedocs.yml b/.readthedocs.yml index de19d2d..f81f402 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,7 +7,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/test.yml + python: "3.12" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt diff --git a/CHANGES.rst b/CHANGES.rst index 55e4c49..fb5fff1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,7 @@ Changes ------------------ * Improved how the :ref:`default retry policy ` handles - :ref:`temporary download errors `. + :ref:`temporary download errors `. Before, 3 HTTP 429 responses followed by a single HTTP 520 response would have prevented a retry. Now, unrelated responses and errors do not count towards the HTTP 520 retry limit. diff --git a/README.rst b/README.rst index 4035e57..e52e951 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,7 @@ Installation pip install zyte-api -.. note:: Python 3.8+ is required. +.. note:: Python 3.9+ is required. .. install-end diff --git a/docs/conf.py b/docs/conf.py index 8ed5574..a652286 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,6 @@ import sys from pathlib import Path -import sphinx_rtd_theme - sys.path.insert(0, os.path.abspath("../")) @@ -58,7 +56,7 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = {".rst": "restructuredtext"} # The master toctree document. master_doc = "index" @@ -86,11 +84,6 @@ # html_theme = "sphinx_rtd_theme" -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/docs/use/api.rst b/docs/use/api.rst index 7d05a47..8e3d34e 100644 --- a/docs/use/api.rst +++ b/docs/use/api.rst @@ -134,15 +134,15 @@ The number of concurrent connections if enforced across all method calls, including different sessions of the same client. For guidelines on how to choose the optimal value for you, and other -optimization tips, see :ref:`zyte-api-optimize`. +optimization tips, see :ref:`zapi-optimize`. Errors and retries ================== Methods of :class:`ZyteAPI` and :class:`AsyncZyteAPI` automatically handle -retries for :ref:`rate-limiting ` and :ref:`unsuccessful -` responses, as well as network errors. +retries for :ref:`rate-limiting ` and :ref:`unsuccessful +` responses, as well as network errors. .. _retry-policy: .. _default-retry-policy: @@ -150,10 +150,10 @@ retries for :ref:`rate-limiting ` and :ref:`unsuccessful The default retry policy, :data:`~zyte_api.zyte_api_retrying`, does the following: -- Retries :ref:`rate-limiting responses ` forever. +- Retries :ref:`rate-limiting responses ` forever. - Retries :ref:`temporary download errors - ` up to 3 times. + ` up to 3 times. - Retries network errors until they have happened for 15 minutes straight. @@ -161,13 +161,13 @@ All retries are done with an exponential backoff algorithm. .. _aggressive-retry-policy: -If some :ref:`unsuccessful responses ` exceed +If some :ref:`unsuccessful responses ` exceed maximum retries with the default retry policy, try using :data:`~zyte_api.aggressive_retrying` instead, which modifies the default retry policy as follows: - Temporary download error are retried 7 times. :ref:`Permanent download - errors ` also count towards this retry + errors ` also count towards this retry limit. - Retries permanent download errors up to 3 times. diff --git a/docs/use/cli.rst b/docs/use/cli.rst index 49f4a9a..abf2479 100644 --- a/docs/use/cli.rst +++ b/docs/use/cli.rst @@ -33,7 +33,7 @@ The input file can be either of the following: :http:`request:browserHtml` set to ``True``. - A `JSON Lines `_ file with a object of :ref:`Zyte - API request parameters ` per line. For example: + API request parameters ` per line. For example: .. code-block:: json @@ -84,19 +84,19 @@ order and hence distribute the load somewhat evenly: zyte-api urls.txt --shuffle … For guidelines on how to choose the optimal ``--n-conn`` value for you, and -other optimization tips, see :ref:`zyte-api-optimize`. +other optimization tips, see :ref:`zapi-optimize`. Errors and retries ================== ``zyte-api`` automatically handles retries for :ref:`rate-limiting -` and :ref:`unsuccessful -` responses, as well as network errors, +` and :ref:`unsuccessful +` responses, as well as network errors, following the :ref:`default retry policy `. Use ``--dont-retry-errors`` to disable the retrying of error responses, and -retrying only :ref:`rate-limiting responses `: +retrying only :ref:`rate-limiting responses `: .. code-block:: shell diff --git a/pyproject.toml b/pyproject.toml index 830e253..c6b28cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,4 @@ profile = "black" multi_line_output = 3 [tool.black] -target-version = ["py38", "py39", "py310", "py311", "py312"] +target-version = ["py39", "py310", "py311", "py312", "py313"] diff --git a/setup.py b/setup.py index 310eb5e..612d6d4 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import os from setuptools import find_packages, setup @@ -41,9 +40,10 @@ def get_version(): "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], ) diff --git a/tests/test_main.py b/tests/test_main.py index 9fbb3bd..7e3c8d8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -109,9 +109,10 @@ async def test_run(queries, expected_response, store_errors, exception): async_client_mock.return_value.iter = request_parallel_mock # Patch the AsyncZyteAPI class in __main__ with the mock - with patch("zyte_api.__main__.AsyncZyteAPI", async_client_mock), patch( - "zyte_api.__main__.create_session" - ) as create_session_mock: + with ( + patch("zyte_api.__main__.AsyncZyteAPI", async_client_mock), + patch("zyte_api.__main__.create_session") as create_session_mock, + ): # Mock create_session to return an AsyncMock create_session_mock.return_value = AsyncMock() diff --git a/tox.ini b/tox.ini index 21023e8..33663b7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,mypy,docs,twine +envlist = py39,py310,py311,py312,py313,mypy,docs,twine [testenv] deps = @@ -19,9 +19,12 @@ commands = [testenv:mypy] deps = - mypy==0.982 + mypy==1.12.0 + pytest==8.3.3 + Twisted==24.7.0 + types-tqdm==4.66.0.20240417 -commands = mypy --ignore-missing-imports --no-warn-no-return \ +commands = mypy --ignore-missing-imports \ zyte_api \ tests @@ -39,8 +42,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twine] deps = - twine==4.0.2 - build==1.0.3 + twine==5.1.1 + build==1.2.2 commands = - python setup.py sdist + python -m build --sdist twine check dist/* diff --git a/zyte_api/_async.py b/zyte_api/_async.py index 261277f..afa20ff 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import time from asyncio import Future @@ -17,8 +19,6 @@ if TYPE_CHECKING: _ResponseFuture = Future[Dict[str, Any]] -else: - _ResponseFuture = Future # Python 3.8 support def _post_func(session): diff --git a/zyte_api/_errors.py b/zyte_api/_errors.py index ea589ae..6476c39 100644 --- a/zyte_api/_errors.py +++ b/zyte_api/_errors.py @@ -10,8 +10,8 @@ class RequestError(ClientResponseError): """Exception raised upon receiving a :ref:`rate-limiting - ` or :ref:`unsuccessful - ` response from Zyte API.""" + ` or :ref:`unsuccessful + ` response from Zyte API.""" def __init__(self, *args, **kwargs): #: Query sent to Zyte API. diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index 04b16b1..3b01c0e 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -137,7 +137,7 @@ class RetryFactory: the corresponding :class:`tenacity.AsyncRetrying` object. For example, to double the number of attempts for :ref:`temporary - download errors ` and the time network + download errors ` and the time network errors are retried: .. code-block:: python