diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..6c7e8ae6 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,111 @@ +version: 2.1 + +orbs: + codecov: codecov/codecov@3.3.0 + +workflows: + ci: + jobs: + - lint + - test: + name: Python (<< matrix.python_version >>) - ArangoDB (<< matrix.arangodb_license >>, << matrix.arangodb_version >> << matrix.arangodb_config >>) + matrix: + parameters: + python_version: ["3.9", "3.10", "3.11", "3.12"] + arangodb_config: ["single", "cluster"] + arangodb_license: ["community", "enterprise"] + arangodb_version: ["3.11", "latest"] + +jobs: + lint: + docker: + - image: python:latest + steps: + - checkout + - run: + name: Install Dependencies + command: pip install .[dev] + + - run: + name: Run black + command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/ + + - run: + name: Run flake8 + command: flake8 ./arango ./tests + + - run: + name: Run isort + command: isort --check ./arango ./tests + + - run: + name: Run mypy + command: mypy ./arango + + test: + parameters: + python_version: + type: string + arangodb_config: + type: string + arangodb_license: + type: string + arangodb_version: + type: string + # TODO: Reconsider using a docker image instead of a machine + # i.e cimg/python:<< parameters.python_version >> + machine: + image: ubuntu-2204:current + steps: + - checkout + + - run: + name: Set Up ArangoDB + command: | + chmod +x starter.sh + ./starter.sh << parameters.arangodb_config >> << parameters.arangodb_license >> << parameters.arangodb_version >> + + - restore_cache: + key: pip-and-local-cache + + # TODO: Revisit this bottleneck + - run: + name: Setup Python + command: | + pyenv --version + pyenv install -f << parameters.python_version >> + pyenv global << parameters.python_version >> + + - run: + name: "Install Dependencies" + command: pip install -e .[dev] + + - run: docker ps -a + + - run: docker logs arango + + - run: + name: "Run pytest" + command: | + mkdir test-results + + args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost" "--port=8529") + if [ << parameters.arangodb_config >> = "cluster" ]; then + args+=("--cluster" "--port=8539" "--port=8549") + fi + + if [ << parameters.arangodb_license >> = "enterprise" ]; then + args+=("--enterprise") + fi + + echo "Running pytest with args: ${args[@]}" + pytest --cov=arango --cov-report=xml --cov-report term-missing --color=yes --code-highlight=yes "${args[@]}" + + - store_artifacts: + path: test-results + + - store_test_results: + path: test-results + + - codecov/upload: + file: coverage.xml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 88a2b318..00000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build -on: - workflow_dispatch: - push: - branches: [ main, dev ] - pull_request: - branches: [ main, dev ] -jobs: - build: - name: Test - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [ 3.6, 3.7, 3.8, 3.9 ] - steps: - - uses: actions/checkout@v2 - - name: Create ArangoDB Docker container - run: > - docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd - arangodb/arangodb:3.7.7 --server.jwt-secret-keyfile=/tmp/keyfile - - name: Copy Foxx service zip into ArangoDB Docker container - run: docker cp tests/static/service.zip arango:/tmp/service.zip - - name: Copy keyfile into ArangoDB Docker container - run: docker cp tests/static/keyfile arango:/tmp/keyfile - - name: Start ArangoDB Docker container - run: docker start arango - - name: Fetch complete history for all tags and branches - run: git fetch --prune --unshallow - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Setup pip - run: python -m pip install --upgrade pip setuptools wheel - - name: Install packages - run: pip install .[dev] - - name: Run black - run: black --check . - - name: Run flake8 - run: flake8 . - - name: Run isort - run: isort --check --profile=black . - - name: Run mypy - run: mypy arango - - name: Run pytest - run: py.test --complete --cov=arango --cov-report=xml - - name: Run Sphinx doctest - run: python -m sphinx -b doctest docs docs/_build - - name: Run Sphinx HTML - run: python -m sphinx -b html -W docs docs/_build - - name: Upload coverge to Codecov - uses: codecov/codecov-action@v1 - if: matrix.python-version == '3.8' - with: - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index bafb67f6..b182474f 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -1,9 +1,8 @@ name: CodeQL + on: - push: - branches: [ main, dev ] pull_request: - branches: [ main, dev ] + branches: [main] schedule: - cron: '21 2 * * 3' @@ -11,12 +10,16 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest + + permissions: + security-events: write + steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: 'python' - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..1c434f70 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,103 @@ +name: Docs + +on: + pull_request: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: Debug with tmate + required: false + default: false + +jobs: + # This has been migrated to CircleCI + # test: + # runs-on: ubuntu-latest + + # strategy: + # fail-fast: false + # matrix: + # python_version: ["3.10"] #["3.8", "3.9", "3.10", "3.11", "3.12"] + # arangodb_config: ["single", "cluster"] + # arangodb_license: ["community", "enterprise"] + # arangodb_version: ["3.10.10", "3.11.4", "latest"] + + # name: Test (${{ matrix.python_version }}:${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }}) + + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python_version }} + + # - name: Setup ArangoDB + # run: | + # chmod +x starter.sh + # ./starter.sh ${{ matrix.arangodb_config }} ${{ matrix.arangodb_license }} ${{ matrix.arangodb_version }} + + # - name: Install Dependencies + # run: pip install -e .[dev] + + # - name: List Docker Containers + # run: docker ps -a + + # - name: Pytest + # run: | + # args=("--host" "localhost" "--port=8529") + + # if [ ${{ matrix.arangodb_config }} = "cluster" ]; then + # args+=("--cluster" "--port=8539" "--port=8549") + # fi + + # if [ ${{ matrix.arangodb_license }} = "enterprise" ]; then + # args+=("--enterprise") + # fi + + # echo "Running pytest with args: ${args[@]}" + # pytest --cov=arango --cov-report=xml "${args[@]}" + + docs: + runs-on: ubuntu-latest + + name: Docs + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch all tags and branches + run: git fetch --prune --unshallow + + - name: Create ArangoDB Docker container + run: > + docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static + arangodb/arangodb:latest --server.jwt-secret-keyfile=/tests/static/keyfile + + - name: Start ArangoDB Docker container + run: docker start arango + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Debug with tmate + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + + - name: Run pre-commit checks + uses: pre-commit/action@v3.0.0 + + - name: Install dependencies + run: pip install .[dev] + + - name: Run Sphinx doctest + run: python -m sphinx -b doctest docs docs/_build + + # No longer needed as this is handled by Read the Docs + #- name: Generate Sphinx HTML + # run: python -m sphinx -b html -W docs docs/_build diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 8c50645d..a24652fc 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -1,31 +1,52 @@ name: Upload to PyPI + on: release: - types: [created] + types: [published] + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: Debug with tmate + required: false + default: false + jobs: upload: runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v2 - - name: Fetch complete history for all tags and branches - run: git fetch --prune --unshallow - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine setuptools-scm[toml] - - name: Build distribution - run: python setup.py sdist bdist_wheel - - name: Publish to PyPI Test - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TEST_TOKEN }} - run: twine upload --repository testpypi dist/* - - name: Publish to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: twine upload --repository pypi dist/* + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Fetch all tags and branches + run: git fetch --prune --unshallow + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Debug with tmate + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build distribution + run: python -m build + + - name: Publish to PyPI Test + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TEST_TOKEN }} + run: twine upload --repository testpypi dist/* + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload --repository pypi dist/* diff --git a/.gitignore b/.gitignore index 2a2bde61..4fa6f46d 100644 --- a/.gitignore +++ b/.gitignore @@ -115,5 +115,15 @@ localdata/ # Node Modules node_modules/ +# direnv +.envrc +.direnv/ + # setuptools_scm arango/version.py + +# test results +*_results.txt + +# devcontainers +.devcontainer diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e78053c0..8098f23e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,31 +1,41 @@ repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + # See https://pre-commit.com/hooks.html hooks: - - id: check-case-conflict - - id: check-executables-have-shebangs - - id: check-json - - id: check-merge-conflict - - id: check-symlinks - - id: check-toml - - id: check-yaml - - id: end-of-file-fixer - - id: mixed-line-ending - - repo: https://github.com/psf/black - rev: 20.8b1 + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: pretty-format-json + - id: trailing-whitespace + +- repo: https://github.com/psf/black + rev: 23.1.0 hooks: - - id: black - - repo: https://github.com/timothycrosley/isort - rev: 5.7.0 + - id: black + +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 hooks: - - id: isort + - id: isort args: [ --profile, black ] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + +- repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 hooks: - - id: mypy - files: ^arango/ - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - id: flake8 + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.991 hooks: - - id: flake8 + - id: mypy + files: ^arango/ + additional_dependencies: ['types-requests', "types-setuptools"] diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..4fda4951 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + builder: html + # Fail on all warnings to avoid broken references + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3913ac85..375d8b0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,13 @@ pre-commit install # Install git pre-commit hooks Run unit tests with coverage: ```shell -py.test --cov=arango --cov-report=html # Open htmlcov/index.html in your browser +pytest --cov=arango --cov-report=html # Open htmlcov/index.html in your browser +``` + +To start and ArangoDB instance locally, run: + +```shell +./starter.sh # Requires docker ``` Build and test documentation: diff --git a/MANIFEST.in b/MANIFEST.in index 8815dcca..95649344 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include README.md LICENSE prune tests +include arango/py.typed diff --git a/README.md b/README.md index daf44f54..f76c608e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,18 @@ ![Logo](https://user-images.githubusercontent.com/2701938/108583516-c3576680-72ee-11eb-883f-2d9b52e74e45.png) -[![Build](https://github.com/joowani/python-arango/actions/workflows/build.yaml/badge.svg)](https://github.com/joowani/python-arango/actions/workflows/build.yaml) -[![CodeQL](https://github.com/joowani/python-arango/actions/workflows/codeql.yaml/badge.svg)](https://github.com/joowani/python-arango/actions/workflows/codeql.yaml) -[![CodeCov](https://codecov.io/gh/joowani/python-arango/branch/main/graph/badge.svg?token=DXg0O4hxnx)](https://codecov.io/gh/joowani/python-arango) -[![PyPI version](https://badge.fury.io/py/python-arango.svg)](https://badge.fury.io/py/python-arango) -[![GitHub license](https://img.shields.io/badge/license-MIT-brightgreen)](https://github.com/joowani/python-arango/blob/main/LICENSE) -![Python version](https://img.shields.io/badge/python-3.6%2B-blue) +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/arangodb/python-arango/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/arangodb/python-arango/tree/main) +[![CodeQL](https://github.com/arangodb/python-arango/actions/workflows/codeql.yaml/badge.svg)](https://github.com/arangodb/python-arango/actions/workflows/codeql.yaml) +[![Docs](https://github.com/arangodb/python-arango/actions/workflows/docs.yaml/badge.svg)](https://github.com/arangodb/python-arango/actions/workflows/docs.yaml) +[![Coverage Status](https://codecov.io/gh/arangodb/python-arango/branch/main/graph/badge.svg?token=M8zrjrzsUY)](https://codecov.io/gh/arangodb/python-arango) +[![Last commit](https://img.shields.io/github/last-commit/arangodb/python-arango)](https://github.com/arangodb/python-arango/commits/main) + +[![PyPI version badge](https://img.shields.io/pypi/v/python-arango?color=3775A9&style=for-the-badge&logo=pypi&logoColor=FFD43B)](https://pypi.org/project/python-arango/) +[![Python versions badge](https://img.shields.io/badge/3.9%2B-3776AB?style=for-the-badge&logo=python&logoColor=FFD43B&label=Python)](https://pypi.org/project/python-arango/) + +[![License](https://img.shields.io/github/license/arangodb/python-arango?color=9E2165&style=for-the-badge)](https://github.com/arangodb/python-arango/blob/main/LICENSE) +[![Code style: black](https://img.shields.io/static/v1?style=for-the-badge&label=code%20style&message=black&color=black)](https://github.com/psf/black) +[![Downloads](https://img.shields.io/pepy/dt/python-arango?style=for-the-badge&color=282661 +)](https://pepy.tech/project/python-arango) # Python-Arango @@ -14,13 +21,13 @@ database natively supporting documents, graphs and search. ## Requirements -- ArangoDB version 3.7+ -- Python version 3.6+ +- ArangoDB version 3.11+ +- Python version 3.9+ ## Installation ```shell -pip install python-arango +pip install python-arango --upgrade ``` ## Getting Started @@ -45,8 +52,8 @@ db = client.db("test", username="root", password="passwd") # Create a new collection named "students". students = db.create_collection("students") -# Add a hash index to the collection. -students.add_hash_index(fields=["name"], unique=True) +# Add a persistent index to the collection. +students.add_index({'type': 'persistent', 'fields': ['name'], 'unique': True}) # Insert new documents into the collection. students.insert({"name": "jane", "age": 39}) @@ -72,6 +79,11 @@ db = client.db("test", username="root", password="passwd") # Create a new graph named "school". graph = db.create_graph("school") +# Create a new EnterpriseGraph [Enterprise Edition] +eegraph = db.create_graph( + name="school", + smart=True) + # Create vertex collections for the graph. students = graph.create_vertex_collection("students") lectures = graph.create_vertex_collection("lectures") @@ -101,12 +113,13 @@ edges.insert({"_from": "students/02", "_to": "lectures/MAT101"}) edges.insert({"_from": "students/02", "_to": "lectures/STA101"}) edges.insert({"_from": "students/03", "_to": "lectures/CSC101"}) -# Traverse the graph in outbound direction, breadth-first. -result = graph.traverse( - start_vertex="students/01", - direction="outbound", - strategy="breadthfirst" -) +# Traverse the graph in outbound direction, breath-first. +query = """ + FOR v, e, p IN 1..3 OUTBOUND 'students/01' GRAPH 'school' + OPTIONS { bfs: true, uniqueVertices: 'global' } + RETURN {vertex: v, edge: e, path: p} + """ +cursor = db.aql.execute(query) ``` Please see the [documentation](https://docs.python-arango.com) for more details. diff --git a/arango/aql.py b/arango/aql.py index dcbff732..25786302 100644 --- a/arango/aql.py +++ b/arango/aql.py @@ -19,6 +19,7 @@ AQLQueryExplainError, AQLQueryKillError, AQLQueryListError, + AQLQueryRulesGetError, AQLQueryTrackingGetError, AQLQueryTrackingSetError, AQLQueryValidateError, @@ -30,11 +31,12 @@ format_aql_tracking, format_body, format_query_cache_entry, + format_query_rule_item, ) from arango.request import Request from arango.response import Response from arango.result import Result -from arango.typings import Json, Jsons +from arango.typings import DataTypes, Json, Jsons class AQLQueryCache(ApiGroup): @@ -142,6 +144,36 @@ def response_handler(resp: Response) -> bool: return self._execute(request, response_handler) + def plan_entries(self) -> Result[Jsons]: + """Return a list of all AQL query plan cache entries. + + :return: List of AQL query plan cache entries. + :rtype: list + :raise arango.exceptions.AQLCacheEntriesError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_api/query-plan-cache") + + def response_handler(resp: Response) -> Jsons: + if not resp.is_success: + raise AQLCacheEntriesError(resp, request) + result: Jsons = resp.body + return result + + return self._execute(request, response_handler) + + def clear_plan(self) -> Result[None]: + """Clear the AQL query plan cache. + + :raises arango.exceptions.AQLCacheClearError: If clearing the cache fails. + """ + request = Request(method="delete", endpoint="/_api/query-plan-cache") + + def response_handler(resp: Response) -> None: + if not resp.is_success: + raise AQLCacheClearError(resp, request) + + return self._execute(request, response_handler) + class AQL(ApiGroup): """AQL (ArangoDB Query Language) API wrapper. @@ -171,7 +203,7 @@ def explain( all_plans: bool = False, max_plans: Optional[int] = None, opt_rules: Optional[Sequence[str]] = None, - bind_vars: Optional[MutableMapping[str, str]] = None, + bind_vars: Optional[MutableMapping[str, DataTypes]] = None, ) -> Result[Union[Json, Jsons]]: """Inspect the query and return its metadata without executing it. @@ -211,11 +243,19 @@ def response_handler(resp: Response) -> Union[Json, Jsons]: if not resp.is_success: raise AQLQueryExplainError(resp, request) if "plan" in resp.body: - plan: Json = resp.body["plan"] - return plan + result: Json = resp.body["plan"] + if "stats" in resp.body: + result["stats"] = resp.body["stats"] + return result else: - plans: Jsons = resp.body["plans"] - return plans + results: Jsons = resp.body["plans"] + if "stats" in resp.body: + # Although "plans" contains an array, "stats" is a single object. + # We need to duplicate "stats" for each plan in order to preserve + # the original structure. + for plan in results: + plan["stats"] = resp.body["stats"] + return results return self._execute(request, response_handler) @@ -247,14 +287,14 @@ def execute( count: bool = False, batch_size: Optional[int] = None, ttl: Optional[Number] = None, - bind_vars: Optional[MutableMapping[str, str]] = None, + bind_vars: Optional[MutableMapping[str, DataTypes]] = None, full_count: Optional[bool] = None, max_plans: Optional[int] = None, optimizer_rules: Optional[Sequence[str]] = None, cache: Optional[bool] = None, memory_limit: int = 0, fail_on_warning: Optional[bool] = None, - profile: Optional[bool] = None, + profile: Optional[Union[bool, int]] = None, max_transaction_size: Optional[int] = None, max_warning_count: Optional[int] = None, intermediate_commit_count: Optional[int] = None, @@ -263,6 +303,11 @@ def execute( stream: Optional[bool] = None, skip_inaccessible_cols: Optional[bool] = None, max_runtime: Optional[Number] = None, + fill_block_cache: Optional[bool] = None, + allow_dirty_read: bool = False, + allow_retry: bool = False, + force_one_shard_attribute_value: Optional[str] = None, + use_plan_cache: Optional[bool] = None, ) -> Result[Cursor]: """Execute the query and return the result cursor. @@ -300,11 +345,15 @@ def execute( development to catch issues early. If set to False, warnings are returned with the query result. There is a server configuration option "--query.fail-on-warning" for setting the default value for - this behaviour so it does not need to be set per-query. + this behaviour, so it does not need to be set per-query. :type fail_on_warning: bool :param profile: Return additional profiling details in the cursor, - unless the query cache is used. - :type profile: bool + unless the query cache is used. If set to True or 1, then query profiling + information can be fetched with `cursor.profile()`. If set to 2, additional + execution stats per query plan node are included via "nodes" in + `cursor.statistics()`, as well as a the query plan which can be fetched + with `cursor.plan()`. + :type profile: bool | int :param max_transaction_size: Transaction size limit in bytes. :type max_transaction_size: int :param max_warning_count: Max number of warnings returned. @@ -345,6 +394,33 @@ def execute( it is killed. The value is specified in seconds. Default value is 0.0 (no timeout). :type max_runtime: int | float + :param fill_block_cache: If set to true or not specified, this will + make the query store the data it reads via the RocksDB storage + engine in the RocksDB block cache. This is usually the desired + behavior. The option can be set to false for queries that are + known to either read a lot of data which would thrash the block + cache, or for queries that read data which are known to be outside + of the hot set. By setting the option to false, data read by the + query will not make it into the RocksDB block cache if not already + in there, thus leaving more room for the actual hot set. + :type fill_block_cache: bool + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool | None + :param allow_retry: Make it possible to retry fetching the latest batch + from a cursor. + :type allow_retry: bool + :param force_one_shard_attribute_value: (Enterprise Only) Explicitly set + a shard key value that will be used during query snippet distribution + to limit the query to a specific server in the cluster. This query option + can be used in complex queries in case the query optimizer cannot + automatically detect that the query can be limited to only a single + server (e.g. in a disjoint smart graph case). If the option is set + incorrectly, i.e. to a wrong shard key value, then the query may be + shipped to a wrong DB server and may not return results + (i.e. empty result set). Use at your own risk. + :param force_one_shard_attribute_value: str | None + :param use_plan_cache: If set to True, the query plan cache is used. + :param use_plan_cache: bool | None :return: Result cursor. :rtype: arango.cursor.Cursor :raise arango.exceptions.AQLQueryExecuteError: If execute fails. @@ -356,14 +432,14 @@ def execute( data["ttl"] = ttl if bind_vars is not None: data["bindVars"] = bind_vars - if cache is not None: - data["cache"] = cache if memory_limit is not None: data["memoryLimit"] = memory_limit options: Json = {} if full_count is not None: options["fullCount"] = full_count + if fill_block_cache is not None: + options["fillBlockCache"] = fill_block_cache if max_plans is not None: options["maxNumberOfPlans"] = max_plans if optimizer_rules is not None: @@ -388,17 +464,30 @@ def execute( options["skipInaccessibleCollections"] = skip_inaccessible_cols if max_runtime is not None: options["maxRuntime"] = max_runtime + if allow_retry is not None: + options["allowRetry"] = allow_retry + if force_one_shard_attribute_value is not None: + options["forceOneShardAttributeValue"] = force_one_shard_attribute_value + if cache is not None: + options["cache"] = cache + if use_plan_cache is not None: + options["usePlanCache"] = use_plan_cache if options: data["options"] = options data.update(options) - request = Request(method="post", endpoint="/_api/cursor", data=data) + request = Request( + method="post", + endpoint="/_api/cursor", + data=data, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, + ) def response_handler(resp: Response) -> Cursor: if not resp.is_success: raise AQLQueryExecuteError(resp, request) - return Cursor(self._conn, resp.body) + return Cursor(self._conn, resp.body, allow_retry=allow_retry) return self._execute(request, response_handler) @@ -619,3 +708,24 @@ def response_handler(resp: Response) -> Union[bool, Json]: return {"deleted": resp.body["deletedCount"]} return self._execute(request, response_handler) + + def query_rules(self) -> Result[Jsons]: + """Return the available optimizer rules for AQL queries + + :return: The available optimizer rules for AQL queries + :rtype: dict + :raise arango.exceptions.AQLQueryRulesGetError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_api/query/rules") + + def response_handler(resp: Response) -> Jsons: + if not resp.is_success: + raise AQLQueryRulesGetError(resp, request) + + rules: Jsons = resp.body + items: Jsons = [] + for rule in rules: + items.append(format_query_rule_item(rule)) + return items + + return self._execute(request, response_handler) diff --git a/arango/backup.py b/arango/backup.py index 270b52f1..c06e4e15 100644 --- a/arango/backup.py +++ b/arango/backup.py @@ -29,7 +29,7 @@ def get(self, backup_id: Optional[str] = None) -> Result[Json]: """Return backup details. :param backup_id: If set, details on only the specified backup is - returned. Otherwise details on all backups are returned. + returned. Otherwise, details on all backups are returned. :type backup_id: str :return: Backup details. :rtype: dict @@ -38,7 +38,7 @@ def get(self, backup_id: Optional[str] = None) -> Result[Json]: request = Request( method="post", endpoint="/_admin/backup/list", - data={} if backup_id is None else {"id": backup_id}, + data=None if backup_id is None else {"id": backup_id}, ) def response_handler(resp: Response) -> Json: diff --git a/arango/client.py b/arango/client.py index eec64a7b..b56755b0 100644 --- a/arango/client.py +++ b/arango/client.py @@ -3,7 +3,7 @@ from json import dumps, loads from typing import Any, Callable, Optional, Sequence, Union -from pkg_resources import get_distribution +import importlib_metadata from arango.connection import ( BasicConnection, @@ -12,25 +12,60 @@ JwtSuperuserConnection, ) from arango.database import StandardDatabase -from arango.exceptions import ServerConnectionError -from arango.http import DefaultHTTPClient, HTTPClient +from arango.exceptions import ArangoClientError, ServerConnectionError +from arango.http import ( + DEFAULT_REQUEST_TIMEOUT, + DefaultHTTPClient, + HTTPClient, + RequestCompression, +) from arango.resolver import ( + FallbackHostResolver, HostResolver, + PeriodicHostResolver, RandomHostResolver, RoundRobinHostResolver, SingleHostResolver, ) +def default_serializer(x: Any) -> str: + """ + Default JSON serializer + + :param x: A JSON data type object to serialize + :type x: Any + :return: The object serialized as a JSON string + :rtype: str + """ + return dumps(x, separators=(",", ":")) + + +def default_deserializer(x: str) -> Any: + """ + Default JSON de-serializer + + :param x: A JSON string to deserialize + :type x: str + :return: The de-serialized JSON object + :rtype: Any + """ + return loads(x) + + class ArangoClient: """ArangoDB client. :param hosts: Host URL or list of URLs (coordinators in a cluster). :type hosts: str | [str] :param host_resolver: Host resolver. This parameter used for clusters (when - multiple host URLs are provided). Accepted values are "roundrobin" and - "random". Any other value defaults to round robin. - :type host_resolver: str + multiple host URLs are provided). Accepted values are "fallback", + "roundrobin", "random" and "periodic". The default value is "fallback". + :type host_resolver: str | arango.resolver.HostResolver + :param resolver_max_tries: Number of attempts to process an HTTP request + before throwing a ConnectionAbortedError. Must not be lower than the + number of hosts. + :type resolver_max_tries: int :param http_client: User-defined HTTP client. :type http_client: arango.http.HTTPClient :param serializer: User-defined JSON serializer. Must be a callable @@ -42,15 +77,41 @@ class ArangoClient: the de-serialized object. If not given, ``json.loads`` is used by default. :type deserializer: callable + :param verify_override: Override TLS certificate verification. This will + override the verify method of the underlying HTTP client. + + - `None`: Do not change the verification behavior of the + underlying HTTP client. + - `True`: Verify TLS certificate using the system CA certificates. + - `False`: Do not verify TLS certificate. + - `str`: Path to a custom CA bundle file or directory. + :type verify_override: Union[bool, str, None] + :param request_timeout: This is the default request timeout (in seconds) + for http requests issued by the client if the parameter http_client is + not specified. The default value is 60. + None: No timeout. + int: Timeout value in seconds. + :type request_timeout: int | float + :param request_compression: Will compress requests to the server according to + the given algorithm. No compression happens by default. + :type request_compression: arango.http.RequestCompression | None + :param response_compression: Tells the server what compression algorithm is + acceptable for the response. No compression happens by default. + :type response_compression: str | None """ def __init__( self, hosts: Union[str, Sequence[str]] = "http://127.0.0.1:8529", - host_resolver: str = "roundrobin", + host_resolver: Union[str, HostResolver] = "fallback", + resolver_max_tries: Optional[int] = None, http_client: Optional[HTTPClient] = None, - serializer: Callable[..., str] = lambda x: dumps(x), - deserializer: Callable[[str], Any] = lambda x: loads(x), + serializer: Callable[..., str] = default_serializer, + deserializer: Callable[[str], Any] = default_deserializer, + verify_override: Union[bool, str, None] = None, + request_timeout: Union[int, float, None] = DEFAULT_REQUEST_TIMEOUT, + request_compression: Optional[RequestCompression] = None, + response_compression: Optional[str] = None, ) -> None: if isinstance(hosts, str): self._hosts = [host.strip("/") for host in hosts.split(",")] @@ -61,17 +122,35 @@ def __init__( self._host_resolver: HostResolver if host_count == 1: - self._host_resolver = SingleHostResolver() + self._host_resolver = SingleHostResolver(1, resolver_max_tries) + elif host_resolver == "fallback": + self._host_resolver = FallbackHostResolver(host_count, resolver_max_tries) elif host_resolver == "random": - self._host_resolver = RandomHostResolver(host_count) + self._host_resolver = RandomHostResolver(host_count, resolver_max_tries) + elif host_resolver == "roundrobin": + self._host_resolver = RoundRobinHostResolver(host_count, resolver_max_tries) + elif host_resolver == "periodic": + self._host_resolver = PeriodicHostResolver(host_count, resolver_max_tries) else: - self._host_resolver = RoundRobinHostResolver(host_count) + if not isinstance(host_resolver, HostResolver): + raise ValueError("Invalid host resolver") + self._host_resolver = host_resolver + + # Initializes the http client + self._http = http_client or DefaultHTTPClient(request_timeout=request_timeout) - self._http = http_client or DefaultHTTPClient() self._serializer = serializer self._deserializer = deserializer self._sessions = [self._http.create_session(h) for h in self._hosts] + # override SSL/TLS certificate verification if provided + if verify_override is not None: + for session in self._sessions: + session.verify = verify_override + + self._request_compression = request_compression + self._response_compression = response_compression + def __repr__(self) -> str: return f"" @@ -96,7 +175,22 @@ def version(self) -> str: :return: Client version. :rtype: str """ - return get_distribution("python-arango").version + version: str = importlib_metadata.version("python-arango") + return version + + @property + def request_timeout(self) -> Any: + """Return the request timeout of the http client. + + :return: Request timeout. + :rtype: Any + """ + return self._http.request_timeout # type: ignore + + # Setter for request_timeout + @request_timeout.setter + def request_timeout(self, value: Any) -> None: + self._http.request_timeout = value # type: ignore def db( self, @@ -105,6 +199,7 @@ def db( password: str = "", verify: bool = False, auth_method: str = "basic", + user_token: Optional[str] = None, superuser_token: Optional[str] = None, ) -> StandardDatabase: """Connect to an ArangoDB database and return the database API wrapper. @@ -122,9 +217,17 @@ def db( refreshed automatically using ArangoDB username and password. This assumes that the clocks of the server and client are synchronized. :type auth_method: str + :param user_token: User generated token for user access. + If set, parameters **username**, **password** and **auth_method** + are ignored. This token is not refreshed automatically. If automatic + token refresh is required, consider setting **auth_method** to "jwt" + and using the **username** and **password** parameters instead. Token + expiry will be checked. + :type user_token: str :param superuser_token: User generated token for superuser access. If set, parameters **username**, **password** and **auth_method** - are ignored. This token is not refreshed automatically. + are ignored. This token is not refreshed automatically. Token + expiry will not be checked. :type superuser_token: str :return: Standard database API wrapper. :rtype: arango.database.StandardDatabase @@ -143,6 +246,21 @@ def db( serializer=self._serializer, deserializer=self._deserializer, superuser_token=superuser_token, + request_compression=self._request_compression, + response_compression=self._response_compression, + ) + elif user_token is not None: + connection = JwtConnection( + hosts=self._hosts, + host_resolver=self._host_resolver, + sessions=self._sessions, + db_name=name, + http_client=self._http, + serializer=self._serializer, + deserializer=self._deserializer, + user_token=user_token, + request_compression=self._request_compression, + response_compression=self._response_compression, ) elif auth_method.lower() == "basic": connection = BasicConnection( @@ -155,6 +273,8 @@ def db( http_client=self._http, serializer=self._serializer, deserializer=self._deserializer, + request_compression=self._request_compression, + response_compression=self._response_compression, ) elif auth_method.lower() == "jwt": connection = JwtConnection( @@ -167,6 +287,8 @@ def db( http_client=self._http, serializer=self._serializer, deserializer=self._deserializer, + request_compression=self._request_compression, + response_compression=self._response_compression, ) else: raise ValueError(f"invalid auth_method: {auth_method}") @@ -177,6 +299,6 @@ def db( except ServerConnectionError as err: raise err except Exception as err: - raise ServerConnectionError(f"bad connection: {err}") + raise ArangoClientError(f"bad connection: {err}") return StandardDatabase(connection) diff --git a/arango/cluster.py b/arango/cluster.py index 04affe3c..ea13279d 100644 --- a/arango/cluster.py +++ b/arango/cluster.py @@ -1,18 +1,21 @@ __all__ = ["Cluster"] -from typing import List +from typing import List, Optional from arango.api import ApiGroup from arango.exceptions import ( ClusterEndpointsError, ClusterHealthError, ClusterMaintenanceModeError, + ClusterRebalanceError, ClusterServerCountError, ClusterServerEngineError, ClusterServerIDError, + ClusterServerModeError, ClusterServerRoleError, ClusterServerStatisticsError, ClusterServerVersionError, + ClusterVpackSortMigrationError, ) from arango.formatter import format_body from arango.request import Request @@ -56,6 +59,27 @@ def response_handler(resp: Response) -> str: return self._execute(request, response_handler) + def server_mode(self) -> Result[str]: + """Return the server mode. + + In a read-only server, all write operations will fail + with an error code of 1004 (ERROR_READ_ONLY). Creating or dropping + databases and collections will also fail with error code 11 (ERROR_FORBIDDEN). + + :return: Server mode. Possible values are "default" or "readonly". + :rtype: str + :raise arango.exceptions.ClusterServerModeError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_admin/server/mode") + + def response_handler(resp: Response) -> str: + if resp.is_success: + return str(resp.body["mode"]) + + raise ClusterServerModeError(resp, request) + + return self._execute(request, response_handler) + def server_version(self, server_id: str) -> Result[Json]: """Return the version of the given server. @@ -139,6 +163,58 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def server_maintenance_mode(self, server_id: str) -> Result[Json]: + """Return the maintenance status for the given server. + + :param server_id: Server ID. + :type server_id: str + :return: Maintenance status for the given server. + :rtype: dict + :raise arango.exceptions.ClusterMaintenanceModeError: If retrieval fails. + """ + request = Request( + method="get", + endpoint=f"/_admin/cluster/maintenance/{server_id}", + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body.get("result", {}) + return result + + raise ClusterMaintenanceModeError(resp, request) + + return self._execute(request, response_handler) + + def toggle_server_maintenance_mode( + self, server_id: str, mode: str, timeout: Optional[int] = None + ) -> Result[Json]: + """Enable or disable the maintenance mode for the given server. + + :param server_id: Server ID. + :type server_id: str + :param mode: Maintenance mode. Allowed values are "normal" and "maintenance". + :type mode: str + :param timeout: Timeout in seconds. + :type timeout: Optional[int] + :return: Result of the operation. + :rtype: dict + :raise arango.exceptions.ClusterMaintenanceModeError: If toggle fails. + """ + request = Request( + method="put", + endpoint=f"/_admin/cluster/maintenance/{server_id}", + data={"mode": mode, "timeout": timeout}, + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_body(resp.body) + + raise ClusterMaintenanceModeError(resp, request) + + return self._execute(request, response_handler) + def health(self) -> Result[Json]: """Return the cluster health. @@ -195,3 +271,231 @@ def response_handler(resp: Response) -> List[str]: return [item["endpoint"] for item in resp.body["endpoints"]] return self._execute(request, response_handler) + + def calculate_imbalance(self) -> Result[Json]: + """Compute the current cluster imbalance, including + the amount of ongoing and pending move shard operations. + + :return: Cluster imbalance information. + :rtype: dict + :raise: arango.exceptions.ClusterRebalanceError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_admin/cluster/rebalance") + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterRebalanceError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) + + def rebalance( + self, + version: int = 1, + max_moves: Optional[int] = None, + leader_changes: Optional[bool] = None, + move_leaders: Optional[bool] = None, + move_followers: Optional[bool] = None, + pi_factor: Optional[float] = None, + exclude_system_collections: Optional[bool] = None, + databases_excluded: Optional[List[str]] = None, + ) -> Result[Json]: + """Compute and execute a cluster rebalance plan. + + :param version: Must be set to 1. + :type version: int + :param max_moves: Maximum number of moves to be computed. + :type max_moves: int | None + :param leader_changes: Allow leader changes without moving data. + :type leader_changes: bool | None + :param move_leaders: Allow moving shard leaders. + :type move_leaders: bool | None + :param move_followers: Allow moving shard followers. + :type move_followers: bool | None + :param pi_factor: A weighting factor that should remain untouched. + :type pi_factor: float | None + :param exclude_system_collections: Ignore system collections in the + rebalance plan. + :type exclude_system_collections: bool | None + :param databases_excluded: List of database names to be excluded + from the analysis. + :type databases_excluded: [str] | None + :return: Cluster rebalance plan that has been executed. + :rtype: dict + :raise: arango.exceptions.ClusterRebalanceError: If retrieval fails. + """ + data: Json = dict(version=version) + if max_moves is not None: + data["maximumNumberOfMoves"] = max_moves + if leader_changes is not None: + data["leaderChanges"] = leader_changes + if move_leaders is not None: + data["moveLeaders"] = move_leaders + if move_followers is not None: + data["moveFollowers"] = move_followers + if pi_factor is not None: + data["piFactor"] = pi_factor + if exclude_system_collections is not None: + data["excludeSystemCollections"] = exclude_system_collections + if databases_excluded is not None: + data["databasesExcluded"] = databases_excluded + + request = Request(method="put", endpoint="/_admin/cluster/rebalance", data=data) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterRebalanceError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) + + def calculate_rebalance_plan( + self, + version: int = 1, + max_moves: Optional[int] = None, + leader_changes: Optional[bool] = None, + move_leaders: Optional[bool] = None, + move_followers: Optional[bool] = None, + pi_factor: Optional[float] = None, + exclude_system_collections: Optional[bool] = None, + databases_excluded: Optional[List[str]] = None, + ) -> Result[Json]: + """Compute the cluster rebalance plan. + + :param version: Must be set to 1. + :type version: int + :param max_moves: Maximum number of moves to be computed. + :type max_moves: int | None + :param leader_changes: Allow leader changes without moving data. + :type leader_changes: bool | None + :param move_leaders: Allow moving shard leaders. + :type move_leaders: bool | None + :param move_followers: Allow moving shard followers. + :type move_followers: bool | None + :param pi_factor: A weighting factor that should remain untouched. + :type pi_factor: float | None + :param exclude_system_collections: Ignore system collections in the + rebalance plan. + :type exclude_system_collections: bool | None + :param databases_excluded: List of database names to be excluded + from the analysis. + :type databases_excluded: [str] | None + :return: Cluster rebalance plan. + :rtype: dict + :raise: arango.exceptions.ClusterRebalanceError: If retrieval fails. + """ + data: Json = dict(version=version) + if max_moves is not None: + data["maximumNumberOfMoves"] = max_moves + if leader_changes is not None: + data["leaderChanges"] = leader_changes + if move_leaders is not None: + data["moveLeaders"] = move_leaders + if move_followers is not None: + data["moveFollowers"] = move_followers + if pi_factor is not None: + data["piFactor"] = pi_factor + if exclude_system_collections is not None: + data["excludeSystemCollections"] = exclude_system_collections + if databases_excluded is not None: + data["databasesExcluded"] = databases_excluded + + request = Request( + method="post", endpoint="/_admin/cluster/rebalance", data=data + ) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterRebalanceError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) + + def execute_rebalance_plan( + self, moves: List[Json], version: int = 1 + ) -> Result[bool]: + """Execute the given set of move shard operations. + + You can use :meth:`Cluster.calculate_rebalance_plan` to calculate + these operations to improve the balance of shards, leader shards, + and follower shards. + + :param moves: List of move shard operations. + :type moves: [dict] + :param version: Must be set to 1. + :type version: int + :return: True if the methods have been accepted and scheduled + for execution. + :rtype: bool + :raise: arango.exceptions.ClusterRebalanceError: If request fails. + """ + data: Json = dict(version=version, moves=moves) + + request = Request( + method="post", endpoint="/_admin/cluster/rebalance/execute", data=data + ) + + def response_handler(resp: Response) -> bool: + if not resp.is_success: + raise ClusterRebalanceError(resp, request) + result: bool = resp.body["code"] == 202 + return result + + return self._execute(request, response_handler) + + def vpack_sort_migration_status(self) -> Result[Json]: + """Query the status of the vpack sorting migration. + + :return: Status of the VPack sort migration. + :rtype: dict + """ + request = Request( + method="get", endpoint="/_admin/cluster/vpackSortMigration/status" + ) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterVpackSortMigrationError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) + + def vpack_sort_migration_index_check(self) -> Result[Json]: + """Check for indexes impacted by the sorting behavior before 3.12.2. + + :return: Status of indexes. + :rtype: dict + """ + request = Request( + method="get", endpoint="/_admin/cluster/vpackSortMigration/check" + ) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterVpackSortMigrationError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) + + def migrate_vpack_sorting(self) -> Result[Json]: + """Migrate instances to the new VPack sorting behavior. + + :return: Status of the VPack sort migration. + :rtype: dict + """ + request = Request( + method="put", endpoint="/_admin/cluster/vpackSortMigration/migrate" + ) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ClusterVpackSortMigrationError(resp, request) + result: Json = resp.body["result"] + return result + + return self._execute(request, response_handler) diff --git a/arango/collection.py b/arango/collection.py index 3bccb5ec..e2dfcd2a 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -2,6 +2,7 @@ from numbers import Number from typing import List, Optional, Sequence, Tuple, Union +from warnings import warn from arango.api import ApiGroup from arango.connection import Connection @@ -9,13 +10,16 @@ from arango.exceptions import ( ArangoServerError, CollectionChecksumError, + CollectionCompactError, CollectionConfigureError, + CollectionInformationError, CollectionLoadError, CollectionPropertiesError, CollectionRecalculateCountError, CollectionRenameError, CollectionResponsibleShardError, CollectionRevisionError, + CollectionShardsError, CollectionStatisticsError, CollectionTruncateError, CollectionUnloadError, @@ -33,16 +37,27 @@ EdgeListError, IndexCreateError, IndexDeleteError, + IndexGetError, IndexListError, IndexLoadError, + IndexMissingError, ) from arango.executor import ApiExecutor from arango.formatter import format_collection, format_edge, format_index, format_vertex from arango.request import Request from arango.response import Response from arango.result import Result -from arango.typings import Fields, Headers, Json, Params -from arango.utils import get_doc_id, is_none_or_int, is_none_or_str +from arango.typings import Fields, Headers, Json, Jsons, Params +from arango.utils import ( + build_filter_conditions, + build_sort_expression, + get_batches, + get_doc_id, + is_none_or_bool, + is_none_or_int, + is_none_or_str, + validate_sort_parameters, +) class Collection(ApiGroup): @@ -300,8 +315,58 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def shards(self, details: bool = False) -> Result[Json]: + """Return collection shards and properties. + + Available only in a cluster setup. + + :param details: Include responsible servers for each shard. + :type details: bool + :return: Collection shards and properties. + :rtype: dict + :raise arango.exceptions.CollectionShardsError: If retrieval fails. + """ + request = Request( + method="get", + endpoint=f"/_api/collection/{self.name}/shards", + params={"details": details}, + read=self.name, + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_collection(resp.body) + raise CollectionShardsError(resp, request) + + return self._execute(request, response_handler) + + def info(self) -> Result[Json]: + """Return the collection information. + + :return: Information about the collection. + :rtype: dict + :raise arango.exceptions.CollectionInformationError: If retrieval fails. + """ + request = Request( + method="get", + endpoint=f"/_api/collection/{self.name}", + read=self.name, + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_collection(resp.body) + raise CollectionInformationError(resp, request) + + return self._execute(request, response_handler) + def configure( - self, sync: Optional[bool] = None, schema: Optional[Json] = None + self, + sync: Optional[bool] = None, + schema: Optional[Json] = None, + replication_factor: Optional[int] = None, + write_concern: Optional[int] = None, + computed_values: Optional[Jsons] = None, ) -> Result[Json]: """Configure collection properties. @@ -309,7 +374,25 @@ def configure( :type sync: bool | None :param schema: document schema for validation of objects. :type schema: dict + :param replication_factor: Number of copies of each shard on different + servers in a cluster. Allowed values are 1 (only one copy is kept + and no synchronous replication), and n (n-1 replicas are kept and + any two copies are replicated across servers synchronously, meaning + every write to the master is copied to all slaves before operation + is reported successful). + :type replication_factor: int + :param write_concern: Write concern for the collection. Determines how + many copies of each shard are required to be in sync on different + DBServers. If there are less than these many copies in the cluster + a shard will refuse to write. Writes to shards with enough + up-to-date copies will succeed at the same time. The value of this + parameter cannot be larger than that of **replication_factor**. + Default value is 1. Used for clusters only. + :type write_concern: int :return: New collection properties. + :param computed_values: Define expressions on the collection level that + run on inserts, modifications, or both. + :type computed_values: dict | None :rtype: dict :raise arango.exceptions.CollectionConfigureError: If operation fails. """ @@ -318,6 +401,12 @@ def configure( data["waitForSync"] = sync if schema is not None: data["schema"] = schema + if replication_factor is not None: + data["replicationFactor"] = replication_factor + if write_concern is not None: + data["writeConcern"] = write_concern + if computed_values is not None: + data["computedValues"] = computed_values request = Request( method="put", @@ -416,6 +505,35 @@ def response_handler(resp: Response) -> str: return self._execute(request, response_handler) + def compact(self) -> Result[Json]: + """Compact a collection. + + Compacts the data of a collection in order to reclaim disk space. + The operation will compact the document and index data by rewriting the + underlying .sst files and only keeping the relevant entries. + + Under normal circumstances, running a compact operation is not necessary, as + the collection data will eventually get compacted anyway. However, in some + situations, e.g. after running lots of update/replace or remove operations, + the disk data for a collection may contain a lot of outdated data for which the + space shall be reclaimed. In this case the compaction operation can be used. + + :return: Collection compact. + :rtype: dict + :raise arango.exceptions.CollectionCompactError: If retrieval fails. + """ + request = Request( + method="put", + endpoint=f"/_api/collection/{self.name}/compact", + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_collection(resp.body) + raise CollectionCompactError(resp, request) + + return self._execute(request, response_handler) + def load(self) -> Result[bool]: """Load the collection into memory. @@ -488,6 +606,7 @@ def has( document: Union[str, Json], rev: Optional[str] = None, check_rev: bool = True, + allow_dirty_read: bool = False, ) -> Result[bool]: """Check if a document exists in the collection. @@ -500,6 +619,8 @@ def has( :param check_rev: If set to True, revision of **document** (if given) is compared against the revision of target document. :type check_rev: bool + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: True if document exists, False otherwise. :rtype: bool :raise arango.exceptions.DocumentInError: If check fails. @@ -507,36 +628,45 @@ def has( """ handle, body, headers = self._prep_from_doc(document, rev, check_rev) + if allow_dirty_read: + headers["x-arango-allow-dirty-read"] = "true" + request = Request( - method="get", + method="head", endpoint=f"/_api/document/{handle}", headers=headers, read=self.name, ) def response_handler(resp: Response) -> bool: - if resp.error_code == 1202: - return False if resp.status_code == 412: raise DocumentRevisionError(resp, request) + if resp.status_code == 404: + return False if not resp.is_success: raise DocumentInError(resp, request) - return bool(resp.body) + return True return self._execute(request, response_handler) - def ids(self) -> Result[Cursor]: + def ids(self, allow_dirty_read: bool = False) -> Result[Cursor]: """Return the IDs of all documents in the collection. + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Document ID cursor. :rtype: arango.cursor.Cursor :raise arango.exceptions.DocumentIDsError: If retrieval fails. """ + query = "FOR doc IN @@collection RETURN doc._id" + bind_vars = {"@collection": self.name} + request = Request( - method="put", - endpoint="/_api/simple/all-keys", - data={"collection": self.name, "type": "id"}, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -546,18 +676,24 @@ def response_handler(resp: Response) -> Cursor: return self._execute(request, response_handler) - def keys(self) -> Result[Cursor]: + def keys(self, allow_dirty_read: bool = False) -> Result[Cursor]: """Return the keys of all documents in the collection. + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Document key cursor. :rtype: arango.cursor.Cursor :raise arango.exceptions.DocumentKeysError: If retrieval fails. """ + query = "FOR doc IN @@collection RETURN doc._key" + bind_vars = {"@collection": self.name} + request = Request( - method="put", - endpoint="/_api/simple/all-keys", - data={"collection": self.name, "type": "key"}, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -568,7 +704,10 @@ def response_handler(resp: Response) -> Cursor: return self._execute(request, response_handler) def all( - self, skip: Optional[int] = None, limit: Optional[int] = None + self, + skip: Optional[int] = None, + limit: Optional[int] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: """Return all documents in the collection. @@ -576,6 +715,8 @@ def all( :type skip: int | None :param limit: Max number of documents returned. :type limit: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Document cursor. :rtype: arango.cursor.Cursor :raise arango.exceptions.DocumentGetError: If retrieval fails. @@ -583,86 +724,38 @@ def all( assert is_none_or_int(skip), "skip must be a non-negative int" assert is_none_or_int(limit), "limit must be a non-negative int" - data: Json = {"collection": self.name} - if skip is not None: - data["skip"] = skip - if limit is not None: - data["limit"] = limit - - request = Request( - method="put", endpoint="/_api/simple/all", data=data, read=self.name - ) - - def response_handler(resp: Response) -> Cursor: - if not resp.is_success: - raise DocumentGetError(resp, request) - return Cursor(self._conn, resp.body) + skip_val = skip if skip is not None else 0 + limit_val = limit if limit is not None else "null" + query = f""" + FOR doc IN @@collection + LIMIT {skip_val}, {limit_val} + RETURN doc + """ - return self._execute(request, response_handler) + bind_vars = {"@collection": self.name} - def export( - self, - limit: Optional[int] = None, - count: bool = False, - batch_size: Optional[int] = None, - flush: bool = False, - flush_wait: Optional[int] = None, - ttl: Optional[Number] = None, - filter_fields: Optional[Sequence[str]] = None, - filter_type: str = "include", - ) -> Result[Cursor]: - """Export all documents in the collection using a server cursor. - - :param flush: If set to True, flush the write-ahead log prior to the - export. If set to False, documents in the write-ahead log during - the export are not included in the result. - :type flush: bool - :param flush_wait: Max wait time in seconds for write-ahead log flush. - :type flush_wait: int | None - :param count: Include the document count in the server cursor. - :type count: bool - :param batch_size: Max number of documents in the batch fetched by - the cursor in one round trip. - :type batch_size: int | None - :param limit: Max number of documents fetched by the cursor. - :type limit: int | None - :param ttl: Time-to-live for the cursor on the server. - :type ttl: int | float | None - :param filter_fields: Document fields to filter with. - :type filter_fields: [str] | None - :param filter_type: Allowed values are "include" or "exclude". - :type filter_type: str - :return: Document cursor. - :rtype: arango.cursor.Cursor - :raise arango.exceptions.DocumentGetError: If export fails. - """ - data: Json = {"count": count, "flush": flush} - if flush_wait is not None: - data["flushWait"] = flush_wait - if batch_size is not None: - data["batchSize"] = batch_size - if limit is not None: - data["limit"] = limit - if ttl is not None: - data["ttl"] = ttl - if filter_fields is not None: - data["restrict"] = {"fields": filter_fields, "type": filter_type} request = Request( method="post", - endpoint="/_api/export", - params={"collection": self.name}, - data=data, + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars, "count": True}, + read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: if not resp.is_success: raise DocumentGetError(resp, request) - return Cursor(self._conn, resp.body, "export") + return Cursor(self._conn, resp.body) return self._execute(request, response_handler) def find( - self, filters: Json, skip: Optional[int] = None, limit: Optional[int] = None + self, + filters: Json, + skip: Optional[int] = None, + limit: Optional[int] = None, + allow_dirty_read: bool = False, + sort: Optional[Jsons] = None, ) -> Result[Cursor]: """Return all documents that match the given filters. @@ -672,24 +765,38 @@ def find( :type skip: int | None :param limit: Max number of documents returned. :type limit: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool + :param sort: Document sort parameters + :type sort: Jsons | None :return: Document cursor. :rtype: arango.cursor.Cursor :raise arango.exceptions.DocumentGetError: If retrieval fails. + :raise arango.exceptions.SortValidationError: If sort parameters are invalid. """ assert isinstance(filters, dict), "filters must be a dict" assert is_none_or_int(skip), "skip must be a non-negative int" assert is_none_or_int(limit), "limit must be a non-negative int" + if sort: + validate_sort_parameters(sort) - data: Json = { - "collection": self.name, - "example": filters, - "skip": skip, - } - if limit is not None: - data["limit"] = limit + skip_val = skip if skip is not None else 0 + limit_val = limit if limit is not None else "null" + query = f""" + FOR doc IN @@collection + {build_filter_conditions(filters)} + LIMIT {skip_val}, {limit_val} + {build_sort_expression(sort)} + RETURN doc + """ + bind_vars = {"@collection": self.name} request = Request( - method="put", endpoint="/_api/simple/by-example", data=data, read=self.name + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars, "count": True}, + read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -700,7 +807,11 @@ def response_handler(resp: Response) -> Cursor: return self._execute(request, response_handler) def find_near( - self, latitude: Number, longitude: Number, limit: Optional[int] = None + self, + latitude: Number, + longitude: Number, + limit: Optional[int] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: """Return documents near a given coordinate. @@ -715,6 +826,8 @@ def find_near( :type longitude: int | float :param limit: Max number of documents returned. :type limit: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :returns: Document cursor. :rtype: arango.cursor.Cursor :raises arango.exceptions.DocumentGetError: If retrieval fails. @@ -743,6 +856,7 @@ def find_near( endpoint="/_api/cursor", data={"query": query, "bindVars": bind_vars, "count": True}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -759,6 +873,7 @@ def find_in_range( upper: int, skip: Optional[int] = None, limit: Optional[int] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: """Return documents within a given range in a random order. @@ -774,6 +889,8 @@ def find_in_range( :type skip: int | None :param limit: Max number of documents returned. :type limit: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :returns: Document cursor. :rtype: arango.cursor.Cursor :raises arango.exceptions.DocumentGetError: If retrieval fails. @@ -802,6 +919,7 @@ def find_in_range( endpoint="/_api/cursor", data={"query": query, "bindVars": bind_vars, "count": True}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -817,6 +935,7 @@ def find_in_radius( longitude: Number, radius: Number, distance_field: Optional[str] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: """Return documents within a given radius around a coordinate. @@ -831,6 +950,8 @@ def find_in_radius( :param distance_field: Document field used to indicate the distance to the given coordinate. This parameter is ignored in transactions. :type distance_field: str + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :returns: Document cursor. :rtype: arango.cursor.Cursor :raises arango.exceptions.DocumentGetError: If retrieval fails. @@ -861,6 +982,7 @@ def find_in_radius( endpoint="/_api/cursor", data={"query": query, "bindVars": bind_vars, "count": True}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -879,8 +1001,9 @@ def find_in_box( skip: Optional[int] = None, limit: Optional[int] = None, index: Optional[str] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: - """Return all documents in an rectangular area. + """Return all documents in a rectangular area. :param latitude1: First latitude. :type latitude1: int | float @@ -897,6 +1020,8 @@ def find_in_box( :param index: ID of the geo index to use (without the collection prefix). This parameter is ignored in transactions. :type index: str | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :returns: Document cursor. :rtype: arango.cursor.Cursor :raises arango.exceptions.DocumentGetError: If retrieval fails. @@ -907,26 +1032,67 @@ def find_in_box( assert isinstance(longitude2, Number), "longitude2 must be a number" assert is_none_or_int(skip), "skip must be a non-negative int" assert is_none_or_int(limit), "limit must be a non-negative int" + assert is_none_or_str(index), "index must be a string" + + def build_coord_str_from_index(index: Json) -> str: + index_field: List[str] = index["fields"] + + if len(index_field) == 1: + return f"doc.{index_field[0]}" + elif len(index_field) == 2: + return f"[doc.{index_field[0]}, doc.{index_field[1]}]" + else: # pragma: no cover + m = "**index** must be a geo index with 1 or 2 fields" + raise ValueError(m) + + coord_str = "" + if index is None: + # Find the first geo index + for collection_index in self.indexes(): # type:ignore[union-attr] + if collection_index["type"] == "geo": + coord_str = build_coord_str_from_index(collection_index) + break + + # If no geo index found, raise + if coord_str == "": + raise IndexMissingError(f"No geo index found in {self.name}") - data: Json = { - "collection": self._name, - "latitude1": latitude1, - "longitude1": longitude1, - "latitude2": latitude2, - "longitude2": longitude2, - } - if skip is not None: - data["skip"] = skip - if limit is not None: - data["limit"] = limit - if index is not None: - data["geo"] = self._name + "/" + index + else: + # Find the geo index with the given ID + geo_index = self.get_index(index) + assert isinstance(geo_index, dict) + + # If the index is not a geo index, raise + if geo_index["type"] != "geo": + raise ValueError("**index** must point to a Geo Index") + + coord_str = build_coord_str_from_index(geo_index) + + skip_val = skip if skip is not None else 0 + limit_val = limit if limit is not None else "null" + query = f""" + LET rect = GEO_POLYGON([ [ + [{longitude1}, {latitude1}], // bottom-left + [{longitude2}, {latitude1}], // bottom-right + [{longitude2}, {latitude2}], // top-right + [{longitude1}, {latitude2}], // top-left + [{longitude1}, {latitude1}], // bottom-left (close polygon) + ] ]) + + FOR doc IN @@collection + FILTER GEO_CONTAINS(rect, {coord_str}) + LIMIT {skip_val}, {limit_val} + RETURN doc + """ # noqa: E201 E202 + + bind_vars = {"@collection": self.name} request = Request( - method="put", - endpoint="/_api/simple/within-rectangle", - data=data, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars, "count": True}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -937,7 +1103,11 @@ def response_handler(resp: Response) -> Cursor: return self._execute(request, response_handler) def find_by_text( - self, field: str, query: str, limit: Optional[int] = None + self, + field: str, + query: str, + limit: Optional[int] = None, + allow_dirty_read: bool = False, ) -> Result[Cursor]: """Return documents that match the given fulltext query. @@ -947,6 +1117,8 @@ def find_by_text( :type query: str :param limit: Max number of documents returned. :type limit: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :returns: Document cursor. :rtype: arango.cursor.Cursor :raises arango.exceptions.DocumentGetError: If retrieval fails. @@ -973,6 +1145,7 @@ def find_by_text( endpoint="/_api/cursor", data={"query": aql, "bindVars": bind_vars, "count": True}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Cursor: @@ -982,52 +1155,69 @@ def response_handler(resp: Response) -> Cursor: return self._execute(request, response_handler) - def get_many(self, documents: Sequence[Union[str, Json]]) -> Result[List[Json]]: + def get_many( + self, + documents: Sequence[Union[str, Json]], + allow_dirty_read: bool = False, + ) -> Result[Jsons]: """Return multiple documents ignoring any missing ones. :param documents: List of document keys, IDs or bodies. Document bodies must contain the "_id" or "_key" fields. :type documents: [str | dict] + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Documents. Missing ones are not included. :rtype: [dict] :raise arango.exceptions.DocumentGetError: If retrieval fails. """ handles = [self._extract_id(d) if isinstance(d, dict) else d for d in documents] + params: Params = {"onlyget": True} + request = Request( method="put", - endpoint="/_api/simple/lookup-by-keys", - data={"collection": self.name, "keys": handles}, + endpoint=f"/_api/document/{self.name}", + params=params, + data=handles, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) - def response_handler(resp: Response) -> List[Json]: + def response_handler(resp: Response) -> Jsons: if not resp.is_success: raise DocumentGetError(resp, request) - docs = resp.body["documents"] - return [doc for doc in docs if "_id" in doc] + return [doc for doc in resp.body if "_id" in doc] return self._execute(request, response_handler) - def random(self) -> Result[Json]: - """Return a random document from the collection. + def random(self, allow_dirty_read: bool = False) -> Result[Optional[Json]]: + """Return a random document from the collection. Returns None + if the collection is empty. + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: A random document. :rtype: dict :raise arango.exceptions.DocumentGetError: If retrieval fails. """ + query = "FOR doc IN @@collection SORT RAND() LIMIT 1 RETURN doc" + bind_vars = {"@collection": self.name} + request = Request( - method="put", - endpoint="/_api/simple/any", - data={"collection": self.name}, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) - def response_handler(resp: Response) -> Json: - if resp.is_success: - result: Json = resp.body["document"] - return result - raise DocumentGetError(resp, request) + def response_handler(resp: Response) -> Optional[Json]: + if not resp.is_success: + raise DocumentGetError(resp, request) + + cursor = Cursor(self._conn, resp.body) + return cursor.pop() if not cursor.empty() else None return self._execute(request, response_handler) @@ -1035,7 +1225,7 @@ def response_handler(resp: Response) -> Json: # Index Management # #################### - def indexes(self) -> Result[List[Json]]: + def indexes(self) -> Result[Jsons]: """Return the collection indexes. :return: Collection indexes. @@ -1048,7 +1238,7 @@ def indexes(self) -> Result[List[Json]]: params={"collection": self.name}, ) - def response_handler(resp: Response) -> List[Json]: + def response_handler(resp: Response) -> Jsons: if not resp.is_success: raise IndexListError(resp, request) result = resp.body["indexes"] @@ -1056,11 +1246,49 @@ def response_handler(resp: Response) -> List[Json]: return self._execute(request, response_handler) - def _add_index(self, data: Json) -> Result[Json]: - """Helper method for creating a new index. + def get_index(self, id: str) -> Result[Json]: + """Return the index with the given id. + + :param id: The id of the index + :type id: str + :return: Index details + :rtype: dict + :raise arango.exceptions.IndexGetError: If retrieval fails. + """ + request = Request( + method="get", + endpoint=f"/_api/index/{self.name}/{id}", + ) - :param data: Index data. + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise IndexGetError(resp, request) + + return format_index(resp.body) + + return self._execute(request, response_handler) + + def add_index(self, data: Json, formatter: bool = False) -> Result[Json]: + """Create an index. + + .. note:: + + As the `add_index` method was made available starting with driver + version 8, we have decided to deprecate the other `add_*_index` + methods, making this the official way to create indexes. While + the other methods still work, we recommend using this one instead. + Note that the other methods would use a formatter by default, + processing the index attributes returned by the server (for the + most part, it does a snake case conversion). This method skips that, + returning the raw index, except for the `id` attribute. However, + if you want the formatter to be applied for backwards compatibility, + you can set the `formatter` parameter to `True`. + + :param data: Index data. Must contain a "type" and "fields" attribute. :type data: dict + :param formatter: If set to True, apply formatting to the returned result. + Should only be used for backwards compatibility. + :type formatter: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. @@ -1075,7 +1303,7 @@ def _add_index(self, data: Json) -> Result[Json]: def response_handler(resp: Response) -> Json: if not resp.is_success: raise IndexCreateError(resp, request) - return format_index(resp.body) + return format_index(resp.body, formatter) return self._execute(request, response_handler) @@ -1090,6 +1318,12 @@ def add_hash_index( ) -> Result[Json]: """Create a new hash index. + .. warning:: + + The index types `hash` and `skiplist` are aliases for the persistent + index type and should no longer be used to create new indexes. The + aliases will be removed in a future version. + :param fields: Document fields to index. :type fields: [str] :param unique: Whether the index is unique. @@ -1108,6 +1342,9 @@ def add_hash_index( :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_hash_index is deprecated. Using add_index with {'type': 'hash'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "hash", "fields": fields} if unique is not None: @@ -1121,7 +1358,7 @@ def add_hash_index( if in_background is not None: data["inBackground"] = in_background - return self._add_index(data) + return self.add_index(data, formatter=True) def add_skiplist_index( self, @@ -1134,6 +1371,12 @@ def add_skiplist_index( ) -> Result[Json]: """Create a new skiplist index. + .. warning:: + + The index types `hash` and `skiplist` are aliases for the persistent + index type and should no longer be used to create new indexes. The + aliases will be removed in a future version. + :param fields: Document fields to index. :type fields: [str] :param unique: Whether the index is unique. @@ -1152,6 +1395,9 @@ def add_skiplist_index( :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_skiplist_index is deprecated. Using add_index with {'type': 'skiplist'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "skiplist", "fields": fields} if unique is not None: @@ -1165,14 +1411,15 @@ def add_skiplist_index( if in_background is not None: data["inBackground"] = in_background - return self._add_index(data) + return self.add_index(data, formatter=True) def add_geo_index( self, fields: Fields, - ordered: Optional[bool] = None, + geo_json: Optional[bool] = None, name: Optional[str] = None, in_background: Optional[bool] = None, + legacyPolygons: Optional[bool] = False, ) -> Result[Json]: """Create a new geo-spatial index. @@ -1181,26 +1428,36 @@ def add_geo_index( with at least two floats. Documents with missing fields or invalid values are excluded. :type fields: str | [str] - :param ordered: Whether the order is longitude, then latitude. - :type ordered: bool | None + :param geo_json: Whether to use GeoJSON data-format or not. This + parameter has been renamed from `ordered`. See Github Issue + #234 for more details. + :type geo_json: bool | None :param name: Optional name for the index. :type name: str | None :param in_background: Do not hold the collection lock. :type in_background: bool | None + :param legacyPolygons: Whether or not to use use the old, pre-3.10 rules + for the parsing GeoJSON polygons + :type legacyPolygons: bool | None :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_geo_index is deprecated. Using add_index with {'type': 'geo'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "geo", "fields": fields} - if ordered is not None: - data["geoJson"] = ordered + if geo_json is not None: + data["geoJson"] = geo_json if name is not None: data["name"] = name if in_background is not None: data["inBackground"] = in_background + if legacyPolygons is not None: + data["legacyPolygons"] = legacyPolygons - return self._add_index(data) + return self.add_index(data, formatter=True) def add_fulltext_index( self, @@ -1211,6 +1468,10 @@ def add_fulltext_index( ) -> Result[Json]: """Create a new fulltext index. + .. warning:: + This method is deprecated since ArangoDB 3.10 and will be removed + in a future version of the driver. + :param fields: Document fields to index. :type fields: [str] :param min_length: Minimum number of characters to index. @@ -1223,6 +1484,9 @@ def add_fulltext_index( :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_fulltext_index is deprecated. Using add_index with {'type': 'fulltext'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "fulltext", "fields": fields} if min_length is not None: @@ -1232,7 +1496,7 @@ def add_fulltext_index( if in_background is not None: data["inBackground"] = in_background - return self._add_index(data) + return self.add_index(data, formatter=True) def add_persistent_index( self, @@ -1241,6 +1505,8 @@ def add_persistent_index( sparse: Optional[bool] = None, name: Optional[str] = None, in_background: Optional[bool] = None, + storedValues: Optional[Sequence[str]] = None, + cacheEnabled: Optional[bool] = None, ) -> Result[Json]: """Create a new persistent index. @@ -1259,10 +1525,23 @@ def add_persistent_index( :type name: str | None :param in_background: Do not hold the collection lock. :type in_background: bool | None + :param storedValues: Additional attributes to include in a persistent + index. These additional attributes cannot be used for index + lookups or sorts, but they can be used for projections. Must be + an array of index attribute paths. There must be no overlap of + attribute paths between fields and storedValues. The maximum + number of values is 32. + :type storedValues: [str] + :param cacheEnabled: Enable an in-memory cache for index values for + persistent indexes. + :type cacheEnabled: bool | None :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_persistent_index is deprecated. Using add_index with {'type': 'persistent'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "persistent", "fields": fields} if unique is not None: @@ -1273,8 +1552,12 @@ def add_persistent_index( data["name"] = name if in_background is not None: data["inBackground"] = in_background + if storedValues is not None: + data["storedValues"] = storedValues + if cacheEnabled is not None: + data["cacheEnabled"] = cacheEnabled - return self._add_index(data) + return self.add_index(data, formatter=True) def add_ttl_index( self, @@ -1297,6 +1580,9 @@ def add_ttl_index( :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ + m = "add_ttl_index is deprecated. Using add_index with {'type': 'ttl'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + data: Json = {"type": "ttl", "fields": fields, "expireAfter": expiry_time} if name is not None: @@ -1304,7 +1590,94 @@ def add_ttl_index( if in_background is not None: data["inBackground"] = in_background - return self._add_index(data) + return self.add_index(data, formatter=True) + + def add_inverted_index( + self, + fields: Json, + name: Optional[str] = None, + inBackground: Optional[bool] = None, + parallelism: Optional[int] = None, + primarySort: Optional[Json] = None, + storedValues: Optional[Sequence[Json]] = None, + analyzer: Optional[str] = None, + features: Optional[Sequence[str]] = None, + includeAllFields: Optional[bool] = None, + trackListPositions: Optional[bool] = None, + searchField: Optional[bool] = None, + primaryKeyCache: Optional[bool] = None, + cache: Optional[bool] = None, + ) -> Result[Json]: + """Create a new inverted index, introduced in version 3.10. + + :param fields: Document fields to index. + :type fields: Json + :param name: Optional name for the index. + :type name: str | None + :param inBackground: Do not hold the collection lock. + :type inBackground: bool | None + :param parallelism: The number of threads to use for indexing the fields. + :type parallelism: int | None + :param primarySort: Primary sort order to enable an AQL optimization. + :type primarySort: Optional[Json] + :param storedValues: An array of objects with paths to additional + attributes to store in the index. + :type storedValues: Sequence[Json] | None + :param analyzer: Analyzer to use by default. + :type analyzer: Optional[str] + :param features: List of Analyzer features. + :type features: Sequence[str] | None + :param includeAllFields: This option only applies if you use the + inverted index in search-alias views. + :type includeAllFields: bool | None + :param trackListPositions: This option only applies if you use the + inverted index in search-alias views, and searchField is true. + :type trackListPositions: bool | None + :param searchField: This option only applies if you use the inverted + index in search-alias views + :type searchField: bool | None + :param primaryKeyCache: Always cache the primary key column in memory. + :type primaryKeyCache: bool | None + :param cache: Always cache the field normalization values in memory + for all fields by default. + :type cache: bool | None + :return: New index details. + :rtype: dict + :raise arango.exceptions.IndexCreateError: If create fails. + """ + m = "add_inverted_index is deprecated. Using add_index with {'type': 'inverted'} instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + + data: Json = {"type": "inverted", "fields": fields} + + if name is not None: + data["name"] = name + if inBackground is not None: + data["inBackground"] = inBackground + if parallelism is not None: + data["parallelism"] = parallelism + if primarySort is not None: + data["primarySort"] = primarySort + if storedValues is not None: + data["storedValues"] = storedValues + if analyzer is not None: + data["analyzer"] = analyzer + if features is not None: + data["features"] = features + if includeAllFields is not None: + data["includeAllFields"] = includeAllFields + if trackListPositions is not None: + data["trackListPositions"] = trackListPositions + if searchField is not None: + data["searchField"] = searchField + if fields is not None: + data["fields"] = fields + if primaryKeyCache is not None: + data["primaryKeyCache"] = primaryKeyCache + if cache is not None: + data["cache"] = cache + + return self.add_index(data, formatter=True) def delete_index(self, index_id: str, ignore_missing: bool = False) -> Result[bool]: """Delete an index. @@ -1358,6 +1731,11 @@ def insert_many( silent: bool = False, overwrite: bool = False, return_old: bool = False, + overwrite_mode: Optional[str] = None, + keep_none: Optional[bool] = None, + merge: Optional[bool] = None, + refill_index_caches: Optional[bool] = None, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Insert multiple documents. @@ -1395,6 +1773,26 @@ def insert_many( :param return_old: Include body of the old documents if replaced. Applies only when value of **overwrite** is set to True. :type return_old: bool + :param overwrite_mode: Overwrite behavior used when the document key + exists already. Allowed values are "replace" (replace-insert), + "update" (update-insert), "ignore" or "conflict". + Implicitly sets the value of parameter **overwrite**. + :type overwrite_mode: str | None + :param keep_none: If set to True, fields with value None are retained + in the document. Otherwise, they are removed completely. Applies + only when **overwrite_mode** is set to "update" (update-insert). + :type keep_none: bool | None + :param merge: If set to True (default), sub-dictionaries are merged + instead of the new one overwriting the old one. Applies only when + **overwrite_mode** is set to "update" (update-insert). + :type merge: bool | None + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param version_attribute: support for simple external versioning to + document operations. + :type version_attribute: str :return: List of document metadata (e.g. document keys, revisions) and any exception, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1411,6 +1809,19 @@ def insert_many( if sync is not None: params["waitForSync"] = sync + if overwrite_mode is not None: + params["overwriteMode"] = overwrite_mode + if keep_none is not None: + params["keepNull"] = keep_none + if merge is not None: + params["mergeObjects"] = merge + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="post", endpoint=f"/_api/document/{self.name}", @@ -1450,6 +1861,9 @@ def update_many( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, + raise_on_document_error: bool = False, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Update multiple documents. @@ -1459,7 +1873,8 @@ def update_many( returned as an object in the result list. It is up to you to inspect the list to determine which documents were updated successfully (returns document metadata) and which were not - (returns exception object). + (returns exception object). Alternatively, you can rely on + setting **raise_on_document_error** to True (defaults to False). .. note:: @@ -1492,6 +1907,18 @@ def update_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param raise_on_document_error: Whether to raise if a DocumentRevisionError + or a DocumentUpdateError is encountered on an individual document, + as opposed to returning the error as an object in the result list. + Defaults to False. + :type raise_on_document_error: bool + :param version_attribute: support for simple external versioning to + document operations. + :type version_attribute: str :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoError] | bool @@ -1508,6 +1935,12 @@ def update_many( } if sync is not None: params["waitForSync"] = sync + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches documents = [self._ensure_key_in_body(doc) for doc in documents] @@ -1541,6 +1974,9 @@ def response_handler( else: # pragma: no cover error = DocumentUpdateError(sub_resp, request) + if raise_on_document_error: + raise error + results.append(error) return results @@ -1555,6 +1991,7 @@ def update_match( keep_none: bool = True, sync: Optional[bool] = None, merge: bool = True, + allow_dirty_read: bool = False, ) -> Result[int]: """Update matching documents. @@ -1582,32 +2019,46 @@ def update_match( :param merge: If set to True, sub-dictionaries are merged instead of the new ones overwriting the old ones. :type merge: bool | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Number of documents updated. :rtype: int :raise arango.exceptions.DocumentUpdateError: If update fails. """ - data: Json = { - "collection": self.name, - "example": filters, - "newValue": body, - "keepNull": keep_none, - "mergeObjects": merge, + assert isinstance(filters, dict), "filters must be a dict" + assert is_none_or_int(limit), "limit must be a non-negative int" + assert is_none_or_bool(sync), "sync must be None or a bool" + + # If the waitForSync parameter is not specified or set to false, + # then the collection’s default waitForSync behavior is applied. + sync_val = f", waitForSync: {sync}" if sync is not None else "" + + query = f""" + FOR doc IN @@collection + {build_filter_conditions(filters)} + {f"LIMIT {limit}" if limit is not None else ""} + UPDATE doc WITH @body IN @@collection + OPTIONS {{ keepNull: @keep_none, mergeObjects: @merge {sync_val} }} + """ # noqa: E201 E202 + + bind_vars = { + "@collection": self.name, + "body": body, + "keep_none": keep_none, + "merge": merge, } - if limit is not None: - data["limit"] = limit - if sync is not None: - data["waitForSync"] = sync request = Request( - method="put", - endpoint="/_api/simple/update-by-example", - data=data, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, write=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> int: if resp.is_success: - result: int = resp.body["updated"] + result: int = resp.body["extra"]["stats"]["writesExecuted"] return result raise DocumentUpdateError(resp, request) @@ -1621,6 +2072,8 @@ def replace_many( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Replace multiple documents. @@ -1658,6 +2111,13 @@ def replace_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param version_attribute: support for simple external versioning to + document operations. + :type version_attribute: str :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1672,6 +2132,12 @@ def replace_many( } if sync is not None: params["waitForSync"] = sync + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches documents = [self._ensure_key_in_body(doc) for doc in documents] @@ -1717,6 +2183,7 @@ def replace_match( body: Json, limit: Optional[int] = None, sync: Optional[bool] = None, + allow_dirty_read: bool = False, ) -> Result[int]: """Replace matching documents. @@ -1737,28 +2204,43 @@ def replace_match( :type limit: int | None :param sync: Block until operation is synchronized to disk. :type sync: bool | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Number of documents replaced. :rtype: int :raise arango.exceptions.DocumentReplaceError: If replace fails. """ - data: Json = {"collection": self.name, "example": filters, "newValue": body} - if limit is not None: - data["limit"] = limit - if sync is not None: - data["waitForSync"] = sync + assert isinstance(filters, dict), "filters must be a dict" + assert is_none_or_int(limit), "limit must be a non-negative int" + assert is_none_or_bool(sync), "sync must be None or a bool" + + # If the waitForSync parameter is not specified or set to false, + # then the collection’s default waitForSync behavior is applied. + sync_val = f"waitForSync: {sync}" if sync is not None else "" + + query = f""" + FOR doc IN @@collection + {build_filter_conditions(filters)} + {f"LIMIT {limit}" if limit is not None else ""} + REPLACE doc WITH @body IN @@collection + {f"OPTIONS {{ {sync_val} }}" if sync_val else ""} + """ # noqa: E201 E202 + + bind_vars = {"@collection": self.name, "body": body} request = Request( - method="put", - endpoint="/_api/simple/replace-by-example", - data=data, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, write=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> int: - if not resp.is_success: - raise DocumentReplaceError(resp, request) - result: int = resp.body["replaced"] - return result + if resp.is_success: + result: int = resp.body["extra"]["stats"]["writesExecuted"] + return result + raise DocumentReplaceError(resp, request) return self._execute(request, response_handler) @@ -1769,6 +2251,7 @@ def delete_many( check_rev: bool = True, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Delete multiple documents. @@ -1801,6 +2284,10 @@ def delete_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1815,6 +2302,10 @@ def delete_many( if sync is not None: params["waitForSync"] = sync + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillCaches"] = refill_index_caches + documents = [ self._ensure_key_in_body(doc) if isinstance(doc, dict) else doc for doc in documents @@ -1855,7 +2346,11 @@ def response_handler( return self._execute(request, response_handler) def delete_match( - self, filters: Json, limit: Optional[int] = None, sync: Optional[bool] = None + self, + filters: Json, + limit: Optional[int] = None, + sync: Optional[bool] = None, + allow_dirty_read: bool = False, ) -> Result[int]: """Delete matching documents. @@ -1874,26 +2369,41 @@ def delete_match( :type limit: int | None :param sync: Block until operation is synchronized to disk. :type sync: bool | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Number of documents deleted. :rtype: int :raise arango.exceptions.DocumentDeleteError: If delete fails. """ - data: Json = {"collection": self.name, "example": filters} - if sync is not None: - data["waitForSync"] = sync - if limit is not None and limit != 0: - data["limit"] = limit + assert isinstance(filters, dict), "filters must be a dict" + assert is_none_or_int(limit), "limit must be a non-negative int" + assert is_none_or_bool(sync), "sync must be None or a bool" + + # If the waitForSync parameter is not specified or set to false, + # then the collection’s default waitForSync behavior is applied. + sync_val = f"waitForSync: {sync}" if sync is not None else "" + + query = f""" + FOR doc IN @@collection + {build_filter_conditions(filters)} + {f"LIMIT {limit}" if limit is not None else ""} + REMOVE doc IN @@collection + {f"OPTIONS {{ {sync_val} }}" if sync_val else ""} + """ # noqa: E201 E202 + + bind_vars = {"@collection": self.name} request = Request( - method="put", - endpoint="/_api/simple/remove-by-example", - data=data, + method="post", + endpoint="/_api/cursor", + data={"query": query, "bindVars": bind_vars}, write=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> int: if resp.is_success: - result: int = resp.body["deleted"] + result: int = resp.body["extra"]["stats"]["writesExecuted"] return result raise DocumentDeleteError(resp, request) @@ -1909,7 +2419,8 @@ def import_bulk( overwrite: Optional[bool] = None, on_duplicate: Optional[str] = None, sync: Optional[bool] = None, - ) -> Result[Json]: + batch_size: Optional[int] = None, + ) -> Union[Result[Json], List[Result[Json]]]: """Insert multiple documents into the collection. .. note:: @@ -1959,10 +2470,24 @@ def import_bulk( :type on_duplicate: str :param sync: Block until operation is synchronized to disk. :type sync: bool | None + :param batch_size: Split up **documents** into batches of max length + **batch_size** and import them in a loop on the client side. If + **batch_size** is specified, the return type of this method + changes from a result object to a list of result objects. + IMPORTANT NOTE: this parameter may go through breaking changes + in the future where the return type may not be a list of result + objects anymore. Use it at your own risk, and avoid + depending on the return value if possible. Cannot be used with + parameter **overwrite**. + :type batch_size: int :return: Result of the bulk import. - :rtype: dict + :rtype: dict | list[dict] :raise arango.exceptions.DocumentInsertError: If import fails. """ + if overwrite and batch_size is not None: + msg = "Cannot use parameter 'batch_size' if 'overwrite' is set to True" + raise ValueError(msg) + documents = [self._ensure_key_from_id(doc) for doc in documents] params: Params = {"type": "array", "collection": self.name} @@ -1981,21 +2506,35 @@ def import_bulk( if sync is not None: params["waitForSync"] = sync - request = Request( - method="post", - endpoint="/_api/import", - data=documents, - params=params, - write=self.name, - ) - def response_handler(resp: Response) -> Json: if resp.is_success: result: Json = resp.body return result raise DocumentInsertError(resp, request) - return self._execute(request, response_handler) + if batch_size is None: + request = Request( + method="post", + endpoint="/_api/import", + data=documents, + params=params, + write=self.name, + ) + + return self._execute(request, response_handler) + else: + results = [] + for batch in get_batches(documents, batch_size): + request = Request( + method="post", + endpoint="/_api/import", + data=batch, + params=params, + write=self.name, + ) + results.append(self._execute(request, response_handler)) + + return results class StandardCollection(Collection): @@ -2012,6 +2551,7 @@ def get( document: Union[str, Json], rev: Optional[str] = None, check_rev: bool = True, + allow_dirty_read: bool = False, ) -> Result[Optional[Json]]: """Return a document. @@ -2024,6 +2564,8 @@ def get( :param check_rev: If set to True, revision of **document** (if given) is compared against the revision of target document. :type check_rev: bool + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: Document, or None if not found. :rtype: dict | None :raise arango.exceptions.DocumentGetError: If retrieval fails. @@ -2031,6 +2573,9 @@ def get( """ handle, body, headers = self._prep_from_doc(document, rev, check_rev) + if allow_dirty_read: + headers["x-arango-allow-dirty-read"] = "true" + request = Request( method="get", endpoint=f"/_api/document/{handle}", @@ -2062,6 +2607,8 @@ def insert( overwrite_mode: Optional[str] = None, keep_none: Optional[bool] = None, merge: Optional[bool] = None, + refill_index_caches: Optional[bool] = None, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, Json]]: """Insert a new document. @@ -2096,6 +2643,13 @@ def insert( instead of the new one overwriting the old one. Applies only when **overwrite_mode** is set to "update" (update-insert). :type merge: bool | None + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param version_attribute: support for simple external versioning to + document operations. + :type version_attribute: str :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2117,6 +2671,12 @@ def insert( params["keepNull"] = keep_none if merge is not None: params["mergeObjects"] = merge + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches request = Request( method="post", @@ -2150,6 +2710,8 @@ def update( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, Json]]: """Update a document. @@ -2176,6 +2738,13 @@ def update( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param version_attribute: support for simple external versioning + to document operations. + :type version_attribute: str :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2194,6 +2763,13 @@ def update( if sync is not None: params["waitForSync"] = sync + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="patch", endpoint=f"/_api/document/{self._extract_id(document)}", @@ -2224,6 +2800,8 @@ def replace( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, + version_attribute: Optional[str] = None, ) -> Result[Union[bool, Json]]: """Replace a document. @@ -2245,6 +2823,13 @@ def replace( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None + :param version_attribute: support for simple external versioning to + document operations. + :type version_attribute: str :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2261,6 +2846,13 @@ def replace( if sync is not None: params["waitForSync"] = sync + if version_attribute is not None: + params["versionAttribute"] = version_attribute + + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="put", endpoint=f"/_api/document/{self._extract_id(document)}", @@ -2294,6 +2886,7 @@ def delete( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, Json]]: """Delete a document. @@ -2318,6 +2911,10 @@ def delete( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: Document metadata (e.g. document key, revision), or True if parameter **silent** was set to True, or False if document was not found and **ignore_missing** was set to True (does not apply in @@ -2337,6 +2934,10 @@ def delete( if sync is not None: params["waitForSync"] = sync + # New in ArangoDB 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="delete", endpoint=f"/_api/document/{handle}", @@ -3033,7 +3634,10 @@ def link( return self.insert(edge, sync=sync, silent=silent, return_new=return_new) def edges( - self, vertex: Union[str, Json], direction: Optional[str] = None + self, + vertex: Union[str, Json], + direction: Optional[str] = None, + allow_dirty_read: bool = False, ) -> Result[Json]: """Return the edge documents coming in and/or out of the vertex. @@ -3042,6 +3646,8 @@ def edges( :param direction: The direction of the edges. Allowed values are "in" and "out". If not set, edges in both directions are returned. :type direction: str + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool :return: List of edges and statistics. :rtype: dict :raise arango.exceptions.EdgeListError: If retrieval fails. @@ -3055,6 +3661,7 @@ def edges( endpoint=f"/_api/edges/{self.name}", params=params, read=self.name, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, ) def response_handler(resp: Response) -> Json: diff --git a/arango/connection.py b/arango/connection.py index 658e43fb..8de2643a 100644 --- a/arango/connection.py +++ b/arango/connection.py @@ -6,17 +6,24 @@ "JwtSuperuserConnection", ] +import logging import sys import time from abc import abstractmethod -from typing import Any, Callable, Optional, Sequence, Union +from typing import Any, Callable, Optional, Sequence, Set, Tuple, Union import jwt -from requests import Session +from jwt.exceptions import ExpiredSignatureError +from requests import ConnectionError, Session from requests_toolbelt import MultipartEncoder -from arango.exceptions import JWTAuthError, ServerConnectionError -from arango.http import HTTPClient +from arango.exceptions import ( + JWTAuthError, + JWTExpiredError, + JWTRefreshError, + ServerConnectionError, +) +from arango.http import HTTPClient, RequestCompression from arango.request import Request from arango.resolver import HostResolver from arango.response import Response @@ -37,7 +44,10 @@ def __init__( http_client: HTTPClient, serializer: Callable[..., str], deserializer: Callable[[str], Any], + request_compression: Optional[RequestCompression] = None, + response_compression: Optional[str] = None, ) -> None: + self._hosts = hosts self._url_prefixes = [f"{host}/_db/{db_name}" for host in hosts] self._host_resolver = host_resolver self._sessions = sessions @@ -46,6 +56,8 @@ def __init__( self._serializer = serializer self._deserializer = deserializer self._username: Optional[str] = None + self._request_compression = request_compression + self._response_compression = response_compression @property def db_name(self) -> str: @@ -103,6 +115,8 @@ def prep_response(self, resp: Response, deserialize: bool = True) -> Response: if isinstance(resp.body, dict): resp.error_code = resp.body.get("errorNum") resp.error_message = resp.body.get("errorMessage") + if resp.status_code == resp.error_code == 503: + raise ConnectionError # Fallback to another host else: resp.body = resp.raw_body @@ -110,6 +124,61 @@ def prep_response(self, resp: Response, deserialize: bool = True) -> Response: resp.is_success = http_ok and resp.error_code is None return resp + def process_request( + self, host_index: int, request: Request, auth: Optional[Tuple[str, str]] = None + ) -> Response: + """Execute a request until a valid response has been returned. + + :param host_index: The index of the first host to try + :type host_index: int + :param request: HTTP request. + :type request: arango.request.Request + :return: HTTP response. + :rtype: arango.response.Response + """ + tries = 0 + indexes_to_filter: Set[int] = set() + + data = self.normalize_data(request.data) + if ( + self._request_compression is not None + and isinstance(data, str) + and self._request_compression.needs_compression(data) + ): + request.headers["content-encoding"] = self._request_compression.encoding() + data = self._request_compression.compress(data) + + if self._response_compression is not None: + request.headers["accept-encoding"] = self._response_compression + + while tries < self._host_resolver.max_tries: + try: + resp = self._http.send_request( + session=self._sessions[host_index], + method=request.method, + url=self._url_prefixes[host_index] + request.endpoint, + params=request.params, + data=data, + headers=request.headers, + auth=auth, + ) + + return self.prep_response(resp, request.deserialize) + except ConnectionError: + url = self._url_prefixes[host_index] + request.endpoint + logging.debug(f"ConnectionError: {url}") + + if len(indexes_to_filter) == self._host_resolver.host_count - 1: + indexes_to_filter.clear() + indexes_to_filter.add(host_index) + + host_index = self._host_resolver.get_host_index(indexes_to_filter) + tries += 1 + + raise ConnectionAbortedError( + f"Can't connect to host(s) within limit ({self._host_resolver.max_tries})" + ) + def prep_bulk_err_response(self, parent_response: Response, body: Json) -> Response: """Build and return a bulk error response. @@ -158,9 +227,13 @@ def ping(self) -> int: request = Request(method="get", endpoint="/_api/collection") resp = self.send_request(request) if resp.status_code in {401, 403}: - raise ServerConnectionError("bad username and/or password") + raise ServerConnectionError( + resp, request, "bad username/password or token is expired" + ) if not resp.is_success: # pragma: no cover - raise ServerConnectionError(resp.error_message or "bad server response") + raise ServerConnectionError( + resp, request, resp.error_message or "bad server response" + ) return resp.status_code @abstractmethod @@ -192,6 +265,10 @@ class BasicConnection(BaseConnection): :type password: str :param http_client: User-defined HTTP client. :type http_client: arango.http.HTTPClient + :param: request_compression: The request compression algorithm. + :type request_compression: arango.http.RequestCompression | None + :param: response_compression: The response compression algorithm. + :type response_compression: str | None """ def __init__( @@ -205,6 +282,8 @@ def __init__( http_client: HTTPClient, serializer: Callable[..., str], deserializer: Callable[[str], Any], + request_compression: Optional[RequestCompression] = None, + response_compression: Optional[str] = None, ) -> None: super().__init__( hosts, @@ -214,6 +293,8 @@ def __init__( http_client, serializer, deserializer, + request_compression, + response_compression, ) self._username = username self._auth = (username, password) @@ -227,16 +308,7 @@ def send_request(self, request: Request) -> Response: :rtype: arango.response.Response """ host_index = self._host_resolver.get_host_index() - resp = self._http.send_request( - session=self._sessions[host_index], - method=request.method, - url=self._url_prefixes[host_index] + request.endpoint, - params=request.params, - data=self.normalize_data(request.data), - headers=request.headers, - auth=self._auth, - ) - return self.prep_response(resp, request.deserialize) + return self.process_request(host_index, request, auth=self._auth) class JwtConnection(BaseConnection): @@ -256,6 +328,10 @@ class JwtConnection(BaseConnection): :type password: str :param http_client: User-defined HTTP client. :type http_client: arango.http.HTTPClient + :param request_compression: The request compression algorithm. + :type request_compression: arango.http.RequestCompression | None + :param response_compression: The response compression algorithm. + :type response_compression: str | None """ def __init__( @@ -264,11 +340,14 @@ def __init__( host_resolver: HostResolver, sessions: Sequence[Session], db_name: str, - username: str, - password: str, http_client: HTTPClient, serializer: Callable[..., str], deserializer: Callable[[str], Any], + username: Optional[str] = None, + password: Optional[str] = None, + user_token: Optional[str] = None, + request_compression: Optional[RequestCompression] = None, + response_compression: Optional[str] = None, ) -> None: super().__init__( hosts, @@ -278,6 +357,8 @@ def __init__( http_client, serializer, deserializer, + request_compression, + response_compression, ) self._username = username self._password = password @@ -287,7 +368,13 @@ def __init__( self._token: Optional[str] = None self._token_exp: int = sys.maxsize - self.refresh_token() + if user_token is not None: + self.set_token(user_token) + elif username is not None and password is not None: + self.refresh_token() + else: + m = "Either **user_token** or **username** & **password** must be set" + raise ValueError(m) def send_request(self, request: Request) -> Response: """Send an HTTP request to ArangoDB server. @@ -302,15 +389,7 @@ def send_request(self, request: Request) -> Response: if self._auth_header is not None: request.headers["Authorization"] = self._auth_header - resp = self._http.send_request( - session=self._sessions[host_index], - method=request.method, - url=self._url_prefixes[host_index] + request.endpoint, - params=request.params, - data=self.normalize_data(request.data), - headers=request.headers, - ) - resp = self.prep_response(resp, request.deserialize) + resp = self.process_request(host_index, request) # Refresh the token and retry on HTTP 401 and error code 11. if resp.error_code != 11 or resp.status_code != 401: @@ -325,22 +404,19 @@ def send_request(self, request: Request) -> Response: if self._auth_header is not None: request.headers["Authorization"] = self._auth_header - resp = self._http.send_request( - session=self._sessions[host_index], - method=request.method, - url=self._url_prefixes[host_index] + request.endpoint, - params=request.params, - data=self.normalize_data(request.data), - headers=request.headers, - ) - return self.prep_response(resp, request.deserialize) + return self.process_request(host_index, request) def refresh_token(self) -> None: """Get a new JWT token for the current user (cannot be a superuser). :return: JWT token. :rtype: str + :raise arango.exceptions.JWTRefreshError: If missing username & password. + :raise arango.exceptions.JWTAuthError: If token retrieval fails. """ + if self._username is None or self._password is None: + raise JWTRefreshError("username and password must be set") + request = Request( method="post", endpoint="/_open/auth", @@ -349,32 +425,39 @@ def refresh_token(self) -> None: host_index = self._host_resolver.get_host_index() - resp = self._http.send_request( - session=self._sessions[host_index], - method=request.method, - url=self._url_prefixes[host_index] + request.endpoint, - data=self.normalize_data(request.data), - ) - resp = self.prep_response(resp) + resp = self.process_request(host_index, request) if not resp.is_success: raise JWTAuthError(resp, request) - self._token = resp.body["jwt"] - assert self._token is not None - - jwt_payload = jwt.decode( - self._token, - issuer="arangodb", - algorithms=["HS256"], - options={ - "require_exp": True, - "require_iat": True, - "verify_iat": True, - "verify_exp": True, - "verify_signature": False, - }, - ) + self.set_token(resp.body["jwt"]) + + def set_token(self, token: str) -> None: + """Set the JWT token. + + :param token: JWT token. + :type token: str + :raise arango.exceptions.JWTExpiredError: If the token is expired. + """ + assert token is not None + + try: + jwt_payload = jwt.decode( + token, + issuer="arangodb", + algorithms=["HS256"], + options={ + "require_exp": True, + "require_iat": True, + "verify_iat": True, + "verify_exp": True, + "verify_signature": False, + }, + ) + except ExpiredSignatureError: + raise JWTExpiredError("JWT token is expired") + + self._token = token self._token_exp = jwt_payload["exp"] self._auth_header = f"bearer {self._token}" @@ -394,6 +477,10 @@ class JwtSuperuserConnection(BaseConnection): :type http_client: arango.http.HTTPClient :param superuser_token: User generated token for superuser access. :type superuser_token: str + :param request_compression: The request compression algorithm. + :type request_compression: arango.http.RequestCompression | None + :param response_compression: The response compression algorithm. + :type response_compression: str | None """ def __init__( @@ -406,6 +493,8 @@ def __init__( serializer: Callable[..., str], deserializer: Callable[[str], Any], superuser_token: str, + request_compression: Optional[RequestCompression] = None, + response_compression: Optional[str] = None, ) -> None: super().__init__( hosts, @@ -415,6 +504,8 @@ def __init__( http_client, serializer, deserializer, + request_compression, + response_compression, ) self._auth_header = f"bearer {superuser_token}" @@ -429,12 +520,31 @@ def send_request(self, request: Request) -> Response: host_index = self._host_resolver.get_host_index() request.headers["Authorization"] = self._auth_header - resp = self._http.send_request( - session=self._sessions[host_index], - method=request.method, - url=self._url_prefixes[host_index] + request.endpoint, - params=request.params, - data=self.normalize_data(request.data), - headers=request.headers, - ) - return self.prep_response(resp, request.deserialize) + return self.process_request(host_index, request) + + def set_token(self, token: str) -> None: + """Set the JWT token. + + :param token: JWT token. + :type token: str + :raise arango.exceptions.JWTExpiredError: If the token is expired. + """ + assert token is not None + + try: + jwt.decode( + token, + issuer="arangodb", + algorithms=["HS256"], + options={ + "require_exp": True, + "require_iat": True, + "verify_iat": True, + "verify_exp": True, + "verify_signature": False, + }, + ) + except ExpiredSignatureError: + raise JWTExpiredError("JWT token is expired") + + self._auth_header = f"bearer {token}" diff --git a/arango/cursor.py b/arango/cursor.py index 00bf3e5e..7c9fabe2 100644 --- a/arango/cursor.py +++ b/arango/cursor.py @@ -27,6 +27,10 @@ class Cursor: :type init_data: dict :param cursor_type: Cursor type ("cursor" or "export"). :type cursor_type: str + :param allow_retry: If set to True, the cursor will always attempt to fetch + the latest batch from server even if the previous attempt failed. + This option is only available for server versions 3.11 and above. + :type allow_retry: bool """ __slots__ = [ @@ -36,10 +40,13 @@ class Cursor: "_count", "_cached", "_stats", + "_plan", "_profile", "_warnings", "_has_more", "_batch", + "_next_batch_id", + "_allow_retry", ] def __init__( @@ -47,16 +54,20 @@ def __init__( connection: BaseConnection, init_data: Json, cursor_type: str = "cursor", + allow_retry: bool = False, ) -> None: self._conn = connection self._type = cursor_type + self._allow_retry = allow_retry self._batch: Deque[Any] = deque() self._id = None self._count: Optional[int] = None self._cached = None self._stats = None + self._plan = None self._profile = None self._warnings = None + self._next_batch_id: Optional[str] = None self._update(init_data) def __iter__(self) -> "Cursor": @@ -99,6 +110,13 @@ def _update(self, data: Json) -> Json: self._cached = data["cached"] result["cached"] = data["cached"] + # New in 3.11 + if "nextBatchId" in data: + # This is only available for server versions 3.11 and above. + # Currently, we are testing against 3.10.9 + self._next_batch_id = data["nextBatchId"] # pragma: no cover + result["next_batch_id"] = data["nextBatchId"] # pragma: no cover + self._has_more = bool(data["hasMore"]) result["has_more"] = data["hasMore"] @@ -116,12 +134,18 @@ def _update(self, data: Json) -> Json: self._warnings = extra["warnings"] result["warnings"] = extra["warnings"] + if "plan" in extra: + self._plan = extra["plan"] + result["plan"] = extra["plan"] + if "stats" in extra: stats = extra["stats"] if "writesExecuted" in stats: stats["modified"] = stats.pop("writesExecuted") if "writesIgnored" in stats: stats["ignored"] = stats.pop("writesIgnored") + if "documentLookups" in stats: + stats["lookups"] = stats.pop("documentLookups") if "scannedFull" in stats: stats["scanned_full"] = stats.pop("scannedFull") if "scannedIndex" in stats: @@ -130,6 +154,22 @@ def _update(self, data: Json) -> Json: stats["execution_time"] = stats.pop("executionTime") if "httpRequests" in stats: stats["http_requests"] = stats.pop("httpRequests") + if "cursorsCreated" in stats: + stats["cursorsCreated"] = stats.pop("cursorsCreated") + if "cursorsRearmed" in stats: + stats["cursorsRearmed"] = stats.pop("cursorsRearmed") + if "cacheHits" in stats: + stats["cacheHits"] = stats.pop("cacheHits") + if "cacheMisses" in stats: + stats["cacheMisses"] = stats.pop("cacheMisses") + + # New in 3.11 + if "peakMemoryUsage" in stats: + stats["peak_memory_usage"] = stats.pop("peakMemoryUsage") + + if "intermediateCommits" in stats: + stats["intermediate_commits"] = stats.pop("intermediateCommits") + self._stats = stats result["statistics"] = stats @@ -210,6 +250,14 @@ def warnings(self) -> Optional[Sequence[Json]]: """ return self._warnings + def plan(self) -> Optional[Json]: + """Return query execution plan. + + :return: Query execution plan. + :rtype: dict + """ + return self._plan + def empty(self) -> bool: """Check if the current batch is empty. @@ -260,7 +308,12 @@ def fetch(self) -> Json: """ if self._id is None: raise CursorStateError("cursor ID not set") - request = Request(method="put", endpoint=f"/_api/{self._type}/{self._id}") + + endpoint = f"/_api/{self._type}/{self._id}" + if self._allow_retry and self._next_batch_id is not None: + endpoint += f"/{self._next_batch_id}" # pragma: no cover + + request = Request(method="post", endpoint=endpoint) resp = self._conn.send_request(request) if not resp.is_success: @@ -279,7 +332,6 @@ def close(self, ignore_missing: bool = False) -> Optional[bool]: smaller than the batch size). :rtype: bool | None :raise arango.exceptions.CursorCloseError: If operation fails. - :raise arango.exceptions.CursorStateError: If cursor ID is not set. """ if self._id is None: return None diff --git a/arango/database.py b/arango/database.py index 5b009fb7..17d7a124 100644 --- a/arango/database.py +++ b/arango/database.py @@ -1,8 +1,15 @@ -__all__ = ["StandardDatabase", "AsyncDatabase", "BatchDatabase", "TransactionDatabase"] +__all__ = [ + "StandardDatabase", + "AsyncDatabase", + "BatchDatabase", + "OverloadControlDatabase", + "TransactionDatabase", +] from datetime import datetime from numbers import Number -from typing import Any, List, Optional, Sequence, Union +from typing import Any, Dict, List, Optional, Sequence, Union +from warnings import warn from arango.api import ApiGroup from arango.aql import AQL @@ -20,10 +27,12 @@ CollectionCreateError, CollectionDeleteError, CollectionListError, + DatabaseCompactError, DatabaseCreateError, DatabaseDeleteError, DatabaseListError, DatabasePropertiesError, + DatabaseSupportInfoError, GraphCreateError, GraphDeleteError, GraphListError, @@ -33,19 +42,30 @@ PermissionListError, PermissionResetError, PermissionUpdateError, + ServerAvailableOptionsGetError, + ServerCurrentOptionsGetError, ServerDetailsError, ServerEchoError, ServerEncryptionError, ServerEngineError, + ServerExecuteError, + ServerLicenseGetError, + ServerLicenseSetError, ServerLogLevelError, + ServerLogLevelResetError, ServerLogLevelSetError, + ServerLogSettingError, + ServerLogSettingSetError, ServerMetricsError, + ServerModeError, + ServerModeSetError, ServerReadLogError, ServerReloadRoutingError, ServerRequiredDBVersionError, ServerRoleError, ServerRunTestsError, ServerShutdownError, + ServerShutdownProgressError, ServerStatisticsError, ServerStatusError, ServerTimeError, @@ -57,6 +77,7 @@ TaskGetError, TaskListError, TransactionExecuteError, + TransactionListError, UserCreateError, UserDeleteError, UserGetError, @@ -75,6 +96,7 @@ AsyncApiExecutor, BatchApiExecutor, DefaultApiExecutor, + OverloadControlApiExecutor, TransactionApiExecutor, ) from arango.formatter import ( @@ -212,6 +234,36 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def execute(self, command: str) -> Result[Any]: + """Execute raw Javascript command on the server. + + Executes the JavaScript code in the body on the server as + the body of a function with no arguments. If you have a + return statement then the return value you produce will be returned + as 'application/json'. + + NOTE: this method endpoint will only be usable if the server + was started with the option `--javascript.allow-admin-execute true`. + The default value of this option is false, which disables the execution + of user-defined code and disables this API endpoint entirely. + This is also the recommended setting for production. + + :param command: Javascript command to execute. + :type command: str + :return: Return value of **command**, if any. + :rtype: Any + :raise arango.exceptions.ServerExecuteError: If execution fails. + """ + request = Request(method="post", endpoint="/_admin/execute", data=command) + + def response_handler(resp: Response) -> Any: + if not resp.is_success: + raise ServerExecuteError(resp, request) + + return resp.body + + return self._execute(request, response_handler) + def execute_transaction( self, command: str, @@ -224,6 +276,7 @@ def execute_transaction( allow_implicit: Optional[bool] = None, intermediate_commit_count: Optional[int] = None, intermediate_commit_size: Optional[int] = None, + allow_dirty_read: bool = False, ) -> Result[Any]: """Execute raw Javascript command in transaction. @@ -256,6 +309,8 @@ def execute_transaction( :param intermediate_commit_size: Max size of operations in bytes after which an intermediate commit is performed automatically. :type intermediate_commit_size: int | None + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool | None :return: Return value of **command**. :rtype: Any :raise arango.exceptions.TransactionExecuteError: If execution fails. @@ -282,7 +337,12 @@ def execute_transaction( if intermediate_commit_size is not None: data["intermediateCommitSize"] = intermediate_commit_size - request = Request(method="post", endpoint="/_api/transaction", data=data) + request = Request( + method="post", + endpoint="/_api/transaction", + data=data, + headers={"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None, + ) def response_handler(resp: Response) -> Any: if not resp.is_success: @@ -292,21 +352,44 @@ def response_handler(resp: Response) -> Any: return self._execute(request, response_handler) - def version(self) -> Result[str]: - """Return ArangoDB server version. + def list_transactions(self) -> Result[Jsons]: + """Return the list of running stream transactions. + + :return: The list of transactions, with each transaction + containing an "id" and a "state" field. + :rtype: List[Dict[str, Any]] + :raise arango.exceptions.TransactionListError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_api/transaction") + + def response_handler(resp: Response) -> Jsons: + if not resp.is_success: + raise TransactionListError(resp, request) + result: Jsons = resp.body["transactions"] + return result + + return self._execute(request, response_handler) + + def version(self, details: bool = False) -> Result[Any]: + """Return ArangoDB server version. + :param details: Return more detailed version output + :type details: bool | None :return: Server version. :rtype: str :raise arango.exceptions.ServerVersionError: If retrieval fails. """ request = Request( - method="get", endpoint="/_api/version", params={"details": False} + method="get", endpoint="/_api/version", params={"details": details} ) - def response_handler(resp: Response) -> str: + def response_handler(resp: Response) -> Any: if not resp.is_success: raise ServerVersionError(resp, request) - return str(resp.body["version"]) + if not details: + return str(resp.body["version"]) + else: + return resp.body return self._execute(request, response_handler) @@ -329,6 +412,54 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def license(self) -> Result[Json]: + """View the license information and status of an + Enterprise Edition instance. Can be called on + single servers, Coordinators, and DB-Servers. + + :return: Server license. + :rtype: dict + :raise arango.exceptions.ServerLicenseGetError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_admin/license") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body + return result + raise ServerLicenseGetError(resp, request) + + return self._execute(request, response_handler) + + def set_license(self, license: str, force: bool = False) -> Result[Json]: + """Set a new license for an Enterprise Edition + instance. Can be called on single servers, Coordinators, + and DB-Servers. + + :param license: The Base64-encoded license string, wrapped in double-quotes. + :type license: str + :param force: If set to True, the new license will be set even if + it expires sooner than the current license. + :type force: bool + :return: Server license. + :rtype: dict + :raise arango.exceptions.ServerLicenseError: If retrieval fails. + """ + request = Request( + method="put", + endpoint="/_admin/license", + params={"force": force}, + data=license, + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body + return result + raise ServerLicenseSetError(resp, request) + + return self._execute(request, response_handler) + def status(self) -> Result[Json]: """Return ArangoDB server status. @@ -348,6 +479,47 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def compact( + self, + change_level: Optional[bool] = None, + compact_bottom_most_level: Optional[bool] = None, + ) -> Result[Json]: + """Compact all databases. + + NOTE: This command can cause a full rewrite of all data in all databases, + which may take very long for large databases. It should thus only be used with + care and only when additional I/O load can be tolerated for a prolonged time. + + This method can be used to reclaim disk space after substantial data deletions + have taken place, by compacting the entire database system data. + + This method requires superuser access. + + :param change_level: Whether or not compacted data should be moved to + the minimum possible level. Default value is False. + :type change_level: bool | None + :param compact_bottom_most_level: Whether or not to compact the + bottom-most level of data. Default value is False. + :type compact_bottom_most_level: bool | None + :return: Collection compact. + :rtype: dict + :raise arango.exceptions.CollectionCompactError: If retrieval fails. + """ + data = {} + if change_level is not None: + data["changeLevel"] = change_level + if compact_bottom_most_level is not None: + data["compactBottomMostLevel"] = compact_bottom_most_level + + request = Request(method="put", endpoint="/_admin/compact", data=data) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_body(resp.body) + raise DatabaseCompactError(resp, request) + + return self._execute(request, response_handler) + def required_db_version(self) -> Result[str]: """Return required version of target database. @@ -419,6 +591,56 @@ def response_handler(resp: Response) -> str: return self._execute(request, response_handler) + def mode(self) -> Result[str]: + """Return the server mode (default or read-only) + + In a read-only server, all write operations will fail + with an error code of 1004 (ERROR_READ_ONLY). Creating or dropping + databases and collections will also fail with error code 11 (ERROR_FORBIDDEN). + + :return: Server mode. Possible values are "default" or "readonly". + :rtype: str + :raise arango.exceptions.ServerModeError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_admin/server/mode") + + def response_handler(resp: Response) -> str: + if resp.is_success: + return str(resp.body["mode"]) + + raise ServerModeError(resp, request) + + return self._execute(request, response_handler) + + def set_mode(self, mode: str) -> Result[Json]: + """Set the server mode to read-only or default. + + Update mode information about a server. The JSON response will + contain a field mode with the value readonly or default. + In a read-only server all write operations will fail with an error + code of 1004 (ERROR_READ_ONLY). Creating or dropping of databases + and collections will also fail with error code 11 (ERROR_FORBIDDEN). + + This is a protected API. It requires authentication and administrative + server rights. + + :param mode: Server mode. Possible values are "default" or "readonly". + :type mode: str + :return: Server mode. + :rtype: str + :raise arango.exceptions.ServerModeSetError: If set fails. + """ + request = Request( + method="put", endpoint="/_admin/server/mode", data={"mode": mode} + ) + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_body(resp.body) + raise ServerModeSetError(resp, request) + + return self._execute(request, response_handler) + def time(self) -> Result[datetime]: """Return server system time. @@ -435,14 +657,23 @@ def response_handler(resp: Response) -> datetime: return self._execute(request, response_handler) - def echo(self) -> Result[Json]: - """Return details of the last request (e.g. headers, payload). + def echo(self, body: Optional[Any] = None) -> Result[Json]: + """Return details of the last request (e.g. headers, payload), + or echo the given request body. + :param body: The body of the request. Can be of any type + and is simply forwarded. If not set, the details of the last + request are returned. + :type body: dict | list | str | int | float | None :return: Details of the last request. :rtype: dict :raise arango.exceptions.ServerEchoError: If retrieval fails. """ - request = Request(method="get", endpoint="/_admin/echo") + request = ( + Request(method="get", endpoint="/_admin/echo") + if body is None + else Request(method="post", endpoint="/_admin/echo", data=body) + ) def response_handler(resp: Response) -> Json: if not resp.is_success: @@ -452,14 +683,21 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) - def shutdown(self) -> Result[bool]: # pragma: no cover + def shutdown(self, soft: bool = False) -> Result[bool]: # pragma: no cover """Initiate server shutdown sequence. + :param soft: If set to true, this initiates a soft shutdown. This is only + available on Coordinators. When issued, the Coordinator tracks a number + of ongoing operations, waits until all have finished, and then shuts + itself down normally. It will still accept new operations. + :type soft: bool :return: True if the server was shutdown successfully. :rtype: bool :raise arango.exceptions.ServerShutdownError: If shutdown fails. """ - request = Request(method="delete", endpoint="/_admin/shutdown") + request = Request( + method="delete", endpoint="/_admin/shutdown", params={"soft": soft} + ) def response_handler(resp: Response) -> bool: if not resp.is_success: @@ -468,6 +706,26 @@ def response_handler(resp: Response) -> bool: return self._execute(request, response_handler) + def shutdown_progress(self) -> Result[Json]: # pragma: no cover + """Query the soft shutdown progress. This call reports progress about a + soft Coordinator shutdown (DELETE /_admin/shutdown?soft=true). This API + is only available on Coordinators. + + :return: Information about the shutdown progress. + :rtype: dict + :raise arango.exceptions.ServerShutdownError: If shutdown fails. + """ + request = Request(method="get", endpoint="/_admin/shutdown") + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ServerShutdownProgressError(resp, request) + + result: Json = resp.body + return result + + return self._execute(request, response_handler) + def run_tests(self, tests: Sequence[str]) -> Result[Json]: # pragma: no cover """Run available unittests on the server. @@ -497,7 +755,10 @@ def read_log( search: Optional[str] = None, sort: Optional[str] = None, ) -> Result[Json]: - """Read the global log from server. + """Read the global log from server. This method is deprecated + in ArangoDB 3.8 and will be removed in a future version + of the driver. Use :func:`arango.database.Database.read_log_entries` + instead. :param upto: Return the log entries up to the given level (mutually exclusive with parameter **level**). Allowed values are "fatal", @@ -524,6 +785,9 @@ def read_log( :rtype: dict :raise arango.exceptions.ServerReadLogError: If read fails. """ + m = "read_log() is deprecated in ArangoDB 3.8 and will be removed in a future version of the driver. Use read_log_entries() instead." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + params = dict() if upto is not None: params["upto"] = upto @@ -553,13 +817,146 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) - def log_levels(self) -> Result[Json]: + def read_log_entries( + self, + upto: Optional[Union[int, str]] = None, + level: Optional[Union[int, str]] = None, + start: Optional[int] = None, + size: Optional[int] = None, + offset: Optional[int] = None, + search: Optional[str] = None, + sort: Optional[str] = None, + server_id: Optional[str] = None, + ) -> Result[Json]: + """Read the global log from server. + + :param upto: Return the log entries up to the given level (mutually + exclusive with parameter **level**). Allowed values are "fatal", + "error", "warning", "info" (default) and "debug". + :type upto: int | str + :param level: Return the log entries of only the given level (mutually + exclusive with **upto**). Allowed values are "fatal", "error", + "warning", "info" (default) and "debug". + :type level: int | str + :param start: Return the log entries whose ID is greater or equal to + the given value. + :type start: int + :param size: Restrict the size of the result to the given value. This + can be used for pagination. + :type size: int + :param offset: Number of entries to skip (e.g. for pagination). + :type offset: int + :param search: Return only the log entries containing the given text. + :type search: str + :param sort: Sort the log entries according to the given fashion, which + can be "sort" or "desc". + :type sort: str + :param server_id: Returns all log entries of the specified server. + All other query parameters remain valid. If no serverId is given, + the asked server will reply. This parameter is only meaningful + on Coordinators. + :type server_id: str + :return: Server log entries. + :rtype: dict + :raise arango.exceptions.ServerReadLogError: If read fails. + """ + params = dict() + if upto is not None: + params["upto"] = upto + if level is not None: + params["level"] = level + if start is not None: + params["start"] = start + if size is not None: + params["size"] = size + if offset is not None: + params["offset"] = offset + if search is not None: + params["search"] = search + if sort is not None: + params["sort"] = sort + if server_id is not None: + params["serverId"] = server_id + + request = Request(method="get", endpoint="/_admin/log/entries", params=params) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ServerReadLogError(resp, request) + + result: Json = resp.body + return result + + return self._execute(request, response_handler) + + def log_settings(self) -> Result[Json]: + """Return the structured log settings. + + :return: Current log settings. False values are not returned. + :rtype: dict + """ + request = Request(method="get", endpoint="/_admin/log/structured") + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ServerLogSettingError(resp, request) + result: Json = resp.body + return result + + return self._execute(request, response_handler) + + def set_log_settings(self, **kwargs: Dict[str, Any]) -> Result[Json]: + """Set the structured log settings. + + This method takes arbitrary keyword arguments where the keys are the + structured log parameters and the values are true or false, for either + enabling or disabling the parameters. + + .. code-block:: python + + arango.set_log_settings( + database=True, + url=True, + username=False, + ) + + :param kwargs: Structured log parameters. + :type kwargs: Dict[str, Any] + :return: New log settings. False values are not returned. + :rtype: dict + """ + request = Request(method="put", endpoint="/_admin/log/structured", data=kwargs) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ServerLogSettingSetError(resp, request) + result: Json = resp.body + return result + + return self._execute(request, response_handler) + + def log_levels( + self, server_id: Optional[str] = None, with_appenders: Optional[bool] = None + ) -> Result[Json]: """Return current logging levels. + :param server_id: Forward log level to a specific server. This makes it + easier to adjust the log levels in clusters because DB-Servers require + JWT authentication whereas Coordinators also support authentication + using usernames and passwords. + :type server_id: str + :param with_appenders: Include appenders in the response. + :type with_appenders: bool :return: Current logging levels. :rtype: dict """ - request = Request(method="get", endpoint="/_admin/log/level") + params: Params = {} + if server_id is not None: + params["serverId"] = server_id + if with_appenders is not None: + params["withAppenders"] = with_appenders + + request = Request(method="get", endpoint="/_admin/log/level", params=params) def response_handler(resp: Response) -> Json: if not resp.is_success: @@ -569,7 +966,12 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) - def set_log_levels(self, **kwargs: str) -> Result[Json]: + def set_log_levels( + self, + server_id: Optional[str] = None, + with_appenders: Optional[bool] = None, + **kwargs: Dict[str, Any], + ) -> Result[Json]: """Set the logging levels. This method takes arbitrary keyword arguments where the keys are the @@ -585,10 +987,27 @@ def set_log_levels(self, **kwargs: str) -> Result[Json]: Keys that are not valid logger names are ignored. + :param server_id: Forward log level to a specific server. This makes it + easier to adjust the log levels in clusters because DB-Servers require + JWT authentication whereas Coordinators also support authentication + using usernames and passwords. + :type server_id: str | None + :param with_appenders: Include appenders in the request. + :type with_appenders: bool | None + :param kwargs: Logging levels. + :type kwargs: Dict[str, Any] :return: New logging levels. :rtype: dict """ - request = Request(method="put", endpoint="/_admin/log/level", data=kwargs) + params: Params = {} + if server_id is not None: + params["serverId"] = server_id + if with_appenders is not None: + params["withAppenders"] = with_appenders + + request = Request( + method="put", endpoint="/_admin/log/level", params=params, data=kwargs + ) def response_handler(resp: Response) -> Json: if not resp.is_success: @@ -598,6 +1017,35 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def reset_log_levels(self, server_id: Optional[str] = None) -> Result[Json]: + """Reset the logging levels. + + Revert the server’s log level settings to the values they had at startup, + as determined by the startup options specified on the command-line, + a configuration file, and the factory defaults. + + :param server_id: Forward log level to a specific server. This makes it + easier to adjust the log levels in clusters because DB-Servers require + JWT authentication whereas Coordinators also support authentication + using usernames and passwords. + :type server_id: str | None + :return: New logging levels. + :rtype: dict + """ + params: Params = {} + if server_id is not None: + params["serverId"] = server_id + + request = Request(method="delete", endpoint="/_admin/log/level", params=params) + + def response_handler(resp: Response) -> Json: + if not resp.is_success: + raise ServerLogLevelResetError(resp, request) + result: Json = resp.body + return result + + return self._execute(request, response_handler) + def reload_routing(self) -> Result[bool]: """Reload the routing information. @@ -620,7 +1068,7 @@ def metrics(self) -> Result[str]: :return: Server metrics in Prometheus format. :rtype: str """ - request = Request(method="get", endpoint="/_admin/metrics") + request = Request(method="get", endpoint="/_admin/metrics/v2") def response_handler(resp: Response) -> str: if resp.is_success: @@ -715,12 +1163,64 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def options(self) -> Result[Json]: + """Return the currently-set server options (ArangoDB 3.12+) + + As this API may reveal sensitive data about the deployment, it can only + be accessed from inside the _system database. In addition, there is a + policy control startup option --server.options-api that determines if and + to whom the API is made available. This option can have the following + values: + - disabled: API is disabled. + - jwt: API can only be accessed via superuser JWT. + - admin: API can be accessed by admin users in the _system database only. + - public: everyone with access to _system database can access the API. + + :return: Server options. + :rtype: dict + """ + request = Request(method="get", endpoint="/_admin/options") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body + return result + raise ServerCurrentOptionsGetError(resp, request) + + return self._execute(request, response_handler) + + def options_available(self) -> Result[Json]: + """Return a description of all available server options (ArangoDB 3.12+) + + As this API may reveal sensitive data about the deployment, it can only + be accessed from inside the _system database. In addition, there is a + policy control startup option --server.options-api that determines if and + to whom the API is made available. This option can have the following + values: + - disabled: API is disabled. + - jwt: API can only be accessed via superuser JWT. + - admin: API can be accessed by admin users in the _system database only. + - public: everyone with access to _system database can access the options API. + + :return: Server options. + :rtype: dict + """ + request = Request(method="get", endpoint="/_admin/options-description") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body + return result + raise ServerAvailableOptionsGetError(resp, request) + + return self._execute(request, response_handler) + ####################### # Database Management # ####################### def databases(self) -> Result[List[str]]: - """Return the names all databases. + """Return the names of all databases. :return: Database names. :rtype: [str] @@ -736,6 +1236,23 @@ def response_handler(resp: Response) -> List[str]: return self._execute(request, response_handler) + def databases_accessible_to_user(self) -> Result[List[str]]: + """Return the names of all databases accessible by the user. + + :return: Database names accesible by the current user. + :rtype: List[str] + :raise arango.exceptions.DatabaseListError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_api/database/user") + + def response_handler(resp: Response) -> List[str]: + if not resp.is_success: + raise DatabaseListError(resp, request) + result: List[str] = resp.body["result"] + return result + + return self._execute(request, response_handler) + def has_database(self, name: str) -> Result[bool]: """Check if a database exists. @@ -767,8 +1284,9 @@ def create_database( :type name: str :param users: List of users with access to the new database, where each user is a dictionary with fields "username", "password", "active" - and "extra" (see below for example). If not set, only the admin and - current user are granted access. + and "extra" (see below for example). If not set, the default user root + will be used to ensure that the new database will be accessible after + it is created. :type users: [dict] :param replication_factor: Default replication factor for collections created in this database. Special values include "satellite" which @@ -933,6 +1451,7 @@ def create_collection( smart_join_attribute: Optional[str] = None, write_concern: Optional[int] = None, schema: Optional[Json] = None, + computedValues: Optional[Jsons] = None, ) -> Result[StandardCollection]: """Create a new collection. @@ -1010,15 +1529,22 @@ def create_collection( for documents. See ArangoDB documentation for more information on document schema validation. :type schema: dict + :param computedValues: Array of computed values for the new collection + enabling default values to new documents or the maintenance of + auxiliary attributes for search queries. Available in ArangoDB + version 3.10 or greater. See ArangoDB documentation for more + information on computed values. + :type computedValues: list :return: Standard collection API wrapper. :rtype: arango.collection.StandardCollection :raise arango.exceptions.CollectionCreateError: If create fails. """ key_options: Json = {"type": key_generator, "allowUserKeys": user_keys} - if key_increment is not None: - key_options["increment"] = key_increment - if key_offset is not None: - key_options["offset"] = key_offset + if key_generator == "autoincrement": + if key_increment is not None: + key_options["increment"] = key_increment + if key_offset is not None: + key_options["offset"] = key_offset data: Json = { "name": name, @@ -1043,6 +1569,8 @@ def create_collection( data["writeConcern"] = write_concern if schema is not None: data["schema"] = schema + if computedValues is not None: + data["computedValues"] = computedValues params: Params = {} if sync_replication is not None: @@ -1165,8 +1693,12 @@ def create_graph( edge_definitions: Optional[Sequence[Json]] = None, orphan_collections: Optional[Sequence[str]] = None, smart: Optional[bool] = None, + disjoint: Optional[bool] = None, smart_field: Optional[str] = None, shard_count: Optional[int] = None, + replication_factor: Optional[int] = None, + write_concern: Optional[int] = None, + satellite_collections: Optional[Sequence[str]] = None, ) -> Result[Graph]: """Create a new graph. @@ -1184,10 +1716,15 @@ def create_graph( **smart_field** below). Applies only to enterprise version of ArangoDB. :type smart: bool | None + :param disjoint: If set to True, create a disjoint SmartGraph instead + of a regular SmartGraph. Applies only to enterprise version of + ArangoDB. + :type disjoint: bool | None :param smart_field: Document field used to shard the vertices of the graph. To use this, parameter **smart** must be set to True and every vertex in the graph must have the smart field. Applies only - to enterprise version of ArangoDB. + to enterprise version of ArangoDB. NOTE: If this field is + None and **smart** is True, an Enterprise Graph will be created. :type smart_field: str | None :param shard_count: Number of shards used for every collection in the graph. To use this, parameter **smart** must be set to True and @@ -1195,6 +1732,27 @@ def create_graph( cannot be modified later once set. Applies only to enterprise version of ArangoDB. :type shard_count: int | None + :param replication_factor: Number of copies of each shard on different + servers in a cluster. Allowed values are 1 (only one copy is kept + and no synchronous replication), and n (n-1 replicas are kept and + any two copies are replicated across servers synchronously, meaning + every write to the master is copied to all slaves before operation + is reported successful). + :type replication_factor: int + :param write_concern: Write concern for the collection. Determines how + many copies of each shard are required to be in sync on different + DBServers. If there are less than these many copies in the cluster + a shard will refuse to write. Writes to shards with enough + up-to-date copies will succeed at the same time. The value of this + parameter cannot be larger than that of **replication_factor**. + Default value is 1. Used for clusters only. + :type write_concern: int + :param satellite_collections: An array of collection names that is + used to create SatelliteCollections for a (Disjoint) SmartGraph + using SatelliteCollections (Enterprise Edition only). Each array + element must be a string and a valid collection name. The + collection type cannot be modified later. + :type satellite_collections: [str] | None :return: Graph API wrapper. :rtype: arango.graph.Graph :raise arango.exceptions.GraphCreateError: If create fails. @@ -1203,11 +1761,13 @@ def create_graph( .. code-block:: python - { - 'edge_collection': 'teach', - 'from_vertex_collections': ['teachers'], - 'to_vertex_collections': ['lectures'] - } + [ + { + 'edge_collection': 'teach', + 'from_vertex_collections': ['teachers'], + 'to_vertex_collections': ['lectures'] + } + ] """ data: Json = {"name": name, "options": dict()} if edge_definitions is not None: @@ -1223,10 +1783,18 @@ def create_graph( data["orphanCollections"] = orphan_collections if smart is not None: # pragma: no cover data["isSmart"] = smart + if disjoint is not None: # pragma: no cover + data["isDisjoint"] = disjoint if smart_field is not None: # pragma: no cover data["options"]["smartGraphAttribute"] = smart_field if shard_count is not None: # pragma: no cover data["options"]["numberOfShards"] = shard_count + if replication_factor is not None: # pragma: no cover + data["options"]["replicationFactor"] = replication_factor + if write_concern is not None: # pragma: no cover + data["options"]["writeConcern"] = write_concern + if satellite_collections is not None: # pragma: no cover + data["options"]["satellites"] = satellite_collections request = Request(method="post", endpoint="/_api/gharial", data=data) @@ -2052,9 +2620,9 @@ def response_handler(resp: Response) -> Jsons: return self._execute(request, response_handler) def view(self, name: str) -> Result[Json]: - """Return view details. + """Return the properties of a View. - :return: View details. + :return: The View properties. :rtype: dict :raise arango.exceptions.ViewGetError: If retrieval fails. """ @@ -2067,6 +2635,22 @@ def response_handler(resp: Response) -> Json: return self._execute(request, response_handler) + def view_info(self, name: str) -> Result[Json]: + """Return the id, name and type of a View. + + :return: Some View information. + :rtype: dict + :raise arango.exceptions.ViewGetError: If retrieval fails. + """ + request = Request(method="get", endpoint=f"/_api/view/{name}") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_view(resp.body) + raise ViewGetError(resp, request) + + return self._execute(request, response_handler) + def create_view( self, name: str, view_type: str, properties: Optional[Json] = None ) -> Result[Json]: @@ -2074,7 +2658,7 @@ def create_view( :param name: View name. :type name: str - :param view_type: View type (e.g. "arangosearch"). + :param view_type: View type (e.g. "arangosearch" or "search-alias"). :type view_type: str :param properties: View properties. For more information see https://www.arangodb.com/docs/stable/http/views-arangosearch.html @@ -2385,6 +2969,38 @@ def response_handler(resp: Response) -> bool: return self._execute(request, response_handler) + ########### + # Support # + ########### + + def support_info(self) -> Result[Json]: + """Return information about the deployment. + + Retrieves deployment information for support purposes. + The endpoint returns data about the ArangoDB version used, + the host (operating system, server ID, CPU and storage capacity, + current utilization, a few metrics) and the other servers in the + deployment (in case of Active Failover or cluster deployments). + + NOTE: This method can only be accessed from inside the **_system** database. + The is a policy control startup option `--server.support-info-api` that controls + if and to whom the API is made available. + + :return: Deployment information. + :rtype: dict + :raise arango.exceptions.DatabaseSupportInfoError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_admin/support-info") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + result: Json = resp.body + return result + + raise DatabaseSupportInfoError(resp, request) + + return self._execute(request, response_handler) + class StandardDatabase(Database): """Standard database API wrapper.""" @@ -2408,18 +3024,46 @@ def begin_async_execution(self, return_result: bool = True) -> "AsyncDatabase": """ return AsyncDatabase(self._conn, return_result) - def begin_batch_execution(self, return_result: bool = True) -> "BatchDatabase": + def begin_batch_execution( + self, + return_result: bool = True, + max_workers: Optional[int] = 1, + ) -> "BatchDatabase": """Begin batch execution. + .. warning:: + + The batch request API is deprecated since ArangoDB 3.8.0. + This functionality should no longer be used. + To send multiple documents at once to an ArangoDB instance, + please use any of :class:`arango.collection.Collection` methods + that accept a list of documents as input. + See :func:`~arango.collection.Collection.insert_many`, + :func:`~arango.collection.Collection.update_many`, + :func:`~arango.collection.Collection.replace_many`, + :func:`~arango.collection.Collection.delete_many`. + :param return_result: If set to True, API executions return instances of :class:`arango.job.BatchJob` that are populated with results on commit. If set to False, API executions return None and no results are tracked client-side. :type return_result: bool + :param max_workers: Maximum number of workers to use for submitting + requests asynchronously. If None, the default value is the minimum + between `os.cpu_count()` and the number of requests. + :type max_workers: Optional[int] :return: Database API wrapper object specifically for batch execution. :rtype: arango.database.BatchDatabase """ - return BatchDatabase(self._conn, return_result) + return BatchDatabase(self._conn, return_result, max_workers) + + def fetch_transaction(self, transaction_id: str) -> "TransactionDatabase": + """Fetch an existing transaction. + + :param transaction_id: The ID of the existing transaction. + :type transaction_id: str + """ + return TransactionDatabase(connection=self._conn, transaction_id=transaction_id) def begin_transaction( self, @@ -2430,6 +3074,7 @@ def begin_transaction( allow_implicit: Optional[bool] = None, lock_timeout: Optional[int] = None, max_size: Optional[int] = None, + skip_fast_lock_round: Optional[bool] = None, ) -> "TransactionDatabase": """Begin a transaction. @@ -2453,6 +3098,9 @@ def begin_transaction( :type lock_timeout: int | None :param max_size: Max transaction size in bytes. :type max_size: int | None + :param skip_fast_lock_round: Whether to disable fast locking for write + operations. + :type skip_fast_lock_round: bool | None :return: Database API wrapper object specifically for transactions. :rtype: arango.database.TransactionDatabase """ @@ -2465,8 +3113,22 @@ def begin_transaction( allow_implicit=allow_implicit, lock_timeout=lock_timeout, max_size=max_size, + skip_fast_lock_round=skip_fast_lock_round, ) + def begin_controlled_execution( + self, max_queue_time_seconds: Optional[float] = None + ) -> "OverloadControlDatabase": + """Begin a controlled connection, with options to handle server-side overload. + + :param max_queue_time_seconds: Maximum time in seconds a request can be queued + on the server-side. If set to 0 or None, the server ignores this setting. + :type max_queue_time_seconds: Optional[float] + :return: Database API wrapper object specifically for queue bounded execution. + :rtype: arango.database.OverloadControlDatabase + """ + return OverloadControlDatabase(self._conn, max_queue_time_seconds) + class AsyncDatabase(Database): """Database API wrapper tailored specifically for async execution. @@ -2494,7 +3156,11 @@ def __repr__(self) -> str: class BatchDatabase(Database): """Database API wrapper tailored specifically for batch execution. - See :func:`arango.database.StandardDatabase.begin_batch_execution`. + .. note:: + + This class is not intended to be instantiated directly. + See + :func:`arango.database.StandardDatabase.begin_batch_execution`. :param connection: HTTP connection. :param return_result: If set to True, API executions return instances of @@ -2502,12 +3168,22 @@ class BatchDatabase(Database): If set to False, API executions return None and no results are tracked client-side. :type return_result: bool + :param max_workers: Use a thread pool of at most `max_workers`. + :type max_workers: Optional[int] """ - def __init__(self, connection: Connection, return_result: bool) -> None: + def __init__( + self, connection: Connection, return_result: bool, max_workers: Optional[int] + ) -> None: self._executor: BatchApiExecutor super().__init__( - connection=connection, executor=BatchApiExecutor(connection, return_result) + connection=connection, + executor=BatchApiExecutor(connection, return_result, max_workers), + ) + warn( + "The batch request API is deprecated since ArangoDB version 3.8.0.", + FutureWarning, + stacklevel=3, ) def __repr__(self) -> str: @@ -2571,6 +3247,11 @@ class TransactionDatabase(Database): :type lock_timeout: int | None :param max_size: Max transaction size in bytes. :type max_size: int | None + :param transaction_id: Initialize using an existing transaction instead of creating + a new transaction. + :type transaction_id: str | None + :param skip_fast_lock_round: Whether to disable fast locking for write operations. + :type skip_fast_lock_round: bool | None """ def __init__( @@ -2583,6 +3264,8 @@ def __init__( allow_implicit: Optional[bool] = None, lock_timeout: Optional[int] = None, max_size: Optional[int] = None, + transaction_id: Optional[str] = None, + skip_fast_lock_round: Optional[bool] = None, ) -> None: self._executor: TransactionApiExecutor super().__init__( @@ -2596,6 +3279,8 @@ def __init__( allow_implicit=allow_implicit, lock_timeout=lock_timeout, max_size=max_size, + transaction_id=transaction_id, + skip_fast_lock_round=skip_fast_lock_round, ), ) @@ -2637,3 +3322,55 @@ def abort_transaction(self) -> bool: :raise arango.exceptions.TransactionAbortError: If abort fails. """ return self._executor.abort() + + +class OverloadControlDatabase(Database): + """Database API wrapper tailored to gracefully handle server overload scenarios. + + See :func:`arango.database.StandardDatabase.begin_controlled_execution`. + + :param connection: HTTP connection. + :param max_queue_time_seconds: Maximum server-side queuing time in seconds. + If the server-side queuing time exceeds the client's specified limit, + the request will be rejected. + :type max_queue_time_seconds: Optional[float] + """ + + def __init__( + self, connection: Connection, max_queue_time_seconds: Optional[float] = None + ) -> None: + self._executor: OverloadControlApiExecutor + super().__init__( + connection=connection, + executor=OverloadControlApiExecutor(connection, max_queue_time_seconds), + ) + + def __repr__(self) -> str: # pragma: no cover + return f"" + + @property + def last_queue_time(self) -> float: + """Return the most recently recorded server-side queuing time in seconds. + + :return: Server-side queuing time in seconds. + :rtype: float + """ + return self._executor.queue_time_seconds + + @property + def max_queue_time(self) -> Optional[float]: + """Return the maximum server-side queuing time in seconds. + + :return: Maximum server-side queuing time in seconds. + :rtype: Optional[float] + """ + return self._executor.max_queue_time_seconds + + def adjust_max_queue_time(self, max_queue_time_seconds: Optional[float]) -> None: + """Adjust the maximum server-side queuing time in seconds. + + :param max_queue_time_seconds: New maximum server-side queuing time + in seconds. Setting it to None disables the limit. + :type max_queue_time_seconds: Optional[float] + """ + self._executor.max_queue_time_seconds = max_queue_time_seconds diff --git a/arango/errno.py b/arango/errno.py index 3c256eaa..1f48b216 100644 --- a/arango/errno.py +++ b/arango/errno.py @@ -1,1184 +1,1168 @@ ################## -# General Errors # +# General errors # ################## -# No error occurred. +# no error NO_ERROR = 0 -# General error occurred. +# failed FAILED = 1 -# Operating system error occurred. +# system error SYS_ERROR = 2 -# Out of memory. +# out of memory OUT_OF_MEMORY = 3 -# Internal error occurred. +# internal error INTERNAL = 4 -# Illegal number representation given. +# illegal number ILLEGAL_NUMBER = 5 -# Numeric overflow occurred. +# numeric overflow NUMERIC_OVERFLOW = 6 -# Unknown option supplied by user. +# illegal option ILLEGAL_OPTION = 7 -# Detected PID without living process. +# dead process identifier DEAD_PID = 8 -# Feature not implemented. +# not implemented NOT_IMPLEMENTED = 9 -# Bad parameter. +# bad parameter BAD_PARAMETER = 10 -# Missing permission. +# forbidden FORBIDDEN = 11 -# Out of memory (mmap). -OUT_OF_MEMORY_MMAP = 12 - -# Corrupt CSV line. +# csv is corrupt CORRUPTED_CSV = 13 -# File not found. +# file not found FILE_NOT_FOUND = 14 -# Cannot write to file. +# cannot write file CANNOT_WRITE_FILE = 15 -# Cannot overwrite file. +# cannot overwrite file CANNOT_OVERWRITE_FILE = 16 -# Type error occurred. +# type error TYPE_ERROR = 17 -# Timed out waiting for a lock. +# lock timeout LOCK_TIMEOUT = 18 -# Cannot create a directory. +# cannot create directory CANNOT_CREATE_DIRECTORY = 19 -# Cannot create a temporary file. +# cannot create temporary file CANNOT_CREATE_TEMP_FILE = 20 -# Request cancelled by user. +# canceled request REQUEST_CANCELED = 21 -# Raised for debugging. +# intentional debug error DEBUG = 22 -# Invalid IP address. +# IP address is invalid IP_ADDRESS_INVALID = 25 -# File exists already. +# file exists FILE_EXISTS = 27 -# Locked resource or operation. +# locked LOCKED = 28 -# Deadlock detected when accessing collections. +# deadlock detected DEADLOCK = 29 -# Call failed as server shutdown is in progress. +# shutdown in progress SHUTTING_DOWN = 30 -# Feature only for enterprise version of ArangoDB. +# only enterprise version ONLY_ENTERPRISE = 31 -# Resource usage exceeded maximum value. +# resource limit exceeded RESOURCE_LIMIT = 32 -# ICU operation failed. +# icu error: %s ICU_ERROR = 33 -# Cannot read a file. +# cannot read file CANNOT_READ_FILE = 34 -# Incompatible version of ArangoDB. +# incompatible server version INCOMPATIBLE_VERSION = 35 -# Requested resource disabled. +# disabled DISABLED = 36 +# malformed json +MALFORMED_JSON = 37 + +# startup ongoing +STARTING_UP = 38 + +# error during deserialization +DESERIALIZE = 39 + +# reached end of file +END_OF_FILE = 40 + ########################### -# HTTP Error Status Codes # +# HTTP error status codes # ########################### -# Bad HTTP parameter. +# bad parameter HTTP_BAD_PARAMETER = 400 -# User unauthorized. +# unauthorized HTTP_UNAUTHORIZED = 401 -# Operation forbidden. +# forbidden HTTP_FORBIDDEN = 403 -# Unknown URI. +# not found HTTP_NOT_FOUND = 404 -# HTTP method unknown. +# method not supported HTTP_METHOD_NOT_ALLOWED = 405 -# HTTP content type not supported. +# request not acceptable HTTP_NOT_ACCEPTABLE = 406 -# Precondition not met. +# request timeout +HTTP_REQUEST_TIMEOUT = 408 + +# conflict +HTTP_CONFLICT = 409 + +# content permanently deleted +HTTP_GONE = 410 + +# precondition failed HTTP_PRECONDITION_FAILED = 412 -# Internal server error occurred. +# enhance your calm +HTTP_ENHANCE_YOUR_CALM = 420 + +# internal server error HTTP_SERVER_ERROR = 500 -# Service temporarily unavailable. +# not implemented +HTTP_NOT_IMPLEMENTED = 501 + +# service unavailable HTTP_SERVICE_UNAVAILABLE = 503 -# Service contacted by ArangoDB did not respond in time. +# gateway timeout HTTP_GATEWAY_TIMEOUT = 504 ########################## -# HTTP Processing Errors # +# HTTP processing errors # ########################## -# Corrupted JSON string. +# invalid JSON object HTTP_CORRUPTED_JSON = 600 -# URL contains superfluous suffices. +# superfluous URL suffices HTTP_SUPERFLUOUS_SUFFICES = 601 #################################### -# Internal ArangoDB Storage Errors # +# Internal ArangoDB storage errors # #################################### -# Datafile in illegal state. +# illegal state ILLEGAL_STATE = 1000 -# User attempted to write to a sealed datafile. -DATAFILE_SEALED = 1002 - -# Read-only datafile or collection. +# read only READ_ONLY = 1004 -# Duplicate identifier detected. +# duplicate identifier DUPLICATE_IDENTIFIER = 1005 -# Datafile unreadable. -DATAFILE_UNREADABLE = 1006 - -# Datafile empty. -DATAFILE_EMPTY = 1007 - -# Error occurred during WAL log file recovery. -RECOVERY = 1008 - -# Required datafile statistics object not found. -DATAFILE_STATISTICS_NOT_FOUND = 1009 - #################################### -# External ArangoDB Storage Errors # +# External ArangoDB storage errors # #################################### -# Datafile corrupted. +# corrupted datafile CORRUPTED_DATAFILE = 1100 -# Parameter file corrupted or cannot be read. +# illegal or unreadable parameter file ILLEGAL_PARAMETER_FILE = 1101 -# Collection contains one or more corrupted datafiles. +# corrupted collection CORRUPTED_COLLECTION = 1102 -# System call mmap failed. -MMAP_FAILED = 1103 - -# Filesystem full. +# filesystem full FILESYSTEM_FULL = 1104 -# Cannot create journal. -NO_JOURNAL = 1105 - -# Datafile of the same name already exists. -DATAFILE_ALREADY_EXISTS = 1106 - -# Database directory locked by another process. +# database directory is locked DATADIR_LOCKED = 1107 -# Directory of the same name already exists. -COLLECTION_DIRECTORY_ALREADY_EXISTS = 1108 - -# System call msync failed. -MSYNC_FAILED = 1109 - -# Cannot lock the database directory on startup. -DATADIR_UNLOCKABLE = 1110 - -# Server waited too long for the datafile to be synced to disk. -SYNC_TIMEOUT = 1111 - ################################### -# General ArangoDB Storage Errors # +# General ArangoDB storage errors # ################################### -# Conflict detected while updating or deleting a document. +# conflict CONFLICT = 1200 -# Database directory invalid. -DATADIR_INVALID = 1201 - -# Unknown document identifier or handle. +# document not found DOCUMENT_NOT_FOUND = 1202 -# Collection with given identifier or name unknown. +# collection or view not found DATA_SOURCE_NOT_FOUND = 1203 -# Missing collection parameter. +# parameter 'collection' not found COLLECTION_PARAMETER_MISSING = 1204 -# Invalid document handle. +# illegal document identifier DOCUMENT_HANDLE_BAD = 1205 -# Maximal journal size too small. -MAXIMAL_SIZE_TOO_SMALL = 1206 - -# Duplicate name detected. +# duplicate name DUPLICATE_NAME = 1207 -# Illegal name detected. +# illegal name ILLEGAL_NAME = 1208 -# No suitable index for query. +# no suitable index known NO_INDEX = 1209 -# Unique constraint violation. +# unique constraint violated UNIQUE_CONSTRAINT_VIOLATED = 1210 -# Index with unknown identifier. +# index not found INDEX_NOT_FOUND = 1212 -# Cross-collection requested. +# cross collection request not allowed CROSS_COLLECTION_REQUEST = 1213 -# Index handle corrupted. +# illegal index identifier INDEX_HANDLE_BAD = 1214 -# Document too large to fit into any datafile. +# document too large DOCUMENT_TOO_LARGE = 1216 -# Collection must be unloaded. -COLLECTION_NOT_UNLOADED = 1217 - -# Invalid collection type. +# collection type invalid COLLECTION_TYPE_INVALID = 1218 -# Failed to parse an attribute name definition. +# parsing attribute name definition failed ATTRIBUTE_PARSER_FAILED = 1220 -# Corrupted document key. +# illegal document key DOCUMENT_KEY_BAD = 1221 -# User-defined document key supplied for collections with auto key generation. +# unexpected document key DOCUMENT_KEY_UNEXPECTED = 1222 -# Database directory not writable for current user. +# server database directory not writable DATADIR_NOT_WRITABLE = 1224 -# Key generator out of keys. +# out of keys OUT_OF_KEYS = 1225 -# Document key missing. +# missing document key DOCUMENT_KEY_MISSING = 1226 -# There was an attempt to create a document of invalid type. +# invalid document type DOCUMENT_TYPE_INVALID = 1227 -# Non-existing database accessed. +# database not found DATABASE_NOT_FOUND = 1228 -# Invalid database used. +# database name invalid DATABASE_NAME_INVALID = 1229 -# Operation requested in non-system database. +# operation only allowed in system database USE_SYSTEM_DATABASE = 1230 -# Invalid key generator. +# invalid key generator INVALID_KEY_GENERATOR = 1232 -# Undefined or invalid "_from" or "_to" values in an edge document. +# expecting both `_from` and `_to` attributes to be defined in the edge document and have the format `/` INVALID_EDGE_ATTRIBUTE = 1233 -# Cannot create index. +# index creation failed INDEX_CREATION_FAILED = 1235 -# Server is write-throttled and a write operation waited too long. -WRITE_THROTTLE_TIMEOUT = 1236 - -# Collection type mismatch. +# collection type mismatch COLLECTION_TYPE_MISMATCH = 1237 -# Collection accessed but not yet loaded. +# collection not loaded COLLECTION_NOT_LOADED = 1238 -# Document revision corrupt or missing. +# illegal document revision DOCUMENT_REV_BAD = 1239 -# Read cannot be completed by storage engine. +# incomplete read INCOMPLETE_READ = 1240 +# not supported by old legacy data format +OLD_ROCKSDB_FORMAT = 1241 + +# an index with legacy sorted keys has been found +INDEX_HAS_LEGACY_SORTED_KEYS = 1242 + ################################### -# Checked ArangoDB Storage Errors # +# Checked ArangoDB storage errors # ################################### -# Datafile full. -DATAFILE_FULL = 1300 - -# Server database directory empty. +# server database directory is empty EMPTY_DATADIR = 1301 -# Operation needs to be retried. +# operation should be tried again TRY_AGAIN = 1302 -# Storage engine busy. +# engine is busy BUSY = 1303 -# Datafile merge in progress and the operation cannot be completed. +# merge in progress MERGE_IN_PROGRESS = 1304 -# Storage engine encountered an I/O error. +# storage engine I/O error IO_ERROR = 1305 ############################### -# ArangoDB Replication Errors # +# ArangoDB replication errors # ############################### -# Replication applier received no (or incomplete) response from master. +# no response REPLICATION_NO_RESPONSE = 1400 -# Replication applier received an invalid response from master. +# invalid response REPLICATION_INVALID_RESPONSE = 1401 -# Replication applier received a server error from master. -REPLICATION_MASTER_ERROR = 1402 +# leader error +REPLICATION_LEADER_ERROR = 1402 -# Replication applier tried to connect to master with incompatible version. -REPLICATION_MASTER_INCOMPATIBLE = 1403 +# leader incompatible +REPLICATION_LEADER_INCOMPATIBLE = 1403 -# Replication applier connected to a different master than before. -REPLICATION_MASTER_CHANGE = 1404 +# leader change +REPLICATION_LEADER_CHANGE = 1404 -# Replication applier was asked to connect to itself for replication. +# loop detected REPLICATION_LOOP = 1405 -# Unexpected marker found in replication log stream. +# unexpected marker REPLICATION_UNEXPECTED_MARKER = 1406 -# Found invalid replication applier state file. +# invalid applier state REPLICATION_INVALID_APPLIER_STATE = 1407 -# Found unexpected transaction ID. +# invalid transaction REPLICATION_UNEXPECTED_TRANSACTION = 1408 -# Invalid replication applier configuration. +# shard synchronization attempt timeout exceeded +REPLICATION_SHARD_SYNC_ATTEMPT_TIMEOUT_EXCEEDED = 1409 + +# invalid replication applier configuration REPLICATION_INVALID_APPLIER_CONFIGURATION = 1410 -# Operation attempted while replication applier is running. +# cannot perform operation while applier is running REPLICATION_RUNNING = 1411 -# Replication applier stopped by user. +# replication stopped REPLICATION_APPLIER_STOPPED = 1412 -# Replication applier started without a known start tick value. +# no start tick REPLICATION_NO_START_TICK = 1413 -# Replication applier started without a known start tick value. +# start tick not present REPLICATION_START_TICK_NOT_PRESENT = 1414 -# Newborn follower submits a wrong checksum. +# wrong checksum REPLICATION_WRONG_CHECKSUM = 1416 -# Shard is not empty and follower tries a shortcut. +# shard not empty REPLICATION_SHARD_NONEMPTY = 1417 +# replicated log {} not found +REPLICATION_REPLICATED_LOG_NOT_FOUND = 1418 + +# not the log leader +REPLICATION_REPLICATED_LOG_NOT_THE_LEADER = 1419 + +# not a log follower +REPLICATION_REPLICATED_LOG_NOT_A_FOLLOWER = 1420 + +# follower rejected append entries request +REPLICATION_REPLICATED_LOG_APPEND_ENTRIES_REJECTED = 1421 + +# a resigned leader instance rejected a request +REPLICATION_REPLICATED_LOG_LEADER_RESIGNED = 1422 + +# a resigned follower instance rejected a request +REPLICATION_REPLICATED_LOG_FOLLOWER_RESIGNED = 1423 + +# the replicated log of the participant is gone +REPLICATION_REPLICATED_LOG_PARTICIPANT_GONE = 1424 + +# an invalid term was given +REPLICATION_REPLICATED_LOG_INVALID_TERM = 1425 + +# log participant unconfigured +REPLICATION_REPLICATED_LOG_UNCONFIGURED = 1426 + +# replicated state {id:} of type {type:} not found +REPLICATION_REPLICATED_STATE_NOT_FOUND = 1427 + +# replicated state {id:} of type {type:} is unavailable +REPLICATION_REPLICATED_STATE_NOT_AVAILABLE = 1428 + +# not enough replicas for the configured write-concern are present +REPLICATION_WRITE_CONCERN_NOT_FULFILLED = 1429 + +# operation aborted because a previous operation failed +REPLICATION_REPLICATED_LOG_SUBSEQUENT_FAULT = 1430 + +# replicated state type {type:} is unavailable +REPLICATION_REPLICATED_STATE_IMPLEMENTATION_NOT_FOUND = 1431 + +# error in the replicated WAL subsystem +REPLICATION_REPLICATED_WAL_ERROR = 1432 + +# replicated WAL {file:} has an invalid or missing file header +REPLICATION_REPLICATED_WAL_INVALID_FILE = 1433 + +# replicated WAL {file:} is corrupt +REPLICATION_REPLICATED_WAL_CORRUPT = 1434 + ########################### -# ArangoDB Cluster Errors # +# ArangoDB cluster errors # ########################### -# Raised on some occasions when one server gets a request from another. +# not a follower +CLUSTER_NOT_FOLLOWER = 1446 + +# follower transaction intermediate commit already performed +CLUSTER_FOLLOWER_TRANSACTION_COMMIT_PERFORMED = 1447 + +# creating collection failed due to precondition +CLUSTER_CREATE_COLLECTION_PRECONDITION_FAILED = 1448 + +# got a request from an unknown server CLUSTER_SERVER_UNKNOWN = 1449 -# Coordinator cannot create a collection as the collection ID already exists. -CLUSTER_COLLECTION_ID_EXISTS = 1453 +# too many shards +CLUSTER_TOO_MANY_SHARDS = 1450 -# Coordinator cannot create an entry for a new collection in Plan hierarchy. +# could not create collection in plan CLUSTER_COULD_NOT_CREATE_COLLECTION_IN_PLAN = 1454 -# Coordinator sees DBServer issues when creating shards for a new collection. +# could not create collection CLUSTER_COULD_NOT_CREATE_COLLECTION = 1456 -# Coordinator runs into a timeout for some cluster wide operation. +# timeout in cluster operation CLUSTER_TIMEOUT = 1457 -# Coordinator cannot remove an entry for a collection in Plan hierarchy. +# could not remove collection from plan CLUSTER_COULD_NOT_REMOVE_COLLECTION_IN_PLAN = 1458 -# Coordinator cannot remove an entry for a collection in Current hierarchy. -CLUSTER_COULD_NOT_REMOVE_COLLECTION_IN_CURRENT = 1459 - -# Coordinator cannot create an entry for a new database in the Plan hierarchy. +# could not create database in plan CLUSTER_COULD_NOT_CREATE_DATABASE_IN_PLAN = 1460 -# Coordinator sees DBServer issues when creating databases for a new cluster. +# could not create database CLUSTER_COULD_NOT_CREATE_DATABASE = 1461 -# Coordinator cannot remove an entry for a database in the Plan hierarchy. +# could not remove database from plan CLUSTER_COULD_NOT_REMOVE_DATABASE_IN_PLAN = 1462 -# Coordinator cannot remove an entry for a database in the Current hierarchy. +# could not remove database from current CLUSTER_COULD_NOT_REMOVE_DATABASE_IN_CURRENT = 1463 -# Coordinator cannot determine the shard responsible for a given document. +# no responsible shard found CLUSTER_SHARD_GONE = 1464 -# Coordinator loses HTTP connection to a DBServer while transferring data. +# cluster internal HTTP connection broken CLUSTER_CONNECTION_LOST = 1465 -# "_key" attribute specified in sharded collection which uses not only "_key" -# as sharding attribute. +# must not specify _key for this collection CLUSTER_MUST_NOT_SPECIFY_KEY = 1466 -# Coordinator gets conflicting results from different shards. +# got contradicting answers from different shards CLUSTER_GOT_CONTRADICTING_ANSWERS = 1467 -# Coordinator tries to find out the shard responsible for a partial document. +# not all sharding attributes given CLUSTER_NOT_ALL_SHARDING_ATTRIBUTES_GIVEN = 1468 -# Not allowed to update the value of a shard attribute. +# must not change the value of a shard key attribute CLUSTER_MUST_NOT_CHANGE_SHARDING_ATTRIBUTES = 1469 -# Operation not supported in sharded collection. +# unsupported operation or parameter for clusters CLUSTER_UNSUPPORTED = 1470 -# Operation is coordinator-only. +# this operation is only valid on a coordinator in a cluster CLUSTER_ONLY_ON_COORDINATOR = 1471 -# Coordinator or DBServer cannot read the Plan. +# error reading Plan in agency CLUSTER_READING_PLAN_AGENCY = 1472 -# Coordinator cannot truncate all shards of a cluster collection. -CLUSTER_COULD_NOT_TRUNCATE_COLLECTION = 1473 - -# Internal communication of the cluster for AQL produces an error. +# error in cluster internal communication for AQL CLUSTER_AQL_COMMUNICATION = 1474 -# Operation is DBServer-only. +# this operation is only valid on a DBserver in a cluster CLUSTER_ONLY_ON_DBSERVER = 1477 -# Cannot reach a required DBServer. +# A cluster backend which was required for the operation could not be reached CLUSTER_BACKEND_UNAVAILABLE = 1478 -# Required collection out of sync during AQL execution. +# collection/view is out of sync CLUSTER_AQL_COLLECTION_OUT_OF_SYNC = 1481 -# Coordinator cannot create an entry for a new index in Plan hierarchy. +# could not create index in plan CLUSTER_COULD_NOT_CREATE_INDEX_IN_PLAN = 1482 -# Coordinator cannot remove an index from Plan hierarchy. +# could not drop index in plan CLUSTER_COULD_NOT_DROP_INDEX_IN_PLAN = 1483 -# One tries to create a collection with "shards_like" attribute which points -# to another collection that also has one. +# chain of distributeShardsLike references CLUSTER_CHAIN_OF_DISTRIBUTESHARDSLIKE = 1484 -# One tries to drop a collection to which another collection points with its -# "shard_like" attribute. +# must not drop collection while another has a distributeShardsLike attribute pointing to it CLUSTER_MUST_NOT_DROP_COLL_OTHER_DISTRIBUTESHARDSLIKE = 1485 -# One tries to create a collection which points to an unknown collection in its -# "shard_like" attribute. +# must not have a distributeShardsLike attribute pointing to an unknown collection CLUSTER_UNKNOWN_DISTRIBUTESHARDSLIKE = 1486 -# One tries to create a collection with a "replication_factor" greater than the -# available number of DBServers. +# the number of current DB-Servers is lower than the requested replicationFactor/writeConcern CLUSTER_INSUFFICIENT_DBSERVERS = 1487 -# Cannot drop follower. +# a follower could not be dropped in agency CLUSTER_COULD_NOT_DROP_FOLLOWER = 1488 -# Replication operation refused by a shard leader. +# a shard leader refuses to perform a replication operation CLUSTER_SHARD_LEADER_REFUSES_REPLICATION = 1489 -# Non-replication operation refused by a shard follower. +# a shard follower refuses to perform an operation CLUSTER_SHARD_FOLLOWER_REFUSES_OPERATION = 1490 -# Shard leader resigned in the meantime. +# a (former) shard leader refuses to perform an operation CLUSTER_SHARD_LEADER_RESIGNED = 1491 -# Agency operation failed after various retries. +# some agency operation failed CLUSTER_AGENCY_COMMUNICATION_FAILED = 1492 -# Servers currently competing for leadership. +# leadership challenge is ongoing CLUSTER_LEADERSHIP_CHALLENGE_ONGOING = 1495 -# Operation sent to a non-leading server. +# not a leader CLUSTER_NOT_LEADER = 1496 -# Coordinator cannot create an entry for a new view in Plan hierarchy. +# could not create view in plan CLUSTER_COULD_NOT_CREATE_VIEW_IN_PLAN = 1497 -# Coordinator tries to create a view and the ID already exists. +# view ID already exists CLUSTER_VIEW_ID_EXISTS = 1498 -# Coordinator cannot drop a collection entry in Plan hierarchy. +# could not drop collection in plan CLUSTER_COULD_NOT_DROP_COLLECTION = 1499 ######################### -# ArangoDB Query Errors # +# ArangoDB query errors # ######################### -# Running query killed by an explicit admin command. +# query killed QUERY_KILLED = 1500 -# Parsed query syntactically invalid. +# %s QUERY_PARSE = 1501 -# Empty query specified. +# query is empty QUERY_EMPTY = 1502 -# Runtime error caused by query. +# runtime error '%s' QUERY_SCRIPT = 1503 -# Number out of range. +# number out of range QUERY_NUMBER_OUT_OF_RANGE = 1504 -# Geo index coordinate invalid or out of range. +# invalid geo coordinate value QUERY_INVALID_GEO_VALUE = 1505 -# Invalid variable name. +# variable name '%s' has an invalid format QUERY_VARIABLE_NAME_INVALID = 1510 -# Variable redeclared in a query. +# variable '%s' is assigned multiple times QUERY_VARIABLE_REDECLARED = 1511 -# Variable name unknown or undefined. +# unknown variable '%s' QUERY_VARIABLE_NAME_UNKNOWN = 1512 -# Cannot acquire lock on collection. +# unable to read-lock collection %s QUERY_COLLECTION_LOCK_FAILED = 1521 -# Too many collections or shards in a query. +# too many collections/shards QUERY_TOO_MANY_COLLECTIONS = 1522 -# Document attribute redeclared. -QUERY_DOCUMENT_ATTRIBUTE_REDECLARED = 1530 +# too much nesting or too many objects +QUERY_TOO_MUCH_NESTING = 1524 -# Undefined function called. +# unknown/invalid OPTIONS attribute used +QUERY_INVALID_OPTIONS_ATTRIBUTE = 1539 + +# usage of unknown function '%s()' QUERY_FUNCTION_NAME_UNKNOWN = 1540 -# Argument number mismatch. +# invalid number of arguments for function '%s()' QUERY_FUNCTION_ARGUMENT_NUMBER_MISMATCH = 1541 -# Argument type mismatch. +# invalid argument type in call to function '%s()' QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH = 1542 -# Invalid regex. +# invalid regex value QUERY_INVALID_REGEX = 1543 -# Invalid bind parameters. +# invalid structure of bind parameters QUERY_BIND_PARAMETERS_INVALID = 1550 -# Bind parameter missing. +# no value specified for declared bind parameter '%s' QUERY_BIND_PARAMETER_MISSING = 1551 -# Bind parameter undeclared. +# bind parameter '%s' was not declared in the query QUERY_BIND_PARAMETER_UNDECLARED = 1552 -# Invalid bind parameter value or type. +# bind parameter '%s' has an invalid value or type QUERY_BIND_PARAMETER_TYPE = 1553 -# Non-boolean value used in logical operation. -QUERY_INVALID_LOGICAL_VALUE = 1560 +# failed vector search +QUERY_VECTOR_SEARCH_NOT_APPLIED = 1554 -# Non-numeric value used in arithmetic operation. +# invalid arithmetic value QUERY_INVALID_ARITHMETIC_VALUE = 1561 -# Divide by zero. +# division by zero QUERY_DIVISION_BY_ZERO = 1562 -# Non-list operand used when expecting an list operand. +# array expected QUERY_ARRAY_EXPECTED = 1563 -# Function "FAIL()" called inside a query. +# collection '%s' used as expression operand +QUERY_COLLECTION_USED_IN_EXPRESSION = 1568 + +# FAIL(%s) called QUERY_FAIL_CALLED = 1569 -# Geo restriction specified but no suitable geo index found. +# no suitable geo index found for geo restriction on '%s' QUERY_GEO_INDEX_MISSING = 1570 -# Fulltext query performed on a collection without suitable fulltext index. +# no suitable fulltext index found for fulltext query on '%s' QUERY_FULLTEXT_INDEX_MISSING = 1571 -# Cannot convert value to a date. +# invalid date value QUERY_INVALID_DATE_VALUE = 1572 -# Query contains more than one data-modifying operation. +# multi-modify query QUERY_MULTI_MODIFY = 1573 -# Query contains an invalid aggregate expression. +# invalid aggregate expression QUERY_INVALID_AGGREGATE_EXPRESSION = 1574 -# Query contains options that cannot be resolved at query compile time. +# query options must be readable at query compile time QUERY_COMPILE_TIME_OPTIONS = 1575 -# Query contains an invalid options specification. -QUERY_EXCEPTION_OPTIONS = 1576 +# FILTER/PRUNE condition complexity is too high +QUERY_DNF_COMPLEXITY = 1576 -# Unusable index hint. +# could not use forced index hint QUERY_FORCED_INDEX_HINT_UNUSABLE = 1577 -# Dynamic function not allowed. +# disallowed dynamic call to '%s' QUERY_DISALLOWED_DYNAMIC_CALL = 1578 -# Collection data accessed after modification. +# access after data-modification by %s QUERY_ACCESS_AFTER_MODIFICATION = 1579 ############################ -# AQL User Function Errors # +# AQL user function errors # ############################ -# User function registered with invalid name. +# invalid user function name QUERY_FUNCTION_INVALID_NAME = 1580 -# User function registered with invalid code. +# invalid user function code QUERY_FUNCTION_INVALID_CODE = 1581 -# User function not found. +# user function '%s()' not found QUERY_FUNCTION_NOT_FOUND = 1582 -# Runtime exception raised by query function. +# user function runtime error: %s QUERY_FUNCTION_RUNTIME_ERROR = 1583 +# query is not eligible for plan caching +QUERY_NOT_ELIGIBLE_FOR_PLAN_CACHING = 1584 + ############################# -# AQL Query Registry Errors # +# AQL query registry errors # ############################# -# Query received an invalid JSON. +# bad execution plan JSON QUERY_BAD_JSON_PLAN = 1590 -# Query ID not found. +# query ID not found QUERY_NOT_FOUND = 1591 -# User provided expression does not evaluate to true. +# %s QUERY_USER_ASSERT = 1593 -# User provided expression does not evaluate to true. +# %s QUERY_USER_WARN = 1594 +# window operation after data-modification +QUERY_WINDOW_AFTER_MODIFICATION = 1595 + ########################## -# ArangoDB Cursor Errors # +# ArangoDB cursor errors # ########################## -# Cursor ID not found. +# cursor not found CURSOR_NOT_FOUND = 1600 -# Concurrent request still using the cursor. +# cursor is busy CURSOR_BUSY = 1601 -############################## -# ArangoDB Validation Errors # -############################## +##################################### +# ArangoDB schema validation errors # +##################################### -# Document does not pass schema validation. +# schema validation failed VALIDATION_FAILED = 1620 -# Schema description is invalid. +# invalid schema validation parameter VALIDATION_BAD_PARAMETER = 1621 ############################### -# ArangoDB Transaction Errors # +# ArangoDB transaction errors # ############################### -# Wrong usage of transactions. This is an internal error. +# internal transaction error TRANSACTION_INTERNAL = 1650 -# Nested transactions. +# nested transactions detected TRANSACTION_NESTED = 1651 -# Unregistered collection used in transaction. +# unregistered collection used in transaction TRANSACTION_UNREGISTERED_COLLECTION = 1652 -# Disallowed operation in transaction. +# disallowed operation inside transaction TRANSACTION_DISALLOWED_OPERATION = 1653 -# Transaction aborted. +# transaction aborted TRANSACTION_ABORTED = 1654 -# Transaction not found. +# transaction not found TRANSACTION_NOT_FOUND = 1655 ########################## -# User Management Errors # +# User management errors # ########################## -# Invalid username. +# invalid user name USER_INVALID_NAME = 1700 -# Username already exists. +# duplicate user USER_DUPLICATE = 1702 -# User not found. +# user not found USER_NOT_FOUND = 1703 -# User authenticated by an external server. +# user is external USER_EXTERNAL = 1705 ###################################### -# Service Management Errors (Legacy) # +# Service management errors (legacy) # ###################################### -# Cannot download service from central repository. +# service download failed SERVICE_DOWNLOAD_FAILED = 1752 -# Service upload from the client to the ArangoDB server failed. +# service upload failed SERVICE_UPLOAD_FAILED = 1753 ############### -# LDAP Errors # +# Task errors # ############### -# Cannot initialize an LDAP connection. -LDAP_CANNOT_INIT = 1800 - -# Cannot set an LDAP option. -LDAP_CANNOT_SET_OPTION = 1801 - -# Cannot bind to an LDAP server. -LDAP_CANNOT_BIND = 1802 - -# Cannot unbind from an LDAP server. -LDAP_CANNOT_UNBIND = 1803 - -# Cannot search the LDAP server. -LDAP_CANNOT_SEARCH = 1804 - -# Cannot start a TLS LDAP session. -LDAP_CANNOT_START_TLS = 1805 - -# LDAP did not find any objects with the specified search query. -LDAP_FOUND_NO_OBJECTS = 1806 - -# LDAP found zero or more than one user. -LDAP_NOT_ONE_USER_FOUND = 1807 - -# LDAP user not identified. -LDAP_USER_NOT_IDENTIFIED = 1808 - -# Cannot distinguish a valid mode for provided LDAP configuration. -LDAP_INVALID_MODE = 1820 - -############### -# Task Errors # -############### - -# Task created with an invalid ID. +# invalid task id TASK_INVALID_ID = 1850 -# Task created with a duplicate ID. +# duplicate task id TASK_DUPLICATE_ID = 1851 -# Task not found. +# task not found TASK_NOT_FOUND = 1852 ############################ -# Graph / Traversal Errors # +# Graph / traversal errors # ############################ -# Invalid name passed to the server. +# invalid graph GRAPH_INVALID_GRAPH = 1901 -# Invalid graph name passed to the server. -GRAPH_COULD_NOT_CREATE_GRAPH = 1902 - -# Invalid vertex ID passed to the server. -GRAPH_INVALID_VERTEX = 1903 - -# Vertex could not be created. -GRAPH_COULD_NOT_CREATE_VERTEX = 1904 - -# Vertex could not be changed. -GRAPH_COULD_NOT_CHANGE_VERTEX = 1905 - -# Invalid edge ID passed to the server. +# invalid edge GRAPH_INVALID_EDGE = 1906 -# Edge could not be created. -GRAPH_COULD_NOT_CREATE_EDGE = 1907 - -# Edge could not be changed. -GRAPH_COULD_NOT_CHANGE_EDGE = 1908 - -# Too many iterations in graph traversal. -GRAPH_TOO_MANY_ITERATIONS = 1909 - -# Invalid filter result returned in graph traversal. +# invalid filter result GRAPH_INVALID_FILTER_RESULT = 1910 -# Edge collection may only be used once in a edge definition. +# multi use of edge collection in edge def GRAPH_COLLECTION_MULTI_USE = 1920 -# Collection already used by another graph in a different edge definition. +# edge collection already used in edge def GRAPH_COLLECTION_USE_IN_MULTI_GRAPHS = 1921 -# Graph name missing. +# missing graph name GRAPH_CREATE_MISSING_NAME = 1922 -# Edge definition malformed (must be a list of dicts). +# malformed edge definition GRAPH_CREATE_MALFORMED_EDGE_DEFINITION = 1923 -# Graph not found. +# graph '%s' not found GRAPH_NOT_FOUND = 1924 -# Graph name already exists. +# graph already exists GRAPH_DUPLICATE = 1925 -# Vertex collection does not exist or is not part of the graph. +# vertex collection does not exist or is not part of the graph GRAPH_VERTEX_COL_DOES_NOT_EXIST = 1926 -# Collection not a vertex collection. +# collection not a vertex collection GRAPH_WRONG_COLLECTION_TYPE_VERTEX = 1927 -# Vertex collection not in orphan collections of the graph. +# collection is not in list of orphan collections GRAPH_NOT_IN_ORPHAN_COLLECTION = 1928 -# Collection already used in an edge definition of the graph. +# collection already used in edge def GRAPH_COLLECTION_USED_IN_EDGE_DEF = 1929 -# Edge collection not used in any edge definition of the graph. +# edge collection not used in graph GRAPH_EDGE_COLLECTION_NOT_USED = 1930 -# Collection "_graphs" does not exist. +# collection _graphs does not exist GRAPH_NO_GRAPH_COLLECTION = 1932 -# Invalid example array object string. -GRAPH_INVALID_EXAMPLE_ARRAY_OBJECT_STRING = 1933 - -# Invalid example type (must be a list or dict). -GRAPH_INVALID_EXAMPLE_ARRAY_OBJECT = 1934 - -# Invalid number of arguments. +# Invalid number of arguments. Expected: GRAPH_INVALID_NUMBER_OF_ARGUMENTS = 1935 # Invalid parameter type. GRAPH_INVALID_PARAMETER = 1936 -# Invalid ID. -GRAPH_INVALID_ID = 1937 - -# Collection already in orphans of the graph. +# collection used in orphans GRAPH_COLLECTION_USED_IN_ORPHANS = 1938 -# Edge collection does not exist or is not part of the graph. +# edge collection does not exist or is not part of the graph GRAPH_EDGE_COL_DOES_NOT_EXIST = 1939 -# Graph has no edge collections. +# empty graph GRAPH_EMPTY = 1940 -# Invalid data in "_graphs" collection. +# internal graph data corrupt GRAPH_INTERNAL_DATA_CORRUPT = 1941 -# Edge collection already defined. -GRAPH_INTERNAL_EDGE_COLLECTION_ALREADY_SET = 1942 +# must not drop collection while part of graph +GRAPH_MUST_NOT_DROP_COLLECTION = 1942 -# Orphan list argument malformed. Must be a list of strings. +# malformed orphan list GRAPH_CREATE_MALFORMED_ORPHAN_LIST = 1943 -# Collection used as a relation exists. +# edge definition collection is a document collection GRAPH_EDGE_DEFINITION_IS_DOCUMENT = 1944 -# Invalid/unknown session ID passed to the server. +# initial collection is not allowed to be removed manually +GRAPH_COLLECTION_IS_INITIAL = 1945 + +# no valid initial collection found +GRAPH_NO_INITIAL_COLLECTION = 1946 + +# referenced vertex collection is not part of the graph +GRAPH_REFERENCED_VERTEX_COLLECTION_NOT_PART_OF_THE_GRAPH = 1947 + +# negative edge weight found +GRAPH_NEGATIVE_EDGE_WEIGHT = 1948 + +# the given collection is not part of the graph +GRAPH_COLLECTION_NOT_PART_OF_THE_GRAPH = 1949 + +################## +# Session errors # +################## + +# unknown session SESSION_UNKNOWN = 1950 -# Session expired. +# session expired SESSION_EXPIRED = 1951 ######################## -# Simple Client Errors # +# Simple Client errors # ######################## -# This error should not happen. +# unknown client error SIMPLE_CLIENT_UNKNOWN_ERROR = 2000 -# Client could not connect to server. +# could not connect to server SIMPLE_CLIENT_COULD_NOT_CONNECT = 2001 -# Client could not write data. +# could not write to server SIMPLE_CLIENT_COULD_NOT_WRITE = 2002 -# Client could not read data. +# could not read from server SIMPLE_CLIENT_COULD_NOT_READ = 2003 -# Will be raised if was erlaube?! +# was erlaube?! WAS_ERLAUBE = 2019 ####################### -# Communicator Errors # -####################### - -# Communicator request aborted. -COMMUNICATOR_REQUEST_ABORTED = 2100 - -# Communicator disabled. -COMMUNICATOR_DISABLED = 2101 - -####################### -# Internal AQL errors # +# internal AQL errors # ####################### -# Internal error during AQL execution. +# General internal AQL error INTERNAL_AQL = 2200 -# AQL block wrote in too few output registers. -WROTE_TOO_FEW_OUTPUT_REGISTERS = 2201 - -# AQL block wrote in too many output registers. -WROTE_TOO_MANY_OUTPUT_REGISTERS = 2202 - -# AQL block wrote in an output register twice. -WROTE_OUTPUT_REGISTER_TWICE = 2203 - -# AQL block wrote in a register that is not its output. -WROTE_IN_WRONG_REGISTER = 2204 - -# AQL block did not copy its input registers. -INPUT_REGISTERS_NOT_COPIED = 2205 - ########################## -# Foxx Management Errors # +# Foxx management errors # ########################## -# Service manifest file not a well-formed JSON. +# failed to parse manifest file MALFORMED_MANIFEST_FILE = 3000 -# Service manifest contains invalid values. +# manifest file is invalid INVALID_SERVICE_MANIFEST = 3001 -# Service folder or bundle does not exist on the server. +# service files missing SERVICE_FILES_MISSING = 3002 -# Local service bundle does not match the checksum in the database. +# service files outdated SERVICE_FILES_OUTDATED = 3003 -# Service options contain invalid values. +# service options are invalid INVALID_FOXX_OPTIONS = 3004 -# Service mountpath contains invalid characters. +# invalid mountpath INVALID_MOUNTPOINT = 3007 -# No service found at given mountpath. +# service not found SERVICE_NOT_FOUND = 3009 -# Service missing configuration or dependencies. +# service needs configuration SERVICE_NEEDS_CONFIGURATION = 3010 -# Service already exists at given mountpath. +# service already exists SERVICE_MOUNTPOINT_CONFLICT = 3011 -# Service directory does not contain a manifest file. +# missing manifest file SERVICE_MANIFEST_NOT_FOUND = 3012 -# Service options are not well-formed JSONs. +# failed to parse service options SERVICE_OPTIONS_MALFORMED = 3013 -# Source path does not match a file or directory. +# source path not found SERVICE_SOURCE_NOT_FOUND = 3014 -# Source path could not be resolved. +# error resolving source SERVICE_SOURCE_ERROR = 3015 -# Unknown service script. +# unknown script SERVICE_UNKNOWN_SCRIPT = 3016 -# API for managing Foxx services disabled. +# service api disabled SERVICE_API_DISABLED = 3099 ################################### -# JavaScript Module Loader Errors # +# JavaScript module loader errors # ################################### -# Cannot resolve module path. +# cannot locate module MODULE_NOT_FOUND = 3100 -# Module could not be parsed because of a syntax error. +# syntax error in module MODULE_SYNTAX_ERROR = 3101 -# Failed to invoke the module in its context. +# failed to invoke module MODULE_FAILURE = 3103 -##################### -# Enterprise Errors # -##################### +############################# +# Enterprise Edition errors # +############################# -# Requested collection needs to be smart. +# collection is not smart NO_SMART_COLLECTION = 4000 -# Given document does not have the smart graph attribute set. +# smart graph attribute not given NO_SMART_GRAPH_ATTRIBUTE = 4001 -# Smart collection cannot be dropped. +# cannot drop this smart collection CANNOT_DROP_SMART_COLLECTION = 4002 -# "_key" not prefixed with the value of the smart graph attribute. +# in smart vertex collections _key must be a string and prefixed with the value of the smart graph attribute KEY_MUST_BE_PREFIXED_WITH_SMART_GRAPH_ATTRIBUTE = 4003 -# Given smart graph attribute is illegal and cannot be used for sharding. +# attribute cannot be used as smart graph attribute ILLEGAL_SMART_GRAPH_ATTRIBUTE = 4004 -# Smart graph attribute of collection does not match the attribute of graph. +# smart graph attribute mismatch SMART_GRAPH_ATTRIBUTE_MISMATCH = 4005 -# Invalid smart join attribute declaration. +# invalid smart join attribute declaration INVALID_SMART_JOIN_ATTRIBUTE = 4006 -# Key must be prefixed with smart join attribute. +# shard key value must be prefixed with the value of the smart join attribute KEY_MUST_BE_PREFIXED_WITH_SMART_JOIN_ATTRIBUTE = 4007 -# Document lacks required smart join attribute. +# smart join attribute not given or invalid NO_SMART_JOIN_ATTRIBUTE = 4008 -# Cannot update the value of the smart join attribute. +# must not change the value of the smartJoinAttribute CLUSTER_MUST_NOT_CHANGE_SMART_JOIN_ATTRIBUTE = 4009 -######################### -# Cluster Repair Errors # -######################### - -# General error during cluster repairs. -CLUSTER_REPAIRS_FAILED = 5000 - -# Cluster repairs not healthy enough. -CLUSTER_REPAIRS_NOT_ENOUGH_HEALTHY = 5001 - -# Raised on various inconsistencies regarding the replication factor. -CLUSTER_REPAIRS_REPLICATION_FACTOR_VIOLATED = 5002 - -# Repaired collection has some shards without DBServers. -CLUSTER_REPAIRS_NO_DBSERVERS = 5003 - -# Shard in collection and its prototype in the corresponding "shard_like" -# collection have mismatching leaders. -CLUSTER_REPAIRS_MISMATCHING_LEADERS = 5004 - -# Shard in collection and its prototype in the corresponding "shard_like" -# collection don't have the same followers. -CLUSTER_REPAIRS_MISMATCHING_FOLLOWERS = 5005 +# non disjoint edge found +INVALID_DISJOINT_SMART_EDGE = 4010 -# Repaired collection does not have "shard_like" as expected. -CLUSTER_REPAIRS_INCONSISTENT_ATTRIBUTES = 5006 - -# Collection and its "shard_like" prototype have unequal number of DBServers. -CLUSTER_REPAIRS_MISMATCHING_SHARDS = 5007 - -# Move shard job failed during cluster repairs. -CLUSTER_REPAIRS_JOB_FAILED = 5008 - -# Move shard job disappeared before finishing. -CLUSTER_REPAIRS_JOB_DISAPPEARED = 5009 - -# Agency transaction failed during either sending or executing it. -CLUSTER_REPAIRS_OPERATION_FAILED = 5010 +# Unsupported alternating Smart and Satellite in Disjoint SmartGraph. +UNSUPPORTED_CHANGE_IN_SMART_TO_SATELLITE_DISJOINT_EDGE_DIRECTION = 4011 ################# -# Agency Errors # +# Agency errors # ################# +# malformed gossip message +AGENCY_MALFORMED_GOSSIP_MESSAGE = 20001 + +# malformed inquire request +AGENCY_MALFORMED_INQUIRE_REQUEST = 20002 + # Inform message must be an object. AGENCY_INFORM_MUST_BE_OBJECT = 20011 -# Inform message must contain a uint parameter 'term'. +# Inform message must contain uint parameter 'term' AGENCY_INFORM_MUST_CONTAIN_TERM = 20012 -# Inform message must contain a string parameter 'ID'. +# Inform message must contain string parameter 'id' AGENCY_INFORM_MUST_CONTAIN_ID = 20013 -# Inform message must contain an array 'active'. +# Inform message must contain array 'active' AGENCY_INFORM_MUST_CONTAIN_ACTIVE = 20014 -# Inform message must contain an object 'pool'. +# Inform message must contain object 'pool' AGENCY_INFORM_MUST_CONTAIN_POOL = 20015 -# Inform message must contain an object 'min ping'. +# Inform message must contain object 'min ping' AGENCY_INFORM_MUST_CONTAIN_MIN_PING = 20016 -# Inform message must contain an object 'max ping'. +# Inform message must contain object 'max ping' AGENCY_INFORM_MUST_CONTAIN_MAX_PING = 20017 -# Inform message must contain an object 'timeoutMult'. +# Inform message must contain object 'timeoutMult' AGENCY_INFORM_MUST_CONTAIN_TIMEOUT_MULT = 20018 -# Cannot rebuild readDB or the spearHead from replicated log. +# Cannot rebuild readDB and spearHead AGENCY_CANNOT_REBUILD_DBS = 20021 +# malformed agency transaction +AGENCY_MALFORMED_TRANSACTION = 20030 + ###################### -# Supervision Errors # +# Supervision errors # ###################### -# General supervision failure. +# general supervision failure SUPERVISION_GENERAL_FAILURE = 20501 -##################### -# Dispatcher Errors # -##################### +#################### +# Scheduler errors # +#################### -# Queue is full. +# queue is full QUEUE_FULL = 21003 +# queue time violated +QUEUE_TIME_REQUIREMENT_VIOLATED = 21004 + +# too many detached scheduler threads +TOO_MANY_DETACHED_THREADS = 21005 + ###################### -# Maintenance Errors # +# Maintenance errors # ###################### -# Maintenance action cannot be stopped once started. -ACTION_OPERATION_UNABORTABLE = 6002 - -# This maintenance action is still processing. +# maintenance action still processing ACTION_UNFINISHED = 6003 -# No such maintenance action exists. -NO_SUCH_ACTION = 6004 - ######################### -# Backup/Restore Errors # +# Backup/Restore errors # ######################### -# Failed to create hot backup set. +# internal hot backup error HOT_BACKUP_INTERNAL = 7001 -# Failed to restore to hot backup set. +# internal hot restore error HOT_RESTORE_INTERNAL = 7002 -# The hot backup set cannot be restored on non-matching cluster topology. +# backup does not match this topology BACKUP_TOPOLOGY = 7003 -# No space left on device. +# no space left on device NO_SPACE_LEFT_ON_DEVICE = 7004 -# Failed to upload hot backup set to remote target. +# failed to upload hot backup set to remote target FAILED_TO_UPLOAD_BACKUP = 7005 -# Failed to download hot backup set from remote source. +# failed to download hot backup set from remote source FAILED_TO_DOWNLOAD_BACKUP = 7006 -# Cannot find a hot backup set with this ID. +# no such hot backup set can be found NO_SUCH_HOT_BACKUP = 7007 -# Invalid remote repository configuration. +# remote hotback repository configuration error REMOTE_REPOSITORY_CONFIG_BAD = 7008 -# Some of the db servers cannot be reached for transaction locks. +# some db servers cannot be reached for transaction locks LOCAL_LOCK_FAILED = 7009 -# Some of the db servers cannot be reached for transaction locks. +# some db servers cannot be reached for transaction locks LOCAL_LOCK_RETRY = 7010 -# Conflict of multiple hot backup processes. +# hot backup conflict HOT_BACKUP_CONFLICT = 7011 -# One or more db servers could not be reached for hot backup inquiry. +# hot backup not all db servers reachable HOT_BACKUP_DBSERVERS_AWOL = 7012 + +######################### +# Plan Analyzers errors # +######################### + +# analyzers in plan could not be modified +CLUSTER_COULD_NOT_MODIFY_ANALYZERS_IN_PLAN = 7021 + +############# +# Licensing # +############# + +# license has expired or is invalid +LICENSE_EXPIRED_OR_INVALID = 9001 + +# license verification failed +LICENSE_SIGNATURE_VERIFICATION = 9002 + +# non-matching license id +LICENSE_NON_MATCHING_ID = 9003 + +# feature is not enabled by the license +LICENSE_FEATURE_NOT_ENABLED = 9004 + +# the resource is exhausted +LICENSE_RESOURCE_EXHAUSTED = 9005 + +# invalid license +LICENSE_INVALID = 9006 + +# conflicting license +LICENSE_CONFLICT = 9007 + +# failed to validate license signature +LICENSE_VALIDATION_FAILED = 9008 diff --git a/arango/exceptions.py b/arango/exceptions.py index 6c78e3cd..29bcdc17 100644 --- a/arango/exceptions.py +++ b/arango/exceptions.py @@ -153,6 +153,10 @@ class AQLFunctionDeleteError(ArangoServerError): """Failed to delete AQL user function.""" +class AQLQueryRulesGetError(ArangoServerError): + """Failed to retrieve AQL query rules.""" + + ############################## # Async Execution Exceptions # ############################## @@ -228,6 +232,15 @@ class BatchExecuteError(ArangoServerError): """Failed to execute batch API request.""" +######################################### +# Overload Control Execution Exceptions # +######################################### + + +class OverloadControlExecutorError(ArangoServerError): + """Failed to execute overload controlled API request.""" + + ######################### # Collection Exceptions # ######################### @@ -237,6 +250,10 @@ class CollectionListError(ArangoServerError): """Failed to retrieve collections.""" +class CollectionInformationError(ArangoServerError): + """Failed to retrieve collection information.""" + + class CollectionPropertiesError(ArangoServerError): """Failed to retrieve collection properties.""" @@ -245,6 +262,10 @@ class CollectionConfigureError(ArangoServerError): """Failed to configure collection properties.""" +class CollectionShardsError(ArangoServerError): + """Failed to retrieve collection shards.""" + + class CollectionStatisticsError(ArangoServerError): """Failed to retrieve collection statistics.""" @@ -257,6 +278,10 @@ class CollectionChecksumError(ArangoServerError): """Failed to retrieve collection checksum.""" +class CollectionCompactError(ArangoServerError): + """Failed to compact collection.""" + + class CollectionCreateError(ArangoServerError): """Failed to create collection.""" @@ -335,6 +360,14 @@ class DatabaseDeleteError(ArangoServerError): """Failed to delete database.""" +class DatabaseSupportInfoError(ArangoServerError): + """Failed to retrieve support info for deployment.""" + + +class DatabaseCompactError(ArangoServerError): + """Failed to compact databases.""" + + ####################### # Document Exceptions # ####################### @@ -543,6 +576,14 @@ class IndexCreateError(ArangoServerError): """Failed to create collection index.""" +class IndexGetError(ArangoServerError): + """Failed to retrieve collection index.""" + + +class IndexMissingError(ArangoClientError): + """Failed to find collection index.""" + + class IndexDeleteError(ArangoServerError): """Failed to delete collection index.""" @@ -573,7 +614,7 @@ class PregelJobDeleteError(ArangoServerError): ##################### -class ServerConnectionError(ArangoClientError): +class ServerConnectionError(ArangoServerError): """Failed to connect to ArangoDB server.""" @@ -589,6 +630,14 @@ class ServerDetailsError(ArangoServerError): """Failed to retrieve server details.""" +class ServerLicenseGetError(ArangoServerError): + """Failed to retrieve server license.""" + + +class ServerLicenseSetError(ArangoServerError): + """Failed to set server license.""" + + class ServerStatusError(ArangoServerError): """Failed to retrieve server status.""" @@ -605,6 +654,10 @@ class ServerShutdownError(ArangoServerError): """Failed to initiate shutdown sequence.""" +class ServerShutdownProgressError(ArangoServerError): + """Failed to retrieve soft shutdown progress.""" + + class ServerRunTestsError(ArangoServerError): """Failed to execute server tests.""" @@ -621,10 +674,22 @@ class ServerLogLevelError(ArangoServerError): """Failed to retrieve server log levels.""" +class ServerLogLevelResetError(ArangoServerError): + """Failed to reset server log levels.""" + + +class ServerLogSettingError(ArangoServerError): + """Failed to retrieve server log settings.""" + + class ServerLogLevelSetError(ArangoServerError): """Failed to set server log levels.""" +class ServerLogSettingSetError(ArangoServerError): + """Failed to set server log settings.""" + + class ServerReloadRoutingError(ArangoServerError): """Failed to reload routing details.""" @@ -638,7 +703,15 @@ class ServerMetricsError(ArangoServerError): class ServerRoleError(ArangoServerError): - """Failed to retrieve server role in a cluster.""" + """Failed to retrieve server role.""" + + +class ServerModeError(ArangoServerError): + """Failed to retrieve server mode.""" + + +class ServerModeSetError(ArangoServerError): + """Failed to set server mode.""" class ServerTLSError(ArangoServerError): @@ -653,6 +726,18 @@ class ServerEncryptionError(ArangoServerError): """Failed to reload user-defined encryption keys.""" +class ServerCurrentOptionsGetError(ArangoServerError): + """Failed to retrieve currently-set server options.""" + + +class ServerAvailableOptionsGetError(ArangoServerError): + """Failed to retrieve available server options.""" + + +class ServerExecuteError(ArangoServerError): + """Failed to execute raw JavaScript command.""" + + ##################### # Task Exceptions # ##################### @@ -699,6 +784,14 @@ class TransactionAbortError(ArangoServerError): """Failed to abort transaction.""" +class TransactionFetchError(ArangoServerError): + """Failed to fetch existing transaction.""" + + +class TransactionListError(ArangoServerError): + """Failed to retrieve transactions.""" + + ################### # User Exceptions # ################### @@ -919,7 +1012,11 @@ class ClusterServerIDError(ArangoServerError): class ClusterServerRoleError(ArangoServerError): - """Failed to retrieve server role.""" + """Failed to retrieve server role in a cluster.""" + + +class ClusterServerModeError(ArangoServerError): + """Failed to retrieve server mode in a cluster.""" class ClusterServerStatisticsError(ArangoServerError): @@ -946,6 +1043,14 @@ class ClusterServerCountError(ArangoServerError): """Failed to retrieve cluster server count.""" +class ClusterRebalanceError(ArangoServerError): + """Failed to execute cluster re-balancing operation (load/set).""" + + +class ClusterVpackSortMigrationError(ArangoServerError): + """Failed to execute vpack sort migration request.""" + + ################## # JWT Exceptions # ################## @@ -961,3 +1066,18 @@ class JWTSecretListError(ArangoServerError): class JWTSecretReloadError(ArangoServerError): """Failed to reload JWT secrets.""" + + +class JWTRefreshError(ArangoClientError): + """Failed to refresh JWT token.""" + + +class JWTExpiredError(ArangoClientError): + """JWT token has expired.""" + + +################################### +# Parameter Validation Exceptions # +################################### +class SortValidationError(ArangoClientError): + """Invalid sort parameters.""" diff --git a/arango/executor.py b/arango/executor.py index 722a1bdd..9330e11c 100644 --- a/arango/executor.py +++ b/arango/executor.py @@ -4,20 +4,22 @@ "AsyncApiExecutor", "BatchApiExecutor", "TransactionApiExecutor", + "OverloadControlApiExecutor", ] from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor +from os import cpu_count from typing import Any, Callable, Optional, Sequence, Tuple, TypeVar, Union -from urllib.parse import urlencode -from uuid import uuid4 from arango.connection import Connection from arango.exceptions import ( AsyncExecuteError, - BatchExecuteError, BatchStateError, + OverloadControlExecutorError, TransactionAbortError, TransactionCommitError, + TransactionFetchError, TransactionInitError, TransactionStatusError, ) @@ -25,13 +27,13 @@ from arango.request import Request from arango.response import Response from arango.typings import Fields, Json -from arango.utils import suppress_warning ApiExecutor = Union[ "DefaultApiExecutor", "AsyncApiExecutor", "BatchApiExecutor", "TransactionApiExecutor", + "OverloadControlApiExecutor", ] T = TypeVar("T") @@ -123,35 +125,29 @@ class BatchApiExecutor: If set to False, API executions return None and no results are tracked client-side. :type return_result: bool + :param max_workers: Use a thread pool of at most `max_workers`. If None, + the default value is the number of CPUs. For backwards compatibility, + the default value is 1, effectively behaving like single-threaded + execution. + :type max_workers: Optional[int] """ - def __init__(self, connection: Connection, return_result: bool) -> None: + def __init__( + self, + connection: Connection, + return_result: bool, + max_workers: Optional[int] = 1, + ) -> None: self._conn = connection self._return_result: bool = return_result self._queue: OrderedDict[str, Tuple[Request, BatchJob[Any]]] = OrderedDict() self._committed: bool = False + self._max_workers: int = max_workers or cpu_count() # type: ignore @property def context(self) -> str: return "batch" - def _stringify_request(self, request: Request) -> str: - path = request.endpoint - - if request.params is not None: - path += f"?{urlencode(request.params)}" - buffer = [f"{request.method} {path} HTTP/1.1"] - - if request.headers is not None: - for key, value in sorted(request.headers.items()): - buffer.append(f"{key}: {value}") - - if request.data is not None: - serialized = self._conn.serialize(request.data) - buffer.append("\r\n" + serialized) - - return "\r\n".join(buffer) - @property def jobs(self) -> Optional[Sequence[BatchJob[Any]]]: """Return the queued batch jobs. @@ -187,7 +183,7 @@ def execute( return job if self._return_result else None def commit(self) -> Optional[Sequence[BatchJob[Any]]]: - """Execute the queued requests in a single batch API request. + """Execute the queued requests in a batch of requests. If **return_result** parameter was set to True during initialization, :class:`arango.job.BatchJob` instances are populated with results. @@ -196,9 +192,7 @@ def commit(self) -> Optional[Sequence[BatchJob[Any]]]: False during initialization. :rtype: [arango.job.BatchJob] | None :raise arango.exceptions.BatchStateError: If batch state is invalid - (e.g. batch was already committed or size of response from server - did not match the expected). - :raise arango.exceptions.BatchExecuteError: If commit fails. + (e.g. batch was already committed). """ if self._committed: raise BatchStateError("batch already committed") @@ -208,65 +202,18 @@ def commit(self) -> Optional[Sequence[BatchJob[Any]]]: if len(self._queue) == 0: return self.jobs - # Boundary used for multipart request - boundary = uuid4().hex - - # Build the batch request payload - buffer = [] - for req, job in self._queue.values(): - buffer.append(f"--{boundary}") - buffer.append("Content-Type: application/x-arango-batchpart") - buffer.append(f"Content-Id: {job.id}") - buffer.append("\r\n" + self._stringify_request(req)) - buffer.append(f"--{boundary}--") + with ThreadPoolExecutor( + max_workers=min(self._max_workers, len(self._queue)) + ) as executor: + for req, job in self._queue.values(): + job._future = executor.submit(self._conn.send_request, req) - request = Request( - method="post", - endpoint="/_api/batch", - headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, - data="\r\n".join(buffer), - ) - with suppress_warning("requests.packages.urllib3.connectionpool"): - resp = self._conn.send_request(request) - - if not resp.is_success: - raise BatchExecuteError(resp, request) + for _, job in self._queue.values(): + job._status = "done" if not self._return_result: return None - url_prefix = resp.url.strip("/_api/batch") - raw_responses = resp.raw_body.split(f"--{boundary}")[1:-1] - - if len(self._queue) != len(raw_responses): - raise BatchStateError( - "expecting {} parts in batch response but got {}".format( - len(self._queue), len(raw_responses) - ) - ) - for raw_resp in raw_responses: - # Parse and breakdown the batch response body - resp_parts = raw_resp.strip().split("\r\n") - raw_content_id = resp_parts[1] - raw_body = resp_parts[-1] - raw_status = resp_parts[3] - job_id = raw_content_id.split(" ")[1] - _, status_code, status_text = raw_status.split(" ", 2) - - # Update the corresponding batch job - queued_req, queued_job = self._queue[job_id] - - queued_job._status = "done" - resp = Response( - method=queued_req.method, - url=url_prefix + queued_req.endpoint, - headers={}, - status_code=int(status_code), - status_text=status_text, - raw_body=raw_body, - ) - queued_job._response = self._conn.prep_response(resp) - return self.jobs @@ -293,6 +240,13 @@ class TransactionApiExecutor: :type lock_timeout: int :param max_size: Max transaction size in bytes. :type max_size: int + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool | None + :param transaction_id: Initialize using an existing transaction instead of starting + a new transaction. + :type transaction_id: str | None + :param skip_fast_lock_round: Whether to disable fast locking for write operations. + :type skip_fast_lock_round: bool | None """ def __init__( @@ -305,6 +259,9 @@ def __init__( allow_implicit: Optional[bool] = None, lock_timeout: Optional[int] = None, max_size: Optional[int] = None, + allow_dirty_read: bool = False, + transaction_id: Optional[str] = None, + skip_fast_lock_round: Optional[bool] = None, ) -> None: self._conn = connection @@ -325,15 +282,32 @@ def __init__( data["lockTimeout"] = lock_timeout if max_size is not None: data["maxTransactionSize"] = max_size + if skip_fast_lock_round is not None: + data["skipFastLockRound"] = skip_fast_lock_round + + if transaction_id is None: + request = Request( + method="post", + endpoint="/_api/transaction/begin", + data=data, + headers=( + {"x-arango-allow-dirty-read": "true"} if allow_dirty_read else None + ), + ) + resp = self._conn.send_request(request) - request = Request(method="post", endpoint="/_api/transaction/begin", data=data) - resp = self._conn.send_request(request) + if not resp.is_success: + raise TransactionInitError(resp, request) - if not resp.is_success: - raise TransactionInitError(resp, request) + result = resp.body["result"] + self._id: str = result["id"] + else: + self._id = transaction_id - result: Json = resp.body["result"] - self._id: str = result["id"] + try: + self.status() + except TransactionStatusError as err: + raise TransactionFetchError(err.response, err.request) @property def context(self) -> str: @@ -348,16 +322,25 @@ def id(self) -> str: """ return self._id - def execute(self, request: Request, response_handler: Callable[[Response], T]) -> T: + def execute( + self, + request: Request, + response_handler: Callable[[Response], T], + allow_dirty_read: bool = False, + ) -> T: """Execute API request in a transaction and return the result. :param request: HTTP request. :type request: arango.request.Request :param response_handler: HTTP response handler. :type response_handler: callable + :param allow_dirty_read: Allow reads from followers in a cluster. + :type allow_dirty_read: bool | None :return: API execution result. """ request.headers["x-arango-trx-id"] = self._id + if allow_dirty_read: + request.headers["x-arango-allow-dirty-read"] = "true" resp = self._conn.send_request(request) return response_handler(resp) @@ -411,3 +394,84 @@ def abort(self) -> bool: if resp.is_success: return True raise TransactionAbortError(resp, request) + + +class OverloadControlApiExecutor: + """Allows setting the maximum acceptable server-side queuing time + for client requests. + + :param connection: HTTP connection. + :type connection: arango.connection.BasicConnection | + arango.connection.JwtConnection | arango.connection.JwtSuperuserConnection + :param max_queue_time_seconds: Maximum server-side queuing time in seconds. + :type max_queue_time_seconds: float + """ + + def __init__( + self, connection: Connection, max_queue_time_seconds: Optional[float] = None + ) -> None: + self._conn = connection + self._max_queue_time_seconds = max_queue_time_seconds + self._queue_time_seconds = 0.0 + + @property + def context(self) -> str: # pragma: no cover + return "overload-control" + + @property + def queue_time_seconds(self) -> float: + """Return the most recent request queuing/de-queuing time. + Defaults to 0 if no request has been sent. + + :return: Server-side queuing time in seconds. + :rtype: float + """ + return self._queue_time_seconds + + @property + def max_queue_time_seconds(self) -> Optional[float]: + """Return the maximum server-side queuing time. + + :return: Maximum server-side queuing time in seconds. + :rtype: Optional[float] + """ + return self._max_queue_time_seconds + + @max_queue_time_seconds.setter + def max_queue_time_seconds(self, value: Optional[float]) -> None: + """Set the maximum server-side queuing time. + Setting it to None disables the feature. + + :param value: Maximum server-side queuing time in seconds. + :type value: Optional[float] + """ + self._max_queue_time_seconds = value + + def execute( + self, + request: Request, + response_handler: Callable[[Response], T], + ) -> T: + """Execute an API request and return the result. + + :param request: HTTP request. + :type request: arango.request.Request + :param response_handler: HTTP response handler. + :type response_handler: callable + :return: API execution result. + """ + if self._max_queue_time_seconds is not None: + request.headers["x-arango-queue-time-seconds"] = str( + self._max_queue_time_seconds + ) + resp = self._conn.send_request(request) + + if not resp.is_success: + raise OverloadControlExecutorError(resp, request) + + if "X-Arango-Queue-Time-Seconds" in resp.headers: + self._queue_time_seconds = float( + resp.headers["X-Arango-Queue-Time-Seconds"] + ) + + return response_handler(resp) diff --git a/arango/formatter.py b/arango/formatter.py index 38a2baf7..1c63bf00 100644 --- a/arango/formatter.py +++ b/arango/formatter.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Sequence from arango.typings import Headers, Json @@ -20,14 +20,22 @@ def format_body(body: Json) -> Json: return body -def format_index(body: Json) -> Json: +def format_index(body: Json, formatter: bool = True) -> Json: """Format index data. :param body: Input body. :type body: dict + :param formatter: Convert (most) keys to snake_case. + :type formatter: bool :return: Formatted body. :rtype: dict """ + if not formatter: + body.pop("code") + body.pop("error") + body["id"] = body["id"].split("/", 1)[-1] + return body + result = {"id": body["id"].split("/", 1)[-1], "fields": body["fields"]} if "type" in body: result["type"] = body["type"] @@ -59,6 +67,54 @@ def format_index(body: Json) -> Json: result["worst_indexed_level"] = body["worstIndexedLevel"] if "maxNumCoverCells" in body: result["max_num_cover_cells"] = body["maxNumCoverCells"] + if "storedValues" in body: + result["storedValues"] = body["storedValues"] + if "cacheEnabled" in body: + result["cacheEnabled"] = body["cacheEnabled"] + if "legacyPolygons" in body: + result["legacyPolygons"] = body["legacyPolygons"] + if "estimates" in body: + result["estimates"] = body["estimates"] + if "analyzer" in body: + result["analyzer"] = body["analyzer"] + if "cleanupIntervalStep" in body: + result["cleanup_interval_step"] = body["cleanupIntervalStep"] + if "commitIntervalMsec" in body: + result["commit_interval_msec"] = body["commitIntervalMsec"] + if "consolidationIntervalMsec" in body: + result["consolidation_interval_msec"] = body["consolidationIntervalMsec"] + if "consolidationPolicy" in body: + result["consolidation_policy"] = format_view_consolidation_policy( + body["consolidationPolicy"] + ) + if "features" in body: + result["features"] = body["features"] + if "includeAllFields" in body: + result["include_all_fields"] = body["includeAllFields"] + if "primarySort" in body: + result["primary_sort"] = body["primarySort"] + if "searchField" in body: + result["search_field"] = body["searchField"] + if "trackListPositions" in body: + result["track_list_positions"] = body["trackListPositions"] + if "version" in body: + result["version"] = body["version"] + if "cache" in body: + result["cache"] = body["cache"] + if "primaryKeyCache" in body: + result["primaryKeyCache"] = body["primaryKeyCache"] + if "writebufferIdle" in body: + result["writebuffer_idle"] = body["writebufferIdle"] + if "writebufferActive" in body: + result["writebuffer_active"] = body["writebufferActive"] + if "writebufferSizeMax" in body: + result["writebuffer_max_size"] = body["writebufferSizeMax"] + if "fieldValueTypes" in body: + result["field_value_types"] = body["fieldValueTypes"] + + # Introduced in 3.12 EE + if "optimizeTopK" in body: + result["optimizeTopK"] = body["optimizeTopK"] return verify_format(body, result) @@ -115,6 +171,8 @@ def format_database(body: Json) -> Json: result["replication_factor"] = body["replicationFactor"] if "writeConcern" in body: result["write_concern"] = body["writeConcern"] + if "replicationVersion" in body: + result["replication_version"] = body["replicationVersion"] return verify_format(body, result) @@ -207,6 +265,11 @@ def format_collection(body: Json) -> Json: result["min_revision"] = body["minRevision"] if "schema" in body: result["schema"] = body["schema"] + if body.get("computedValues") is not None: + result["computedValues"] = body["computedValues"] + + if "internalValidatorType" in body: + result["internal_validator_type"] = body["internalValidatorType"] return verify_format(body, result) @@ -298,6 +361,16 @@ def format_aql_query(body: Json) -> Json: result["stream"] = body["stream"] if "user" in body: result["user"] = body["user"] + + # New in 3.11 + if "peakMemoryUsage" in body: + result["peak_memory_usage"] = body["peakMemoryUsage"] + + # New in 3.12.2 + if "modificationQuery" in body: + result["modification_query"] = body["modificationQuery"] + if "warnings" in body: + result["warnings"] = body["warnings"] return verify_format(body, result) @@ -373,6 +446,10 @@ def format_server_status(body: Json) -> Json: """ result: Json = {} + if "agency" in body: + result["agency"] = body["agency"] + if "coordinator" in body: + result["coordinator"] = body["coordinator"] if "foxxApi" in body: result["foxx_api"] = body["foxxApi"] if "host" in body: @@ -774,6 +851,27 @@ def format_view_link(body: Json) -> Json: result["track_list_positions"] = body["trackListPositions"] if "storeValues" in body: result["store_values"] = body["storeValues"] + if "primaryKeyCache" in body: + result["primaryKeyCache"] = body["primaryKeyCache"] + if "companies" in body: + result["companies"] = body["companies"] + + return verify_format(body, result) + + +def format_view_index(body: Json) -> Json: + """Format view index data. + + :param body: Input body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + result: Json = {} + if "collection" in body: + result["collection"] = body["collection"] + if "index" in body: + result["index"] = body["index"] return verify_format(body, result) @@ -845,9 +943,19 @@ def format_view(body: Json) -> Json: if "writebufferSizeMax" in body: result["writebuffer_max_size"] = body["writebufferSizeMax"] if "links" in body: - result["links"] = { - name: format_view_link(link) for name, link in body["links"].items() - } + result["links"] = body["links"] + if "indexes" in body: + result["indexes"] = body["indexes"] + + # Introduced in 3.9.6 EE + if "primaryKeyCache" in body: + result["primaryKeyCache"] = body["primaryKeyCache"] + if "primarySortCache" in body: + result["primarySortCache"] = body["primarySortCache"] + + # Introduced in 3.12 EE + if "optimizeTopK" in body: + result["optimizeTopK"] = body["optimizeTopK"] return verify_format(body, result) @@ -944,6 +1052,9 @@ def format_backup(body: Json) -> Json: if "nrPiecesPresent" in body: result["pieces_present"] = body["nrPiecesPresent"] + if "countIncludesFilesOnly" in body: + result["count_includes_files_only"] = body["countIncludesFilesOnly"] + return verify_format(body, result) @@ -1052,26 +1163,75 @@ def format_pregel_job_data(body: Json) -> Json: """ result: Json = {} - if "aggregators" in body: - result["aggregators"] = body["aggregators"] + if "id" in body: + result["id"] = body["id"] + if "algorithm" in body: + result["algorithm"] = body["algorithm"] + if "created" in body: + result["created"] = body["created"] + if "expires" in body: + result["expires"] = body["expires"] + if "ttl" in body: + result["ttl"] = body["ttl"] + if "algorithm" in body: + result["algorithm"] = body["algorithm"] + if "state" in body: + result["state"] = body["state"] + if "gss" in body: + result["gss"] = body["gss"] + if "totalRuntime" in body: + result["total_runtime"] = body["totalRuntime"] + if "startupTime" in body: + result["startup_time"] = body["startupTime"] if "computationTime" in body: result["computation_time"] = body["computationTime"] + if "storageTime" in body: + result["storageTime"] = body["storageTime"] + if "gssTimes" in body: + result["gssTimes"] = body["gssTimes"] + if "reports" in body: + result["reports"] = body["reports"] + if "vertexCount" in body: + result["vertex_count"] = body["vertexCount"] if "edgeCount" in body: result["edge_count"] = body["edgeCount"] - if "gss" in body: - result["gss"] = body["gss"] + if "aggregators" in body: + result["aggregators"] = body["aggregators"] if "receivedCount" in body: result["received_count"] = body["receivedCount"] if "sendCount" in body: result["send_count"] = body["sendCount"] - if "startupTime" in body: - result["startup_time"] = body["startupTime"] - if "state" in body: - result["state"] = body["state"] - if "totalRuntime" in body: - result["total_runtime"] = body["totalRuntime"] - if "vertexCount" in body: - result["vertex_count"] = body["vertexCount"] + + # The detail element was introduced in 3.10 + if "detail" in body: + result["detail"] = body["detail"] + if "database" in body: + result["database"] = body["database"] + if "masterContext" in body: + result["master_context"] = body["masterContext"] + if "parallelism" in body: + result["parallelism"] = body["parallelism"] + if "useMemoryMaps" in body: + result["use_memory_maps"] = body["useMemoryMaps"] + + # Introduced in 3.11 + if "user" in body: + result["user"] = body["user"] + if "graphLoaded" in body: + result["graph_loaded"] = body["graphLoaded"] + + return verify_format(body, result) + + +def format_pregel_job_list(body: Sequence[Json]) -> Json: + """Format Pregel job list data. + + :param body: Input body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + result: Json = {"jobs": [format_pregel_job_data(j) for j in body]} return verify_format(body, result) @@ -1101,12 +1261,20 @@ def format_graph_properties(body: Json) -> Json: } if "isSmart" in body: result["smart"] = body["isSmart"] + if "isDisjoint" in body: + result["disjoint"] = body["isDisjoint"] + if "isSatellite" in body: + result["is_satellite"] = body["isSatellite"] if "smartGraphAttribute" in body: result["smart_field"] = body["smartGraphAttribute"] if "numberOfShards" in body: result["shard_count"] = body["numberOfShards"] if "replicationFactor" in body: result["replication_factor"] = body["replicationFactor"] + if "minReplicationFactor" in body: + result["min_replication_factor"] = body["minReplicationFactor"] + if "writeConcern" in body: + result["write_concern"] = body["writeConcern"] return verify_format(body, result) @@ -1141,3 +1309,47 @@ def format_query_cache_entry(body: Json) -> Json: result["data_sources"] = body["dataSources"] return verify_format(body, result) + + +def format_query_rule_item(body: Json) -> Json: + """Format AQL query rule item. + + :param body: Input body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + result = {} + + if "name" in body: + result["name"] = body["name"] + if "flags" in body: + result["flags"] = format_query_rule_item_flags(body["flags"]) + + return verify_format(body, result) + + +def format_query_rule_item_flags(body: Json) -> Json: + """Format AQL query rule item flags. + + :param body: Input body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + result = {} + + if "hidden" in body: + result["hidden"] = body["hidden"] + if "clusterOnly" in body: + result["clusterOnly"] = body["clusterOnly"] + if "canBeDisabled" in body: + result["canBeDisabled"] = body["canBeDisabled"] + if "canCreateAdditionalPlans" in body: + result["canCreateAdditionalPlans"] = body["canCreateAdditionalPlans"] + if "disabledByDefault" in body: + result["disabledByDefault"] = body["disabledByDefault"] + if "enterpriseOnly" in body: + result["enterpriseOnly"] = body["enterpriseOnly"] + + return verify_format(body, result) diff --git a/arango/graph.py b/arango/graph.py index 0933c72e..3279129f 100644 --- a/arango/graph.py +++ b/arango/graph.py @@ -1,6 +1,7 @@ __all__ = ["Graph"] from typing import List, Optional, Sequence, Union +from warnings import warn from arango.api import ApiGroup from arango.collection import EdgeCollection, VertexCollection @@ -107,9 +108,9 @@ def response_handler(resp: Response) -> bool: return self._execute(request, response_handler) def vertex_collections(self) -> Result[List[str]]: - """Return vertex collections in the graph that are not orphaned. + """Return vertex collections in the graph. - :return: Names of vertex collections that are not orphaned. + :return: Names of vertex collections in Edge Definitions and Orphan Collections. :rtype: [str] :raise arango.exceptions.VertexCollectionListError: If retrieval fails. """ @@ -384,6 +385,11 @@ def traverse( ) -> Result[Json]: """Traverse the graph and return the visited vertices and edges. + .. warning:: + + This method is deprecated and no longer works since ArangoDB 3.12. + The preferred way to traverse graphs is via AQL. + :param start_vertex: Start vertex document ID or body with "_id" field. :type start_vertex: str | dict :param direction: Traversal direction. Allowed values are "outbound" @@ -441,6 +447,9 @@ def traverse( :rtype: dict :raise arango.exceptions.GraphTraverseError: If traversal fails. """ + m = "The HTTP traversal API is deprecated since version 3.4.0. The preferred way to traverse graphs is via AQL." # noqa: E501 + warn(m, DeprecationWarning, stacklevel=2) + if strategy is not None: if strategy.lower() == "dfs": strategy = "depthfirst" diff --git a/arango/http.py b/arango/http.py index 195eecf6..d0b17939 100644 --- a/arango/http.py +++ b/arango/http.py @@ -1,16 +1,27 @@ -__all__ = ["HTTPClient", "DefaultHTTPClient"] +__all__ = [ + "HTTPClient", + "DefaultHTTPClient", + "DeflateRequestCompression", + "RequestCompression", + "DEFAULT_REQUEST_TIMEOUT", +] +import typing +import zlib from abc import ABC, abstractmethod -from typing import MutableMapping, Optional, Tuple, Union +from typing import Any, MutableMapping, Optional, Tuple, Union from requests import Session -from requests.adapters import HTTPAdapter +from requests.adapters import DEFAULT_POOLBLOCK, DEFAULT_POOLSIZE, HTTPAdapter from requests_toolbelt import MultipartEncoder +from urllib3.poolmanager import PoolManager from urllib3.util.retry import Retry from arango.response import Response from arango.typings import Headers +DEFAULT_REQUEST_TIMEOUT = 60 + class HTTPClient(ABC): # pragma: no cover """Abstract base class for HTTP clients.""" @@ -36,7 +47,7 @@ def send_request( url: str, headers: Optional[Headers] = None, params: Optional[MutableMapping[str, str]] = None, - data: Union[str, MultipartEncoder, None] = None, + data: Union[str, bytes, MultipartEncoder, None] = None, auth: Optional[Tuple[str, str]] = None, ) -> Response: """Send an HTTP request. @@ -54,7 +65,7 @@ def send_request( :param params: URL (query) parameters. :type params: dict :param data: Request payload. - :type data: str | MultipartEncoder | None + :type data: str | bytes | MultipartEncoder | None :param auth: Username and password. :type auth: tuple :returns: HTTP response. @@ -63,12 +74,101 @@ def send_request( raise NotImplementedError +class DefaultHTTPAdapter(HTTPAdapter): + """Default transport adapter implementation + + :param connection_timeout: Socket timeout in seconds for each individual connection. + :type connection_timeout: int | float + :param pool_connections: The number of urllib3 connection pools to cache. + :type pool_connections: int + :param pool_maxsize: The maximum number of connections to save in the pool. + :type pool_maxsize: int + :param pool_timeout: If set, then the pool will be set to block=True, + and requests will block for pool_timeout seconds and raise + EmptyPoolError if no connection is available within the time period. + :type pool_timeout: int | float | None + :param kwargs: Additional keyword arguments passed to the HTTPAdapter constructor. + :type kwargs: Any + """ + + __attrs__ = [ + "max_retries", + "config", + "_connection_timeout", + "_pool_connections", + "_pool_maxsize", + "_pool_timeout", + "_pool_block", + ] + + def __init__( + self, + connection_timeout: Union[int, float, None] = DEFAULT_REQUEST_TIMEOUT, + pool_connections: int = DEFAULT_POOLSIZE, + pool_maxsize: int = DEFAULT_POOLSIZE, + pool_timeout: Union[int, float, None] = None, + **kwargs: Any, + ) -> None: + self._connection_timeout = connection_timeout + self._pool_timeout = pool_timeout + super().__init__( + pool_connections=pool_connections, pool_maxsize=pool_maxsize, **kwargs + ) + + @typing.no_type_check + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ) -> None: + kwargs = pool_kwargs + kwargs.update( + dict( + num_pools=connections, + maxsize=maxsize, + timeout=self._connection_timeout, + ) + ) + if self._pool_timeout is not None: + kwargs["block"] = True + kwargs["timeout"] = self._pool_timeout + else: + kwargs["block"] = False + self.poolmanager = PoolManager(**kwargs) + + class DefaultHTTPClient(HTTPClient): - """Default HTTP client implementation.""" + """Default HTTP client implementation. - REQUEST_TIMEOUT = 60 - RETRY_ATTEMPTS = 3 - BACKOFF_FACTOR = 1 + :param request_timeout: Timeout in seconds for each individual connection. + :type request_timeout: int | float + :param retry_attempts: Number of retry attempts. + :type retry_attempts: int + :param backoff_factor: Backoff factor for retry attempts. + :type backoff_factor: float + :param pool_connections: The number of urllib3 connection pools to cache. + :type pool_connections: int + :param pool_maxsize: The maximum number of connections to save in the pool. + :type pool_maxsize: int + :param pool_timeout: If set, then the pool will be set to block=True, + and requests will block for pool_timeout seconds and raise + EmptyPoolError if no connection is available within the time period. + :type pool_timeout: int | float | None + """ + + def __init__( + self, + request_timeout: Union[int, float, None] = DEFAULT_REQUEST_TIMEOUT, + retry_attempts: int = 3, + backoff_factor: float = 1.0, + pool_connections: int = 10, + pool_maxsize: int = 10, + pool_timeout: Union[int, float, None] = None, + ) -> None: + self.request_timeout = request_timeout + self._retry_attempts = retry_attempts + self._backoff_factor = backoff_factor + self._pool_connections = pool_connections + self._pool_maxsize = pool_maxsize + self._pool_timeout = pool_timeout def create_session(self, host: str) -> Session: """Create and return a new session/connection. @@ -79,12 +179,18 @@ def create_session(self, host: str) -> Session: :rtype: requests.Session """ retry_strategy = Retry( - total=self.RETRY_ATTEMPTS, - backoff_factor=self.BACKOFF_FACTOR, + total=self._retry_attempts, + backoff_factor=self._backoff_factor, status_forcelist=[429, 500, 502, 503, 504], allowed_methods=["HEAD", "GET", "OPTIONS"], ) - http_adapter = HTTPAdapter(max_retries=retry_strategy) + http_adapter = DefaultHTTPAdapter( + connection_timeout=self.request_timeout, + pool_connections=self._pool_connections, + pool_maxsize=self._pool_maxsize, + pool_timeout=self._pool_timeout, + max_retries=retry_strategy, + ) session = Session() session.mount("https://", http_adapter) @@ -99,7 +205,7 @@ def send_request( url: str, headers: Optional[Headers] = None, params: Optional[MutableMapping[str, str]] = None, - data: Union[str, MultipartEncoder, None] = None, + data: Union[str, bytes, MultipartEncoder, None] = None, auth: Optional[Tuple[str, str]] = None, ) -> Response: """Send an HTTP request. @@ -115,7 +221,7 @@ def send_request( :param params: URL (query) parameters. :type params: dict :param data: Request payload. - :type data: str | MultipartEncoder | None + :type data: str | bytes | MultipartEncoder | None :param auth: Username and password. :type auth: tuple :returns: HTTP response. @@ -128,7 +234,7 @@ def send_request( data=data, headers=headers, auth=auth, - timeout=self.REQUEST_TIMEOUT, + timeout=self.request_timeout, ) return Response( method=method, @@ -138,3 +244,75 @@ def send_request( status_text=response.reason, raw_body=response.text, ) + + +class RequestCompression(ABC): # pragma: no cover + """Abstract base class for request compression.""" + + @abstractmethod + def needs_compression(self, data: str) -> bool: + """ + :param data: Data to be compressed. + :type data: str + :returns: True if the data needs to be compressed. + :rtype: bool + """ + raise NotImplementedError + + @abstractmethod + def compress(self, data: str) -> bytes: + """Compress the data. + + :param data: Data to be compressed. + :type data: str + :returns: Compressed data. + :rtype: bytes + """ + raise NotImplementedError + + @abstractmethod + def encoding(self) -> str: + """Return the content encoding exactly as it should + appear in the headers. + + :returns: Content encoding. + :rtype: str + """ + raise NotImplementedError + + +class DeflateRequestCompression(RequestCompression): + """Compress requests using the 'deflate' algorithm.""" + + def __init__(self, threshold: int = 1024, level: int = 6): + """ + :param threshold: Will compress requests to the server if + the size of the request body (in bytes) is at least the value of this + option. + :type threshold: int + :param level: Compression level, in 0-9 or -1. + :type level: int + """ + self._threshold = threshold + self._level = level + + def needs_compression(self, data: str) -> bool: + """ + :param data: Data to be compressed. + :type data: str + :returns: True if the data needs to be compressed. + :rtype: bool + """ + return len(data) >= self._threshold + + def compress(self, data: str) -> bytes: + """ + :param data: Data to be compressed. + :type data: str + :returns: Compressed data. + :rtype: bytes + """ + return zlib.compress(data.encode("utf-8"), level=self._level) + + def encoding(self) -> str: + return "deflate" diff --git a/arango/job.py b/arango/job.py index d2ce1c2d..85c96bc8 100644 --- a/arango/job.py +++ b/arango/job.py @@ -1,5 +1,6 @@ __all__ = ["AsyncJob", "BatchJob"] +from concurrent.futures import Future from typing import Callable, Generic, Optional, TypeVar from uuid import uuid4 @@ -59,10 +60,10 @@ def status(self) -> str: fail. :return: Async job status. Possible values are "pending" (job is still - in queue), "done" (job finished or raised an error), or "cancelled" - (job was cancelled before completion). + in queue), "done" (job finished or raised an error). :rtype: str - :raise arango.exceptions.AsyncJobStatusError: If retrieval fails. + :raise arango.exceptions.AsyncJobStatusError: If retrieval fails or + job is not found. """ request = Request(method="get", endpoint=f"/_api/job/{self._id}") resp = self._conn.send_request(request) @@ -160,13 +161,13 @@ class BatchJob(Generic[T]): :type response_handler: callable """ - __slots__ = ["_id", "_status", "_response", "_response_handler"] + __slots__ = ["_id", "_status", "_response_handler", "_future"] def __init__(self, response_handler: Callable[[Response], T]) -> None: self._id = uuid4().hex self._status = "pending" - self._response: Optional[Response] = None self._response_handler = response_handler + self._future: Optional[Future[Response]] = None def __repr__(self) -> str: return f"" @@ -200,7 +201,7 @@ def result(self) -> T: :raise arango.exceptions.BatchJobResultError: If job result is not available (i.e. batch is not committed yet). """ - if self._status == "pending" or self._response is None: + if self._status == "pending" or self._future is None or not self._future.done(): raise BatchJobResultError("result not available yet") - return self._response_handler(self._response) + return self._response_handler(self._future.result()) diff --git a/arango/pregel.py b/arango/pregel.py index 81313047..0cde3ce5 100644 --- a/arango/pregel.py +++ b/arango/pregel.py @@ -1,6 +1,6 @@ __all__ = ["Pregel"] -from typing import Optional +from typing import Optional, Sequence from arango.api import ApiGroup from arango.exceptions import ( @@ -8,7 +8,7 @@ PregelJobDeleteError, PregelJobGetError, ) -from arango.formatter import format_pregel_job_data +from arango.formatter import format_pregel_job_data, format_pregel_job_list from arango.request import Request from arango.response import Response from arango.result import Result @@ -49,6 +49,8 @@ def create_job( async_mode: Optional[bool] = None, result_field: Optional[str] = None, algorithm_params: Optional[Json] = None, + vertexCollections: Optional[Sequence[str]] = None, + edgeCollections: Optional[Sequence[str]] = None, ) -> Result[int]: """Start a new Pregel job. @@ -74,12 +76,21 @@ def create_job( :type result_field: str | None :param algorithm_params: Additional algorithm parameters. :type algorithm_params: dict | None + :param vertexCollections: List of vertex collection names. + :type vertexCollections: Sequence[str] | None + :param edgeCollections: List of edge collection names. + :type edgeCollections: Sequence[str] | None :return: Pregel job ID. :rtype: int :raise arango.exceptions.PregelJobCreateError: If create fails. """ data: Json = {"algorithm": algorithm, "graphName": graph} + if vertexCollections is not None: + data["vertexCollections"] = vertexCollections + if edgeCollections is not None: + data["edgeCollections"] = edgeCollections + if algorithm_params is None: algorithm_params = {} @@ -122,3 +133,20 @@ def response_handler(resp: Response) -> bool: raise PregelJobDeleteError(resp, request) return self._execute(request, response_handler) + + def jobs(self) -> Result[Json]: + """Returns a list of currently running and recently + finished Pregel jobs without retrieving their results. + + :return: Details of each running or recently finished Pregel job. + :rtype: dict + :raise arango.exceptions.PregelJobGetError: If retrieval fails. + """ + request = Request(method="get", endpoint="/_api/control_pregel") + + def response_handler(resp: Response) -> Json: + if resp.is_success: + return format_pregel_job_list(resp.body) + raise PregelJobGetError(resp, request) + + return self._execute(request, response_handler) diff --git a/arango/py.typed b/arango/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/arango/replication.py b/arango/replication.py index 0ecfc20e..d5fae457 100644 --- a/arango/replication.py +++ b/arango/replication.py @@ -180,13 +180,15 @@ def response_handler(resp: Response) -> Json: if resp.is_success: result = format_replication_header(resp.headers) result["content"] = [ - [ - self._conn.deserialize(line) - for line in resp.body.split("\n") - if line - ] - if deserialize - else resp.body + ( + [ + self._conn.deserialize(line) + for line in resp.body.split("\n") + if line + ] + if deserialize + else resp.body + ) ] return result diff --git a/arango/request.py b/arango/request.py index fa7a7483..41a0ac66 100644 --- a/arango/request.py +++ b/arango/request.py @@ -2,13 +2,22 @@ from typing import Any, MutableMapping, Optional -from arango.typings import Fields, Headers, Params - - -def normalize_headers(headers: Optional[Headers]) -> Headers: +from arango.typings import DriverFlags, Fields, Headers, Params + + +def normalize_headers( + headers: Optional[Headers], driver_flags: Optional[DriverFlags] = None +) -> Headers: + flags = "" + if driver_flags is not None: + for flag in driver_flags: + flags = flags + flag + ";" + driver_version = "8.1.5" + driver_header = "python-arango/" + driver_version + " (" + flags + ")" normalized_headers: Headers = { "charset": "utf-8", "content-type": "application/json", + "x-arango-driver": driver_header, } if headers is not None: for key, value in headers.items(): @@ -53,6 +62,8 @@ class Request: :type exclusive: str | [str] | None :param deserialize: Whether the response body can be deserialized. :type deserialize: bool + :param driver_flags: List of flags for the driver + :type driver_flags: list :ivar method: HTTP method in lowercase (e.g. "post"). :vartype method: str @@ -74,6 +85,8 @@ class Request: :vartype exclusive: str | [str] | None :ivar deserialize: Whether the response body can be deserialized. :vartype deserialize: bool + :ivar driver_flags: List of flags for the driver + :vartype driver_flags: list """ __slots__ = ( @@ -86,6 +99,7 @@ class Request: "write", "exclusive", "deserialize", + "driver_flags", ) def __init__( @@ -99,13 +113,15 @@ def __init__( write: Optional[Fields] = None, exclusive: Optional[Fields] = None, deserialize: bool = True, + driver_flags: Optional[DriverFlags] = None, ) -> None: self.method = method self.endpoint = endpoint - self.headers: Headers = normalize_headers(headers) + self.headers: Headers = normalize_headers(headers, driver_flags) self.params: MutableMapping[str, str] = normalize_params(params) self.data = data self.read = read self.write = write self.exclusive = exclusive self.deserialize = deserialize + self.driver_flags = driver_flags diff --git a/arango/resolver.py b/arango/resolver.py index 72dfe8bb..714d76c1 100644 --- a/arango/resolver.py +++ b/arango/resolver.py @@ -1,46 +1,126 @@ __all__ = [ "HostResolver", + "FallbackHostResolver", + "PeriodicHostResolver", "SingleHostResolver", "RandomHostResolver", "RoundRobinHostResolver", ] +import logging import random +import time from abc import ABC, abstractmethod +from typing import Optional, Set class HostResolver(ABC): # pragma: no cover """Abstract base class for host resolvers.""" + def __init__(self, host_count: int = 1, max_tries: Optional[int] = None) -> None: + max_tries = max_tries or host_count * 3 + if max_tries < host_count: + raise ValueError("max_tries cannot be less than host_count") + + self._host_count = host_count + self._max_tries = max_tries + @abstractmethod - def get_host_index(self) -> int: + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: raise NotImplementedError + @property + def host_count(self) -> int: + return self._host_count + + @property + def max_tries(self) -> int: + return self._max_tries + class SingleHostResolver(HostResolver): """Single host resolver.""" - def get_host_index(self) -> int: + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: return 0 class RandomHostResolver(HostResolver): """Random host resolver.""" - def __init__(self, host_count: int) -> None: - self._max = host_count - 1 + def __init__(self, host_count: int, max_tries: Optional[int] = None) -> None: + super().__init__(host_count, max_tries) - def get_host_index(self) -> int: - return random.randint(0, self._max) + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + host_index = None + indexes_to_filter = indexes_to_filter or set() + while host_index is None or host_index in indexes_to_filter: + host_index = random.randint(0, self.host_count - 1) + + return host_index class RoundRobinHostResolver(HostResolver): """Round-robin host resolver.""" - def __init__(self, host_count: int) -> None: + def __init__(self, host_count: int, max_tries: Optional[int] = None) -> None: + super().__init__(host_count, max_tries) self._index = -1 - self._count = host_count - def get_host_index(self) -> int: - self._index = (self._index + 1) % self._count + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + self._index = (self._index + 1) % self.host_count + return self._index + + +class PeriodicHostResolver(HostResolver): + """ + Changes the host every N requests. + An optional timeout may be applied between host changes, + such that all coordinators get a chance to update their view of the agency. + For example, if one coordinator creates a database, the others may not be + immediately aware of it. If the timeout is set to 1 second, then the host + resolver waits for 1 second before changing the host. + """ + + def __init__( + self, + host_count: int, + max_tries: Optional[int] = None, + requests_period: int = 100, + switch_timeout: float = 0, + ) -> None: + super().__init__(host_count, max_tries) + self._requests_period = requests_period + self._switch_timeout = switch_timeout + self._req_count = 0 + self._index = 0 + + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + indexes_to_filter = indexes_to_filter or set() + self._req_count = (self._req_count + 1) % self._requests_period + if self._req_count == 0 or self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + while self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + self._req_count = 0 + time.sleep(self._switch_timeout) + return self._index + + +class FallbackHostResolver(HostResolver): + """ + Fallback host resolver. + If the current host fails, the next one is used. + """ + + def __init__(self, host_count: int, max_tries: Optional[int] = None) -> None: + super().__init__(host_count, max_tries) + self._index = 0 + self._logger = logging.getLogger(self.__class__.__name__) + + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + indexes_to_filter = indexes_to_filter or set() + while self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + self._logger.debug(f"Trying fallback on host {self._index}") return self._index diff --git a/arango/typings.py b/arango/typings.py index ed685d99..157802f9 100644 --- a/arango/typings.py +++ b/arango/typings.py @@ -1,9 +1,28 @@ -__all__ = ["Fields", "Headers", "Json", "Jsons", "Params"] +__all__ = [ + "Fields", + "Headers", + "Json", + "Jsons", + "Params", + "PrimitiveDataTypes", + "CompoundDataTypes", + "DataTypes", +] -from typing import Any, Dict, List, MutableMapping, Sequence, Union +from numbers import Number +from typing import Any, Dict, List, MutableMapping, Optional, Sequence, Union Json = Dict[str, Any] Jsons = List[Json] Params = MutableMapping[str, Union[bool, int, str]] Headers = MutableMapping[str, str] Fields = Union[str, Sequence[str]] +DriverFlags = List[str] +PrimitiveDataTypes = Optional[Union[bool, Number, str]] +CompoundDataTypes = Optional[ + Union[ + Sequence[Optional[Union[PrimitiveDataTypes, "CompoundDataTypes"]]], + MutableMapping[str, Optional[Union[PrimitiveDataTypes, "CompoundDataTypes"]]], + ] +] +DataTypes = Optional[Union[PrimitiveDataTypes, CompoundDataTypes]] diff --git a/arango/utils.py b/arango/utils.py index 42d7fff3..89a0eca5 100644 --- a/arango/utils.py +++ b/arango/utils.py @@ -6,12 +6,13 @@ "is_none_or_str", ] +import json import logging from contextlib import contextmanager -from typing import Any, Iterator, Union +from typing import Any, Iterator, Optional, Sequence, Union -from arango.exceptions import DocumentParseError -from arango.typings import Json +from arango.exceptions import DocumentParseError, SortValidationError +from arango.typings import Json, Jsons @contextmanager @@ -66,7 +67,7 @@ def is_none_or_int(obj: Any) -> bool: """Check if obj is None or an integer. :param obj: Object to check. - :type obj: object + :type obj: Any :return: True if object is None or an integer. :rtype: bool """ @@ -77,8 +78,90 @@ def is_none_or_str(obj: Any) -> bool: """Check if obj is None or a string. :param obj: Object to check. - :type obj: object + :type obj: Any :return: True if object is None or a string. :rtype: bool """ return obj is None or isinstance(obj, str) + + +def is_none_or_bool(obj: Any) -> bool: + """Check if obj is None or a bool. + + :param obj: Object to check. + :type obj: Any + :return: True if object is None or a bool. + :rtype: bool + """ + return obj is None or isinstance(obj, bool) + + +def get_batches(elements: Sequence[Json], batch_size: int) -> Iterator[Sequence[Json]]: + """Generator to split a list in batches + of (maximum) **batch_size** elements each. + + :param elements: The list of elements. + :type elements: Sequence[Json] + :param batch_size: Max number of elements per batch. + :type batch_size: int + """ + for index in range(0, len(elements), batch_size): + yield elements[index : index + batch_size] + + +def build_filter_conditions(filters: Json) -> str: + """Build a filter condition for an AQL query. + + :param filters: Document filters. + :type filters: Dict[str, Any] + :return: The complete AQL filter condition. + :rtype: str + """ + if not filters: + return "" + + conditions = [] + for k, v in filters.items(): + field = k if "." in k else f"`{k}`" + conditions.append(f"doc.{field} == {json.dumps(v)}") + + return "FILTER " + " AND ".join(conditions) + + +def validate_sort_parameters(sort: Sequence[Json]) -> bool: + """Validate sort parameters for an AQL query. + + :param sort: Document sort parameters. + :type sort: Sequence[Json] + :return: Validation success. + :rtype: bool + :raise arango.exceptions.SortValidationError: If sort parameters are invalid. + """ + assert isinstance(sort, Sequence) + for param in sort: + if "sort_by" not in param or "sort_order" not in param: + raise SortValidationError( + "Each sort parameter must have 'sort_by' and 'sort_order'." + ) + if param["sort_order"].upper() not in ["ASC", "DESC"]: + raise SortValidationError("'sort_order' must be either 'ASC' or 'DESC'") + return True + + +def build_sort_expression(sort: Optional[Jsons]) -> str: + """Build a sort condition for an AQL query. + + :param sort: Document sort parameters. + :type sort: Jsons | None + :return: The complete AQL sort condition. + :rtype: str + """ + if not sort: + return "" + + sort_chunks = [] + for sort_param in sort: + chunk = f"doc.{sort_param['sort_by']} {sort_param['sort_order']}" + sort_chunks.append(chunk) + + return "SORT " + ", ".join(sort_chunks) diff --git a/arango/version.py b/arango/version.py deleted file mode 100644 index 0895d736..00000000 --- a/arango/version.py +++ /dev/null @@ -1,5 +0,0 @@ -# coding: utf-8 -# file generated by setuptools_scm -# don't change, don't track in version control -version = '6.1.1.dev0+g0e82788.d20210214' -version_tuple = (6, 1, 1, 'dev0+g0e82788', 'd20210214') diff --git a/docs/admin.rst b/docs/admin.rst index 744b44b3..27e29b0b 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -32,9 +32,22 @@ database. # Retrieve the server time. sys_db.time() - # Retrieve the server role in a cluster. + # Retrieve the server role. sys_db.role() + # Retrieve the server role in a cluster. + sys_db.cluster.server_role() + + # Retrieve the server mode. + sys_db.mode() + + # Retrieve the server mode in a cluster. + sys_db.cluster.server_mode() + + # Set the server mode. + sys_db.set_mode('readonly') + sys_db.set_mode('default') + # Retrieve the server statistics. sys_db.statistics() @@ -47,13 +60,16 @@ database. # Set the log . sys_db.set_log_levels( agency='DEBUG', - collector='INFO', + deprecation='INFO', threads='WARNING' ) # Echo the last request. sys_db.echo() + # Echo a request + sys_db.echo('request goes here') + # Reload the routing collection. sys_db.reload_routing() diff --git a/docs/analyzer.rst b/docs/analyzer.rst index 17e70bbd..5821d19a 100644 --- a/docs/analyzer.rst +++ b/docs/analyzer.rst @@ -32,4 +32,4 @@ to `ArangoDB manual`_. # Delete an analyzer. db.delete_analyzer('test_analyzer', ignore_missing=True) -Refer to :ref:`StandardDatabase` class for API specification. \ No newline at end of file +Refer to :ref:`StandardDatabase` class for API specification. diff --git a/docs/async.rst b/docs/async.rst index 82690b29..5e480248 100644 --- a/docs/async.rst +++ b/docs/async.rst @@ -45,8 +45,8 @@ the results can be retrieved once available via :ref:`AsyncJob` objects. # Retrieve the status of each async job. for job in [job1, job2, job3, job4]: - # Job status can be "pending", "done" or "cancelled". - assert job.status() in {'pending', 'done', 'cancelled'} + # Job status can be "pending" or "done". + assert job.status() in {'pending', 'done'} # Let's wait until the jobs are finished. while job.status() != 'done': diff --git a/docs/auth.rst b/docs/auth.rst index a0cd9ac6..11f62985 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -59,7 +59,7 @@ to work correctly. # compensate for out-of-sync clocks between the client and server. db.conn.ext_leeway = 2 -User generated JWT token can be used for superuser access. +User generated JWT token can be used for user and superuser access. **Example:** @@ -89,3 +89,29 @@ User generated JWT token can be used for superuser access. # Connect to "test" database as superuser using the token. db = client.db('test', superuser_token=token) + + # Connect to "test" database as user using the token. + db = client.db('test', user_token=token) + +User and superuser tokens can be set on the connection object as well. + +**Example:** + +.. code-block:: python + + from arango import ArangoClient + + # Initialize the ArangoDB client. + client = ArangoClient() + + # Connect to "test" database as superuser using the token. + db = client.db('test', user_token='token') + + # Set the user token on the connection object. + db.conn.set_token('new token') + + # Connect to "test" database as superuser using the token. + db = client.db('test', superuser_token='superuser token') + + # Set the user token on the connection object. + db.conn.set_token('new superuser token') diff --git a/docs/batch.rst b/docs/batch.rst index a4e3a047..6aa3ed75 100644 --- a/docs/batch.rst +++ b/docs/batch.rst @@ -1,9 +1,30 @@ Batch API Execution ------------------- - -In **batch API executions**, requests to ArangoDB server are stored in client-side -in-memory queue, and committed together in a single HTTP call. After the commit, -results can be retrieved later from :ref:`BatchJob` objects. +.. warning:: + + The batch request API is deprecated since ArangoDB 3.8.0. + We discourage its use, as it will be removed in a future release. + It is already slow and seems to regularly create weird errors when + used with recent versions of ArangoDB. + + The driver functionality has been refactored to no longer use the batch API, + but a `ThreadPoolExecutor` instead. For backwards compatibility, + `max_workers` is set to 1 by default, but can be increased to speed up + batch operations. Essentially, the batch API can now be used to send + multiple requests in parallel, but not to send multiple requests in a + single HTTP call. Note that sending multiple requests in parallel may + cause conflicts on the servers side (for example, requests that modify the same document). + + To send multiple documents at once to an ArangoDB instance, + please use any of :class:`arango.collection.Collection` methods + that accept a list of documents as input, such as: + + * :func:`~arango.collection.Collection.insert_many` + * :func:`~arango.collection.Collection.update_many` + * :func:`~arango.collection.Collection.replace_many` + * :func:`~arango.collection.Collection.delete_many` + +After the commit, results can be retrieved later from :ref:`BatchJob` objects. **Example:** diff --git a/docs/certificates.rst b/docs/certificates.rst new file mode 100644 index 00000000..e6ffedbc --- /dev/null +++ b/docs/certificates.rst @@ -0,0 +1,50 @@ +TLS certificate verification +---------------------------- + +When connecting against a server using an https/TLS connection, TLS certificates +are verified by default. +By default, self-signed certificates will cause trouble when connecting. + +.. code-block:: python + + client = ArangoClient(hosts="https://localhost:8529") + +To make connections work even when using self-signed certificates, you can +provide the certificate CA bundle or turn the verification off. + +If you want to have fine-grained control over the HTTP connection, you should define +your HTTP client as described in the :ref:`HTTPClients` section. + +The ``ArangoClient`` class provides an option to override the verification behavior, +no matter what has been defined in the underlying HTTP session. +You can use this option to disable verification. + +.. code-block:: python + + client = ArangoClient(hosts="https://localhost:8529", verify_override=False) + +This will allow connecting, but the underlying `urllib3` library may still issue +warnings due to the insecurity of using self-signed certificates. + +To turn off these warnings as well, you can add the following code to your client +application: + +.. code-block:: python + + import requests + requests.packages.urllib3.disable_warnings() + +You can also provide a custom CA bundle without defining a custom HTTP Client: + +.. code-block:: python + + client = ArangoClient(hosts="https://localhost:8529", verify_override="path/to/certfile") + +If `verify_override` is set to a path to a directory, the directory must have been processed using the `c_rehash` utility +supplied with OpenSSL. For more information, see the `requests documentation `_. + +Setting `verify_override` to `True` will use the system's default CA bundle. + +.. code-block:: python + + client = ArangoClient(hosts="https://localhost:8529", verify_override=True) diff --git a/docs/cluster.rst b/docs/cluster.rst index 28f80ed9..fdb45bca 100644 --- a/docs/cluster.rst +++ b/docs/cluster.rst @@ -86,9 +86,17 @@ Below is an example on how to manage clusters using python-arango. cluster.server_engine(server_id) cluster.server_version(server_id) cluster.server_statistics(server_id) + cluster.server_maintenance_mode(server_id) - # Toggle maintenance mode (allowed values are "on" and "off"). + # Toggle Server maintenance mode (allowed values are "normal" and "maintenance"). + cluster.toggle_server_maintenance_mode(server_id, 'normal') + cluster.toggle_server_maintenance_mode(server_id, 'maintenance', timeout=30) + + # Toggle Cluster maintenance mode (allowed values are "on" and "off"). cluster.toggle_maintenance_mode('on') cluster.toggle_maintenance_mode('off') + # Rebalance the distribution of shards. Available with ArangoDB 3.10+. + cluster.rebalance() + See :ref:`ArangoClient` and :ref:`Cluster` for API specification. diff --git a/docs/compression.rst b/docs/compression.rst new file mode 100644 index 00000000..526e20f1 --- /dev/null +++ b/docs/compression.rst @@ -0,0 +1,40 @@ +Compression +------------ + +The :ref:`ArangoClient` lets you define the preferred compression policy for request and responses. By default +compression is disabled. You can change this by setting the `request_compression` and `response_compression` parameters +when creating the client. Currently, only the "deflate" compression algorithm is supported. + +.. testcode:: + + from arango import ArangoClient + + from arango.http import DeflateRequestCompression + + client = ArangoClient( + hosts='http://localhost:8529', + request_compression=DeflateRequestCompression(), + response_compression="deflate" + ) + +Furthermore, you can customize the request compression policy by defining the minimum size of the request body that +should be compressed and the desired compression level. For example, the following code sets the minimum size to 2 KB +and the compression level to 8: + +.. code-block:: python + + client = ArangoClient( + hosts='http://localhost:8529', + request_compression=DeflateRequestCompression( + threshold=2048, + level=8), + ) + +If you want to implement your own compression policy, you can do so by implementing the +:class:`arango.http.RequestCompression` interface. + +.. note:: + The `response_compression` parameter is only used to inform the server that the client prefers compressed responses + (in the form of an *Accept-Encoding* header). Note that the server may or may not honor this preference, depending + on how it is configured. This can be controlled by setting the `--http.compress-response-threshold` option to + a value greater than 0 when starting the ArangoDB server. diff --git a/docs/conf.py b/docs/conf.py index cd9bea82..361d1fbe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,10 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("..")) + project = "python-arango" -copyright = "2016-2021, Joohwan Oh" +copyright = "2016-2025, Joohwan Oh" author = "Joohwan Oh" extensions = [ "sphinx_rtd_theme", @@ -12,6 +17,9 @@ html_theme = "sphinx_rtd_theme" master_doc = "index" +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "docs.python-arango.com") + autodoc_member_order = "bysource" doctest_global_setup = """ diff --git a/docs/contributing.rst b/docs/contributing.rst index 18e74ed2..2093f72f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,16 +7,16 @@ Requirements Before submitting a pull request on GitHub_, please make sure you meet the following requirements: -* The pull request points to dev_ branch. +* The pull request points to main_ branch. * Changes are squashed into a single commit. I like to use git rebase for this. * Commit message is in present tense. For example, "Fix bug" is good while "Fixed bug" is not. * Sphinx_-compatible docstrings. * PEP8_ compliance. * No missing docstrings or commented-out lines. -* Test coverage_ remains at %100. If a piece of code is trivial and does not +* Test coverage remains at %100. If a piece of code is trivial and does not need unit tests, use this_ to exclude it from coverage. -* No build failures on `Travis CI`_. Builds automatically trigger on pull +* No build failures. Builds automatically trigger on pull request submissions. * Documentation is kept up-to-date with the new changes (see below). @@ -40,7 +40,7 @@ To ensure PEP8_ compliance, run flake8_: .. code-block:: bash ~$ pip install flake8 - ~$ git clone https://github.com/joowani/python-arango.git + ~$ git clone https://github.com/arangodb/python-arango.git ~$ cd python-arango ~$ flake8 @@ -57,7 +57,7 @@ To run the test suite (use your own host, port and root password): .. code-block:: bash ~$ pip install pytest - ~$ git clone https://github.com/joowani/python-arango.git + ~$ git clone https://github.com/arangodb/python-arango.git ~$ cd python-arango ~$ py.test --complete --host=127.0.0.1 --port=8529 --passwd=passwd @@ -66,7 +66,7 @@ To run the test suite with coverage report: .. code-block:: bash ~$ pip install coverage pytest pytest-cov - ~$ git clone https://github.com/joowani/python-arango.git + ~$ git clone https://github.com/arangodb/python-arango.git ~$ cd python-arango ~$ py.test --complete --host=127.0.0.1 --port=8529 --passwd=passwd --cov=kq @@ -82,18 +82,16 @@ Sphinx_. To build an HTML version on your local machine: .. code-block:: bash ~$ pip install sphinx sphinx_rtd_theme - ~$ git clone https://github.com/joowani/python-arango.git - ~$ cd python-arango/docs - ~$ sphinx-build . build # Open build/index.html in a browser + ~$ git clone https://github.com/arangodb/python-arango.git + ~$ cd python-arango + ~$ python -m sphinx -b html -W docs docs/_build/ # Open build/index.html in a browser As always, thank you for your contribution! -.. _dev: https://github.com/joowani/python-arango/tree/dev -.. _GitHub: https://github.com/joowani/python-arango +.. _main: https://github.com/arangodb/python-arango/tree/main +.. _GitHub: https://github.com/arangodb/python-arango .. _PEP8: https://www.python.org/dev/peps/pep-0008/ -.. _coverage: https://coveralls.io/github/joowani/python-arango .. _this: http://coverage.readthedocs.io/en/latest/excluding.html -.. _Travis CI: https://travis-ci.org/joowani/python-arango .. _Sphinx: https://github.com/sphinx-doc/sphinx .. _flake8: http://flake8.pycqa.org .. _here: http://flake8.pycqa.org/en/latest/user/violations.html#in-line-ignoring-errors diff --git a/docs/cursor.rst b/docs/cursor.rst index 10d12292..9efdddde 100644 --- a/docs/cursor.rst +++ b/docs/cursor.rst @@ -64,7 +64,7 @@ number of items in the result set may or may not be known in advance. cursor.warnings() # Return the next item from the cursor. If current batch is depleted, the - # next batch if fetched from the server automatically. + # next batch is fetched from the server automatically. cursor.next() # Return the next item from the cursor. If current batch is depleted, an @@ -116,3 +116,54 @@ instead. cursor.fetch() while not cursor.empty(): # Pop until nothing is left on the cursor. cursor.pop() + +With ArangoDB 3.11.0 or higher, you can also use the `allow_retry` +parameter of :func:`arango.aql.AQL.execute` to automatically retry +the request if the cursor encountered any issues during the previous +fetch operation. Note that this feature causes the server to cache the +last batch. To allow re-fetching of the very last batch of the query, +the server cannot automatically delete the cursor. Once you have successfully +received the last batch, you should call :func:`arango.cursor.Cursor.close`. + +**Example:** + +.. code-block:: python + + from arango import ArangoClient + + # Initialize the ArangoDB client. + client = ArangoClient() + + # Connect to "test" database as root user. + db = client.db('test', username='root', password='passwd') + + # Set up some test data to query against. + db.collection('students').insert_many([ + {'_key': 'Abby', 'age': 22}, + {'_key': 'John', 'age': 18}, + {'_key': 'Mary', 'age': 21}, + {'_key': 'Suzy', 'age': 23}, + {'_key': 'Dave', 'age': 20} + ]) + + # Execute an AQL query which returns a cursor object. + cursor = db.aql.execute( + 'FOR doc IN students FILTER doc.age > @val RETURN doc', + bind_vars={'val': 17}, + batch_size=2, + count=True, + allow_retry=True + ) + + while cursor.has_more(): + try: + cursor.fetch() + except ConnectionError: + # Retry the request. + continue + + while not cursor.empty(): + cursor.pop() + + # Delete the cursor from the server. + cursor.close() diff --git a/docs/document.rst b/docs/document.rst index 62ad0886..0f0d7d10 100644 --- a/docs/document.rst +++ b/docs/document.rst @@ -103,6 +103,12 @@ Standard documents are managed via collection API wrapper: assert student['GPA'] == 3.6 assert student['last'] == 'Kim' + # Retrieve one or more matching documents, sorted by a field. + for student in students.find({'first': 'John'}, sort=[{'sort_by': 'GPA', 'sort_order': 'DESC'}]): + assert student['_key'] == 'john' + assert student['GPA'] == 3.6 + assert student['last'] == 'Kim' + # Retrieve a document by key. students.get('john') diff --git a/docs/foxx.rst b/docs/foxx.rst index 97c20605..4f6ce35e 100644 --- a/docs/foxx.rst +++ b/docs/foxx.rst @@ -31,7 +31,7 @@ information, refer to `ArangoDB manual`_. # Create a service using source on server. foxx.create_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, development=True, @@ -42,7 +42,7 @@ information, refer to `ArangoDB manual`_. # Update (upgrade) a service. service = db.foxx.update_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, teardown=True, @@ -53,7 +53,7 @@ information, refer to `ArangoDB manual`_. # Replace (overwrite) a service. service = db.foxx.replace_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, teardown=True, diff --git a/docs/graph.rst b/docs/graph.rst index 0fe8b205..0b37154f 100644 --- a/docs/graph.rst +++ b/docs/graph.rst @@ -318,8 +318,10 @@ See :ref:`Graph` and :ref:`EdgeCollection` for API specification. Graph Traversals ================ -**Graph traversals** are executed via the :func:`arango.graph.Graph.traverse` -method. Each traversal can span across multiple vertex collections, and walk +**Graph traversals** are executed via AQL. The old +:func:`arango.graph.Graph.traverse` has been deprecated and can no longer be +used with ArangoDB 3.12 or later. +Each traversal can span across multiple vertex collections, and walk over edges and vertices using various algorithms. **Example:** @@ -371,13 +373,12 @@ over edges and vertices using various algorithms. teach.insert({'_from': 'teachers/jon', '_to': 'lectures/STA201'}) teach.insert({'_from': 'teachers/jon', '_to': 'lectures/MAT223'}) - # Traverse the graph in outbound direction, breath-first. - school.traverse( - start_vertex='teachers/jon', - direction='outbound', - strategy='bfs', - edge_uniqueness='global', - vertex_uniqueness='global', - ) + # AQL to perform a graph traversal + query = """ + FOR v, e, p IN 1..3 OUTBOUND 'teachers/jon' GRAPH 'school' + OPTIONS { bfs: true, uniqueVertices: 'global' } + RETURN {vertex: v, edge: e, path: p} + """ -See :func:`arango.graph.Graph.traverse` for API specification. + # Traverse the graph in outbound direction, breath-first. + cursor = db.aql.execute(query) diff --git a/docs/http.rst b/docs/http.rst index e0330801..44ff1724 100644 --- a/docs/http.rst +++ b/docs/http.rst @@ -1,3 +1,5 @@ +.. _HTTPClients: + HTTP Clients ------------ @@ -58,8 +60,8 @@ Your ``CustomHTTPClient`` class might look something like this: method_whitelist=["HEAD", "GET", "OPTIONS"], ) http_adapter = HTTPAdapter(max_retries=retry_strategy) - session.mount('https://', adapter) - session.mount('http://', adapter) + session.mount('https://', http_adapter) + session.mount('http://', http_adapter) return session diff --git a/docs/index.rst b/docs/index.rst index 8566733d..06955e0d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,53 +10,88 @@ Welcome to the documentation for **python-arango**, a Python driver for ArangoDB Requirements ============= -- ArangoDB version 3.7+ -- Python version 3.6+ +- ArangoDB version 3.11+ +- Python version 3.9+ Installation ============ .. code-block:: bash - ~$ pip install python-arango + ~$ pip install python-arango --upgrade Contents ======== +Basics + .. toctree:: :maxdepth: 1 overview database collection - document - schema indexes + document graph - aql simple - cursor + aql + +Specialized Features + +.. toctree:: + :maxdepth: 1 + + pregel + foxx + replication + transaction + cluster + analyzer + view + wal + +API Executions + +.. toctree:: + :maxdepth: 1 + async batch - transaction + overload + +Administration + +.. toctree:: + :maxdepth: 1 + admin user + +Miscellaneous + +.. toctree:: + :maxdepth: 1 + task - wal - pregel - foxx - view - analyzer threading + certificates errors logging auth http - replication - cluster + compression serializer + schema + cursor backup errno + +Development + +.. toctree:: + :maxdepth: 1 + contributing specs diff --git a/docs/indexes.rst b/docs/indexes.rst index 5705517a..8df3048f 100644 --- a/docs/indexes.rst +++ b/docs/indexes.rst @@ -27,27 +27,37 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to # List the indexes in the collection. cities.indexes() - # Add a new hash index on document fields "continent" and "country". - index = cities.add_hash_index(fields=['continent', 'country'], unique=True) + # Add a new persistent index on document fields "continent" and "country". + persistent_index = {'type': 'persistent', 'fields': ['continent', 'country'], 'unique': True} + index = cities.add_index(persistent_index) # Add new fulltext indexes on fields "continent" and "country". - index = cities.add_fulltext_index(fields=['continent']) - index = cities.add_fulltext_index(fields=['country']) + index = cities.add_index({'type': 'fulltext', 'fields': ['continent']}) + index = cities.add_index({'type': 'fulltext', 'fields': ['country']}) - # Add a new skiplist index on field 'population'. - index = cities.add_skiplist_index(fields=['population'], sparse=False) + # Add a new persistent index on field 'population'. + persistent_index = {'type': 'persistent', 'fields': ['population'], 'sparse': False} + index = cities.add_index(persistent_index) # Add a new geo-spatial index on field 'coordinates'. - index = cities.add_geo_index(fields=['coordinates']) + geo_index = {'type': 'geo', 'fields': ['coordinates']} + index = cities.add_index(geo_index) # Add a new persistent index on field 'currency'. - index = cities.add_persistent_index(fields=['currency'], sparse=True) + persistent_index = {'type': 'persistent', 'fields': ['currency'], 'sparse': True} + index = cities.add_index(persistent_index) # Add a new TTL (time-to-live) index on field 'currency'. - index = cities.add_ttl_index(fields=['ttl'], expiry_time=200) + ttl_index = {'type': 'ttl', 'fields': ['currency'], 'expireAfter': 200} + index = cities.add_index(ttl_index) + + # Add MDI (multi-dimensional) index on field 'x' and 'y'. + mdi_index = {'type': 'mdi', 'fields': ['x', 'y'], 'fieldValueTypes': 'double'} + index = cities.add_index(mdi_index) # Indexes may be added with a name that can be referred to in AQL queries. - index = cities.add_hash_index(fields=['country'], name='my_hash_index') + persistent_index = {'type': 'persistent', 'fields': ['country'], 'unique': True, 'name': 'my_hash_index'} + index = cities.add_index(persistent_index) # Delete the last index from the collection. cities.delete_index(index['id']) diff --git a/docs/overload.rst b/docs/overload.rst new file mode 100644 index 00000000..25724663 --- /dev/null +++ b/docs/overload.rst @@ -0,0 +1,58 @@ +Overload API Execution +---------------------- +:ref:`OverloadControlDatabase` is designed to handle time-bound requests. It allows setting a maximum server-side +queuing time for client requests via the *max_queue_time_seconds* parameter. If the server's queueing time for a +request surpasses this defined limit, the request will be rejected. This mechanism provides you with more control over +request handling, enabling your application to react effectively to potential server overloads. + +Additionally, the response from ArangoDB will always include the most recent request queuing/dequeuing time from the +server's perspective. This can be accessed via the :attr:`~.OverloadControlDatabase.last_queue_time` property. + +**Example:** + +.. testcode:: + + from arango import errno + from arango import ArangoClient + from arango.exceptions import OverloadControlExecutorError + + # Initialize the ArangoDB client. + client = ArangoClient() + + # Connect to "test" database as root user. + db = client.db('test', username='root', password='passwd') + + # Begin controlled execution. + controlled_db = db.begin_controlled_execution(max_queue_time_seconds=7.5) + + # All requests surpassing the specified limit will be rejected. + controlled_aql = controlled_db.aql + controlled_col = controlled_db.collection('students') + + # On API execution, the last_queue_time property gets updated. + controlled_col.insert({'_key': 'Neal'}) + + # Retrieve the last recorded queue time. + assert controlled_db.last_queue_time >= 0 + + try: + controlled_aql.execute('RETURN 100000') + except OverloadControlExecutorError as err: + assert err.http_code == errno.HTTP_PRECONDITION_FAILED + assert err.error_code == errno.QUEUE_TIME_REQUIREMENT_VIOLATED + + # Retrieve the maximum allowed queue time. + assert controlled_db.max_queue_time == 7.5 + + # Adjust the maximum allowed queue time. + controlled_db.adjust_max_queue_time(0.0001) + + # Disable the maximum allowed queue time. + controlled_db.adjust_max_queue_time(None) + +.. note:: + Setting *max_queue_time_seconds* to 0 or a non-numeric value will cause ArangoDB to ignore the header. + +See :ref:`OverloadControlDatabase` for API specification. +See the `official documentation `_ for +details on ArangoDB's overload control options. diff --git a/docs/overview.rst b/docs/overview.rst index 76ff4155..053658df 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -29,8 +29,8 @@ Here is an example showing how **python-arango** client can be used: else: students = db.create_collection('students') - # Add a hash index to the collection. - students.add_hash_index(fields=['name'], unique=False) + # Add a persistent index to the collection. + students.add_index({'type': 'persistent', 'fields': ['name'], 'unique': False}) # Truncate the collection. students.truncate() diff --git a/docs/pregel.rst b/docs/pregel.rst index 45c55f4a..5aad7abe 100644 --- a/docs/pregel.rst +++ b/docs/pregel.rst @@ -1,6 +1,10 @@ Pregel ------ +.. warning:: + Starting from ArangoDB 3.12, the Pregel API has been dropped. + Currently, the driver still supports it for the 3.10 and 3.11 versions, but note that it will be dropped eventually. + Python-arango provides support for **Pregel**, ArangoDB module for distributed iterative graph processing. For more information, refer to `ArangoDB manual`_. @@ -8,7 +12,7 @@ iterative graph processing. For more information, refer to `ArangoDB manual`_. **Example:** -.. testcode:: +.. code-block:: python from arango import ArangoClient diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..7d6d37d6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +requests_toolbelt +importlib_metadata +PyJWT +sphinx_rtd_theme diff --git a/docs/serializer.rst b/docs/serializer.rst index 6b67bf1e..e99871fa 100644 --- a/docs/serializer.rst +++ b/docs/serializer.rst @@ -19,4 +19,4 @@ initialization. They must be callables that take a single argument. deserializer=json.loads ) -See :ref:`ArangoClient` for API specification. \ No newline at end of file +See :ref:`ArangoClient` for API specification. diff --git a/docs/simple.rst b/docs/simple.rst index acbd2532..8f28f634 100644 --- a/docs/simple.rst +++ b/docs/simple.rst @@ -1,8 +1,6 @@ Simple Queries -------------- -.. caution:: There is no option to add a TTL (Time to live) or batch size optimizations to the Simple Queries due to how Arango is handling simple collection HTTP requests. Your request may time out and you'll see a CursorNextError exception. The AQL queries provide full functionality. - Here is an example of using ArangoDB's **simply queries**: .. testcode:: @@ -48,7 +46,6 @@ Here is an example of using ArangoDB's **simply queries**: Here are all simple query (and other utility) methods available: * :func:`arango.collection.Collection.all` -* :func:`arango.collection.Collection.export` * :func:`arango.collection.Collection.find` * :func:`arango.collection.Collection.find_near` * :func:`arango.collection.Collection.find_in_range` diff --git a/docs/specs.rst b/docs/specs.rst index 0e2d3b13..87e1d184 100644 --- a/docs/specs.rst +++ b/docs/specs.rst @@ -103,6 +103,12 @@ DefaultHTTPClient .. autoclass:: arango.http.DefaultHTTPClient :members: +DeflateRequestCompression +========================= + +.. autoclass:: arango.http.DeflateRequestCompression + :members: + .. _EdgeCollection: EdgeCollection @@ -135,6 +141,15 @@ HTTPClient .. autoclass:: arango.http.HTTPClient :members: +.. _OverloadControlDatabase: + +OverloadControlDatabase +======================= + +.. autoclass:: arango.database.OverloadControlDatabase + :inherited-members: + :members: + .. _Pregel: Pregel diff --git a/docs/transaction.rst b/docs/transaction.rst index 84d03cc4..8bdeb18d 100644 --- a/docs/transaction.rst +++ b/docs/transaction.rst @@ -75,6 +75,16 @@ logical unit of work (ACID compliant). assert 'Lily' not in col assert len(col) == 3 # transaction is aborted so txn_col cannot be used + # Fetch an existing transaction. Useful if you have received a Transaction ID + # from some other part of your system or an external system. + original_txn = db.begin_transaction(write='students') + txn_col = original_txn.collection('students') + assert '_rev' in txn_col.insert({'_key': 'Chip'}) + txn_db = db.fetch_transaction(original_txn.transaction_id) + txn_col = txn_db.collection('students') + assert '_rev' in txn_col.insert({'_key': 'Alya'}) + txn_db.abort_transaction() + See :ref:`TransactionDatabase` for API specification. Alternatively, you can use @@ -121,4 +131,4 @@ Javascript code in a transaction. assert result is True assert 'Lucy' in students assert 'Greg' in students - assert 'Dona' not in students \ No newline at end of file + assert 'Dona' not in students diff --git a/docs/user.rst b/docs/user.rst index 9ffd9344..346a025f 100644 --- a/docs/user.rst +++ b/docs/user.rst @@ -93,4 +93,4 @@ these operations can only be performed by admin users via ``_system`` database. collection='students' ) -See :ref:`StandardDatabase` for API specification. \ No newline at end of file +See :ref:`StandardDatabase` for API specification. diff --git a/pyproject.toml b/pyproject.toml index 1d2140e8..8ed943be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,79 @@ [build-system] -requires = [ +requires = ["setuptools>=42", "wheel", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +normalize = true + +[project] +name = "python-arango" +description = "Python Driver for ArangoDB" +authors = [ {name= "Joohwan Oh", email = "joohwan.oh@outlook.com" }] +maintainers = [ + {name = "Joohwan Oh", email = "joohwan.oh@outlook.com"}, + {name = "Alexandru Petenchea", email = "alex.petenchea@gmail.com"}, + {name = "Anthony Mahanna", email = "anthony.mahanna@arangodb.com"} +] +keywords = ["arangodb", "python", "driver"] +readme = "README.md" +dynamic = ["version"] +license = { file = "LICENSE" } +requires-python = ">=3.9" + +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Documentation :: Sphinx", + "Typing :: Typed", +] + +dependencies = [ + "urllib3>=1.26.0", + "requests", + "requests_toolbelt", + "PyJWT", "setuptools>=42", - "setuptools_scm[toml]>=3.4", - "wheel", + "importlib_metadata>=4.7.1", + "packaging>=23.1", ] -build-backend = "setuptools.build_meta" + +[project.optional-dependencies] +dev = [ + "black>=22.3.0", + "flake8>=4.0.1", + "isort>=5.10.1", + "mypy>=0.942", + "mock", + "pre-commit>=2.17.0", + "pytest>=7.1.1", + "pytest-cov>=3.0.0", + "sphinx", + "sphinx_rtd_theme", + "types-requests", + "types-setuptools", +] + +[tool.setuptools.package-data] +"arango" = ["py.typed"] + +[project.urls] +homepage = "https://github.com/arangodb/python-arango" + +[tool.setuptools] +packages = ["arango"] + + +[tool.pytest.ini_options] +addopts = "-s -vv -p no:warnings" +minversion = "6.0" +testpaths = ["tests"] [tool.coverage.run] omit = [ @@ -16,10 +85,8 @@ omit = [ [tool.isort] profile = "black" -[tool.pytest.ini_options] -addopts = "-s -vv -p no:warnings" -minversion = "6.0" -testpaths = ["tests"] - -[tool.setuptools_scm] -write_to = "arango/version.py" +[tool.mypy] +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true +strict = true diff --git a/setup.cfg b/setup.cfg index 894042b5..846ab7fb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,5 @@ [flake8] max-line-length = 88 extend-ignore = E203, E741, W503 -exclude =.git .idea .*_cache dist htmlcov venv +exclude =.git .idea .*_cache dist htmlcov venv arango/errno.py per-file-ignores = __init__.py:F401 - -[mypy] -ignore_missing_imports = True -strict = True diff --git a/setup.py b/setup.py index 8af9bf48..60684932 100644 --- a/setup.py +++ b/setup.py @@ -1,54 +1,3 @@ -from setuptools import find_packages, setup +from setuptools import setup -with open("./README.md") as fp: - long_description = fp.read() - -setup( - name="python-arango", - description="Python Driver for ArangoDB", - long_description=long_description, - long_description_content_type="text/markdown", - author="Joohwan Oh", - author_email="joohwan.oh@outlook.com", - url="https://github.com/joowani/python-arango", - keywords=["arangodb", "python", "driver"], - packages=find_packages(exclude=["tests"]), - include_package_data=True, - python_requires=">=3.6", - license="MIT", - use_scm_version=True, - setup_requires=["setuptools_scm"], - install_requires=[ - "urllib3>=1.26.0", - "dataclasses>=0.6; python_version < '3.7'", - "requests", - "requests_toolbelt", - "PyJWT", - "setuptools>=42", - "setuptools_scm[toml]>=3.4", - ], - extras_require={ - "dev": [ - "black", - "flake8>=3.8.4", - "isort>=5.0.0", - "mypy>=0.790", - "mock", - "pre-commit>=2.9.3", - "pytest>=6.0.0", - "pytest-cov>=2.0.0", - "sphinx", - "sphinx_rtd_theme", - ], - }, - classifiers=[ - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - "Operating System :: Unix", - "Programming Language :: Python :: 3", - "Topic :: Documentation :: Sphinx", - ], -) +setup() diff --git a/starter.sh b/starter.sh new file mode 100755 index 00000000..b4e39f24 --- /dev/null +++ b/starter.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Starts a local ArangoDB server or cluster (community or enterprise). +# Useful for testing the python-arango driver against a local ArangoDB setup. + +# Usage: +# ./starter.sh [single|cluster] [community|enterprise] [version] +# Example: +# ./starter.sh cluster enterprise 3.12.1 + +setup="${1:-single}" +license="${2:-community}" +version="${3:-latest}" + +extra_ports="" +if [ "$setup" == "single" ]; then + echo "" +elif [ "$setup" == "cluster" ]; then + extra_ports="-p 8539:8539 -p 8549:8549" +else + echo "Invalid argument. Please provide either 'single' or 'cluster'." + exit 1 +fi + +image_name="" +if [ "$license" == "community" ]; then + image_name="arangodb" +elif [ "$license" == "enterprise" ]; then + image_name="enterprise" +else + echo "Invalid argument. Please provide either 'community' or 'enterprise'." + exit 1 +fi + +if [ "$version" == "latest" ]; then + conf_file="${setup}-3.12" +elif [[ "$version" == *.*.* ]]; then + conf_file="${setup}-${version%.*}" +else + conf_file="${setup}-${version}" +fi + +docker run -d \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + $extra_ports \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + "arangodb/$image_name:$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/$conf_file.conf" + +wget --quiet --waitretry=1 --tries=120 -O - http://localhost:8528/version | jq +if [ $? -eq 0 ]; then + echo "OK starter ready" + exit 0 +else + echo "ERROR starter not ready, giving up" + exit 1 +fi diff --git a/tests/conftest.py b/tests/conftest.py index cfad2a4c..ee5a0cd3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,13 @@ from dataclasses import dataclass import pytest +from packaging import version from arango import ArangoClient, formatter from arango.database import StandardDatabase +from arango.http import DefaultHTTPClient from arango.typings import Json -from tests.executors import ( - TestAsyncApiExecutor, - TestBatchExecutor, - TestTransactionApiExecutor, -) +from tests.executors import TestAsyncApiExecutor, TestTransactionApiExecutor from tests.helpers import ( empty_collection, generate_col_name, @@ -44,6 +42,7 @@ class GlobalData: enterprise: bool = None secret: str = None root_password: str = None + db_version: version = version.parse("0.0.0") global_data = GlobalData() @@ -51,7 +50,7 @@ class GlobalData: def pytest_addoption(parser): parser.addoption("--host", action="store", default="127.0.0.1") - parser.addoption("--port", action="store", default="8529") + parser.addoption("--port", action="append", default=None) parser.addoption("--passwd", action="store", default="passwd") parser.addoption("--complete", action="store_true") parser.addoption("--cluster", action="store_true") @@ -61,16 +60,29 @@ def pytest_addoption(parser): def pytest_configure(config): - url = f"http://{config.getoption('host')}:{config.getoption('port')}" + ports = config.getoption("port") + if ports is None: + ports = ["8529"] + hosts = [f"http://{config.getoption('host')}:{p}" for p in ports] + url = hosts[0] secret = config.getoption("secret") - client = ArangoClient(hosts=[url, url, url]) + cluster = config.getoption("cluster") + + host_resolver = "fallback" + http_client = DefaultHTTPClient(request_timeout=120) + + client = ArangoClient( + hosts=hosts, host_resolver=host_resolver, http_client=http_client + ) sys_db = client.db( name="_system", username="root", password=config.getoption("passwd"), superuser_token=generate_jwt(secret), + verify=True, ) - sys_db.version() + + db_version = sys_db.version() # Create a user and non-system database for testing. username = generate_username() @@ -94,9 +106,9 @@ def pytest_configure(config): col_name = generate_col_name() tst_col = tst_db.create_collection(col_name, edge=False) - tst_col.add_skiplist_index(["val"]) - tst_col.add_fulltext_index(["text"]) - geo_index = tst_col.add_geo_index(["loc"]) + tst_col.add_index({"type": "skiplist", "fields": ["val"]}) + tst_col.add_index({"type": "fulltext", "fields": ["text"]}) + geo_index = tst_col.add_index({"type": "geo", "fields": ["loc"]}) # Create a legacy edge collection for testing. icol_name = generate_col_name() @@ -122,6 +134,7 @@ def pytest_configure(config): global_data.username = username global_data.password = password global_data.db_name = tst_db_name + global_data.db_version = version.parse(db_version.split("-")[0]) global_data.sys_db = sys_db global_data.tst_db = tst_db global_data.bad_db = bad_db @@ -132,7 +145,7 @@ def pytest_configure(config): global_data.ecol_name = ecol_name global_data.fvcol_name = fvcol_name global_data.tvcol_name = tvcol_name - global_data.cluster = config.getoption("cluster") + global_data.cluster = cluster global_data.complete = config.getoption("complete") global_data.replication = config.getoption("replication") global_data.enterprise = config.getoption("enterprise") @@ -210,13 +223,15 @@ def pytest_generate_tests(metafunc): bad_async_db._executor = TestAsyncApiExecutor(bad_conn) bad_dbs.append(bad_async_db) - # Add test batch databases + # Skip test batch databases, as they are deprecated. + """ tst_batch_db = StandardDatabase(tst_conn) tst_batch_db._executor = TestBatchExecutor(tst_conn) tst_dbs.append(tst_batch_db) bad_batch_bdb = StandardDatabase(bad_conn) bad_batch_bdb._executor = TestBatchExecutor(bad_conn) bad_dbs.append(bad_batch_bdb) + """ if "db" in metafunc.fixturenames and "bad_db" in metafunc.fixturenames: metafunc.parametrize("db,bad_db", zip(tst_dbs, bad_dbs)) @@ -234,6 +249,12 @@ def mock_verify_format(body, result): body.pop("error", None) body.pop("code", None) result.pop("edge", None) + + # Remove all None values + # Sometimes they are expected to be excluded from the body (see computedValues) + result = {k: v for k, v in result.items() if v is not None} + body = {k: v for k, v in body.items() if v is not None} + if len(body) != len(result): before = sorted(body, key=lambda x: x.strip("_")) after = sorted(result, key=lambda x: x.strip("_")) @@ -243,6 +264,11 @@ def mock_verify_format(body, result): monkeypatch.setattr(formatter, "verify_format", mock_verify_format) +@pytest.fixture(autouse=False) +def db_version(): + return global_data.db_version + + @pytest.fixture(autouse=False) def url(): return global_data.url diff --git a/tests/executors.py b/tests/executors.py index 6ba9a11d..7f8a6fa0 100644 --- a/tests/executors.py +++ b/tests/executors.py @@ -30,7 +30,6 @@ def execute(self, request, response_handler): class TestTransactionApiExecutor(TransactionApiExecutor): - # noinspection PyMissingConstructor def __init__(self, connection) -> None: self._conn = connection diff --git a/tests/helpers.py b/tests/helpers.py index 91c0ece3..ef25a786 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -6,7 +6,7 @@ import pytest from arango.cursor import Cursor -from arango.exceptions import AsyncExecuteError, BatchExecuteError, TransactionInitError +from arango.exceptions import AsyncExecuteError, TransactionInitError def generate_db_name(): @@ -180,6 +180,4 @@ def assert_raises(*exc): :param exc: Expected exception(s). :type: exc """ - return pytest.raises( - exc + (AsyncExecuteError, BatchExecuteError, TransactionInitError) - ) + return pytest.raises(exc + (AsyncExecuteError, TransactionInitError)) diff --git a/tests/static/cluster-3.11.conf b/tests/static/cluster-3.11.conf new file mode 100644 index 00000000..86f78556 --- /dev/null +++ b/tests/static/cluster-3.11.conf @@ -0,0 +1,14 @@ +[starter] +mode = cluster +local = true +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.database.extended-names = true +all.log.api-enabled = true +all.javascript.allow-admin-execute = true diff --git a/tests/static/cluster-3.12.conf b/tests/static/cluster-3.12.conf new file mode 100644 index 00000000..d33e07a3 --- /dev/null +++ b/tests/static/cluster-3.12.conf @@ -0,0 +1,15 @@ +[starter] +mode = cluster +local = true +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.database.extended-names = true +all.log.api-enabled = true +all.javascript.allow-admin-execute = true +all.server.options-api = admin diff --git a/tests/static/keyfile b/tests/static/keyfile index 536aca34..d97c5ead 100644 --- a/tests/static/keyfile +++ b/tests/static/keyfile @@ -1 +1 @@ -secret \ No newline at end of file +secret diff --git a/tests/static/setup.sh b/tests/static/setup.sh new file mode 100644 index 00000000..0d2189ba --- /dev/null +++ b/tests/static/setup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +mkdir -p /tests/static +wget -O /tests/static/service.zip "http://localhost:8000/$PROJECT/tests/static/service.zip" +wget -O /tests/static/keyfile "http://localhost:8000/$PROJECT/tests/static/keyfile" +wget -O /tests/static/arangodb.conf "http://localhost:8000/$PROJECT/tests/static/$ARANGODB_CONF" +arangodb --configuration=/tests/static/arangodb.conf diff --git a/tests/static/single-3.11.conf b/tests/static/single-3.11.conf new file mode 100644 index 00000000..df45cb76 --- /dev/null +++ b/tests/static/single-3.11.conf @@ -0,0 +1,12 @@ +[starter] +mode = single +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.database.extended-names = true +all.javascript.allow-admin-execute = true diff --git a/tests/static/single-3.12.conf b/tests/static/single-3.12.conf new file mode 100644 index 00000000..d5df3aa9 --- /dev/null +++ b/tests/static/single-3.12.conf @@ -0,0 +1,13 @@ +[starter] +mode = single +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.database.extended-names = true +all.javascript.allow-admin-execute = true +all.server.options-api = admin diff --git a/tests/test_analyzer.py b/tests/test_analyzer.py index af4b91cb..a7573bd6 100644 --- a/tests/test_analyzer.py +++ b/tests/test_analyzer.py @@ -1,3 +1,5 @@ +from packaging import version + from arango.exceptions import ( AnalyzerCreateError, AnalyzerDeleteError, @@ -7,18 +9,28 @@ from tests.helpers import assert_raises, generate_analyzer_name -def test_analyzer_management(db, bad_db, cluster): +def test_analyzer_management(db, bad_db, cluster, enterprise, db_version): analyzer_name = generate_analyzer_name() full_analyzer_name = db.name + "::" + analyzer_name bad_analyzer_name = generate_analyzer_name() - # Test create analyzer + # Test create identity analyzer result = db.create_analyzer(analyzer_name, "identity", {}) assert result["name"] == full_analyzer_name assert result["type"] == "identity" assert result["properties"] == {} assert result["features"] == [] + # Test create delimiter analyzer + result = db.create_analyzer( + name=generate_analyzer_name(), + analyzer_type="delimiter", + properties={"delimiter": ","}, + ) + assert result["type"] == "delimiter" + assert result["properties"] == {"delimiter": ","} + assert result["features"] == [] + # Test create duplicate with bad database with assert_raises(AnalyzerCreateError) as err: bad_db.create_analyzer(analyzer_name, "identity", {}, []) @@ -56,3 +68,35 @@ def test_analyzer_management(db, bad_db, cluster): # Test delete missing analyzer with ignore_missing set to True assert db.delete_analyzer(analyzer_name, ignore_missing=True) is False + + # Test create geo_s2 analyzer (EE only) + if enterprise: + analyzer_name = generate_analyzer_name() + result = db.create_analyzer(analyzer_name, "geo_s2", {}) + assert result["type"] == "geo_s2" + assert result["features"] == [] + assert result["properties"] == { + "options": {"maxCells": 20, "minLevel": 4, "maxLevel": 23}, + "type": "shape", + "format": "latLngDouble", + } + assert db.delete_analyzer(analyzer_name) + + # Test create delimieter analyzer with multiple delimiters + if db_version >= version.parse("3.12.0"): + result = db.create_analyzer( + name=generate_analyzer_name(), + analyzer_type="multi_delimiter", + properties={"delimiters": [",", "."]}, + ) + + assert result["type"] == "multi_delimiter" + assert result["properties"] == {"delimiters": [",", "."]} + assert result["features"] == [] + + if db_version >= version.parse("3.12.0"): + analyzer_name = generate_analyzer_name() + result = db.create_analyzer(analyzer_name, "wildcard", {"ngramSize": 4}) + assert result["type"] == "wildcard" + assert result["features"] == [] + assert result["properties"] == {"ngramSize": 4} diff --git a/tests/test_aql.py b/tests/test_aql.py index 9f6d1c8e..6c1d3ea3 100644 --- a/tests/test_aql.py +++ b/tests/test_aql.py @@ -1,3 +1,7 @@ +import pytest +from packaging import version + +from arango.errno import FORBIDDEN from arango.exceptions import ( AQLCacheClearError, AQLCacheConfigureError, @@ -15,7 +19,7 @@ AQLQueryTrackingSetError, AQLQueryValidateError, ) -from tests.helpers import assert_raises, extract +from tests.helpers import assert_raises, extract, generate_col_name def test_aql_attributes(db, username): @@ -26,37 +30,60 @@ def test_aql_attributes(db, username): assert repr(db.aql.cache) == f"" -def test_aql_query_management(db, bad_db, col, docs): - plan_fields = [ +def test_aql_query_management(db_version, db, bad_db, col, docs): + explain_fields = [ "estimatedNrItems", "estimatedCost", "rules", "variables", "collections", + "stats", ] + stats_fields = { + "0.0.0": [ + "rulesExecuted", + "rulesSkipped", + "plansCreated", + ], + "3.10.4": [ + "peakMemoryUsage", + "executionTime", + ], + } + # Test explain invalid query with assert_raises(AQLQueryExplainError) as err: db.aql.explain("INVALID QUERY") assert err.value.error_code == 1501 # Test explain valid query with all_plans set to False - plan = db.aql.explain( + explain = db.aql.explain( f"FOR d IN {col.name} RETURN d", all_plans=False, opt_rules=["-all", "+use-index-range"], ) - assert all(field in plan for field in plan_fields) + assert all(field in explain for field in explain_fields) + for v, fields in stats_fields.items(): + if db_version >= version.parse(v): + assert all(field in explain["stats"] for field in fields) + else: + assert all(field not in explain["stats"] for field in fields) # Test explain valid query with all_plans set to True - plans = db.aql.explain( + explanations = db.aql.explain( f"FOR d IN {col.name} RETURN d", all_plans=True, opt_rules=["-all", "+use-index-range"], max_plans=10, ) - for plan in plans: - assert all(field in plan for field in plan_fields) - assert len(plans) < 10 + for explain in explanations: + assert all(field in explain for field in explain_fields) + for v, fields in stats_fields.items(): + if db_version >= version.parse(v): + assert all(field in explain["stats"] for field in fields) + else: + assert all(field not in explain["stats"] for field in fields) + assert len(explanations) < 10 # Test validate invalid query with assert_raises(AQLQueryValidateError) as err: @@ -161,7 +188,7 @@ def test_aql_query_management(db, bad_db, col, docs): assert new_tracking["track_bind_vars"] is True assert new_tracking["track_slow_queries"] is True - # Kick off some long lasting queries in the background + # Kick off some long-lasting queries in the background db.begin_async_execution().aql.execute("RETURN SLEEP(100)") db.begin_async_execution().aql.execute("RETURN SLEEP(50)") @@ -174,6 +201,7 @@ def test_aql_query_management(db, bad_db, col, docs): assert "state" in query assert "bind_vars" in query assert "runtime" in query + assert "peak_memory_usage" in query assert len(queries) == 2 # Test list queries with bad database @@ -219,6 +247,36 @@ def test_aql_query_management(db, bad_db, col, docs): assert err.value.error_code in {11, 1228} +def test_aql_query_force_one_shard_attribute_value(db, db_version, enterprise, cluster): + if not enterprise or not cluster: + return + + name = generate_col_name() + col = db.create_collection(name, shard_fields=["foo"], shard_count=3) + + doc = {"foo": "bar"} + col.insert(doc) + + cursor = db.aql.execute( + "FOR d IN @@c RETURN d", + bind_vars={"@c": name}, + force_one_shard_attribute_value="bar", + ) + + results = [doc for doc in cursor] + assert len(results) == 1 + assert results[0]["foo"] == "bar" + + cursor = db.aql.execute( + "FOR d IN @@c RETURN d", + bind_vars={"@c": name}, + force_one_shard_attribute_value="ooo", + ) + + results = [doc for doc in cursor] + assert len(results) == 0 + + def test_aql_function_management(db, bad_db): fn_group = "functions::temperature" fn_name_1 = "functions::temperature::celsius_to_fahrenheit" @@ -290,6 +348,86 @@ def test_aql_function_management(db, bad_db): assert db.aql.functions() == [] +def test_cache_results_management(db, bad_db, col, docs, cluster): + if cluster: + pytest.skip("Cluster mode does not support query result cache management") + + aql = db.aql + cache = aql.cache + + # Sanity check, just see if the response is OK. + _ = cache.properties() + with pytest.raises(AQLCachePropertiesError) as err: + _ = bad_db.aql.cache.properties() + assert err.value.error_code == FORBIDDEN + + # Turn on caching + result = cache.configure(mode="on") + assert result["mode"] == "on" + result = cache.properties() + assert result["mode"] == "on" + with pytest.raises(AQLCacheConfigureError) as err: + _ = bad_db.aql.cache.configure(mode="on") + assert err.value.error_code == FORBIDDEN + + # Run a simple query to use the cache + col.insert(docs[0]) + _ = aql.execute( + query="FOR doc IN @@collection RETURN doc", + bind_vars={"@collection": col.name}, + cache=True, + ) + + # Check the entries + entries = cache.entries() + assert isinstance(entries, list) + assert len(entries) > 0 + + with pytest.raises(AQLCacheEntriesError) as err: + _ = bad_db.aql.cache.entries() + assert err.value.error_code == FORBIDDEN + + # Clear the cache + cache.clear() + entries = cache.entries() + assert len(entries) == 0 + with pytest.raises(AQLCacheClearError) as err: + bad_db.aql.cache.clear() + assert err.value.error_code == FORBIDDEN + + +def test_cache_plan_management(db, bad_db, col, docs, db_version): + if db_version < version.parse("3.12.4"): + pytest.skip("Query plan cache is supported in ArangoDB 3.12.4+") + + aql = db.aql + cache = aql.cache + + # Run a simple query to use the cache + col.insert(docs[0]) + _ = aql.execute( + query="FOR doc IN @@collection RETURN doc", + bind_vars={"@collection": col.name}, + use_plan_cache=True, + ) + + # Check the entries + entries = cache.plan_entries() + assert isinstance(entries, list) + assert len(entries) > 0 + with pytest.raises(AQLCacheEntriesError) as err: + _ = bad_db.aql.cache.plan_entries() + assert err.value.error_code == FORBIDDEN + + # Clear the cache + cache.clear_plan() + entries = cache.plan_entries() + assert len(entries) == 0 + with pytest.raises(AQLCacheClearError) as err: + bad_db.aql.cache.clear_plan() + assert err.value.error_code == FORBIDDEN + + def test_aql_cache_management(db, bad_db): # Test get AQL cache properties properties = db.aql.cache.properties() diff --git a/tests/test_async.py b/tests/test_async.py index 61335777..0b7ca0a6 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -16,6 +16,12 @@ from tests.helpers import extract +@pytest.fixture(autouse=True) +def teardown(db): + yield + db.clear_async_jobs() + + def wait_on_job(job): """Block until the async job is done.""" while job.status() != "done": @@ -65,7 +71,10 @@ def test_async_execute_without_result(db, col, docs): assert async_col.insert(docs[2]) is None # Ensure that the operations went through - wait_on_jobs(db) + for _ in range(10): + if col.count() == 3: + break + time.sleep(0.5) assert extract("_key", col.all()) == ["1", "2", "3"] @@ -242,7 +251,7 @@ def test_async_list_jobs(db, col, docs): assert job3.id in job_ids # Test list async jobs that are pending - job4 = async_db.aql.execute("RETURN SLEEP(0.3)") + job4 = async_db.aql.execute("RETURN SLEEP(3)") assert db.async_jobs(status="pending") == [job4.id] wait_on_job(job4) # Make sure the job is done diff --git a/tests/test_auth.py b/tests/test_auth.py index 9688799a..0f747563 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -2,8 +2,10 @@ from arango.errno import FORBIDDEN, HTTP_UNAUTHORIZED from arango.exceptions import ( JWTAuthError, + JWTExpiredError, JWTSecretListError, JWTSecretReloadError, + ServerConnectionError, ServerEncryptionError, ServerTLSError, ServerTLSReloadError, @@ -37,7 +39,8 @@ def test_auth_basic(client, db_name, username, password): assert isinstance(db.properties(), dict) -def test_auth_jwt(client, db_name, username, password): +def test_auth_jwt(client, db_name, username, password, secret): + # Test JWT authentication with username and password. db = client.db( name=db_name, username=username, @@ -54,6 +57,13 @@ def test_auth_jwt(client, db_name, username, password): client.db(db_name, username, bad_password, auth_method="jwt") assert err.value.error_code == HTTP_UNAUTHORIZED + # Test JWT authentication with user token. + token = generate_jwt(secret) + db = client.db("_system", user_token=token) + assert isinstance(db.conn, JwtConnection) + assert isinstance(db.version(), str) + assert isinstance(db.properties(), dict) + # TODO re-examine commented out code def test_auth_superuser_token(client, db_name, root_password, secret): @@ -116,13 +126,32 @@ def test_auth_superuser_token(client, db_name, root_password, secret): def test_auth_jwt_expiry(client, db_name, root_password, secret): # Test automatic token refresh on expired token. db = client.db("_system", "root", root_password, auth_method="jwt") + valid_token = generate_jwt(secret) expired_token = generate_jwt(secret, exp=-1000) db.conn._token = expired_token db.conn._auth_header = f"bearer {expired_token}" assert isinstance(db.version(), str) - # Test correct error on token expiry. + # Test expiry error on db instantiation (superuser) + with assert_raises(ServerConnectionError) as err: + client.db("_system", superuser_token=expired_token, verify=True) + + # Test expiry error on db version (superuser) db = client.db("_system", superuser_token=expired_token) with assert_raises(ServerVersionError) as err: db.version() assert err.value.error_code == FORBIDDEN + + # Test expiry error on set_token (superuser). + db = client.db("_system", superuser_token=valid_token) + with assert_raises(JWTExpiredError) as err: + db.conn.set_token(expired_token) + + # Test expiry error on db instantiation (user) + with assert_raises(JWTExpiredError) as err: + db = client.db("_system", user_token=expired_token) + + # Test expiry error on set_token (user). + db = client.db("_system", user_token=valid_token) + with assert_raises(JWTExpiredError) as err: + db.conn.set_token(expired_token) diff --git a/tests/test_backup.py b/tests/test_backup.py index 23ea563c..e030c5e6 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -1,6 +1,6 @@ import pytest -from arango.errno import DATABASE_NOT_FOUND, FILE_NOT_FOUND, FORBIDDEN +from arango.errno import DATABASE_NOT_FOUND, FILE_NOT_FOUND, FORBIDDEN, HTTP_NOT_FOUND from arango.exceptions import ( BackupCreateError, BackupDeleteError, @@ -12,7 +12,7 @@ from tests.helpers import assert_raises -def test_backup_management(sys_db, bad_db, enterprise): +def test_backup_management(sys_db, bad_db, enterprise, cluster): if not enterprise: pytest.skip("Only for ArangoDB enterprise edition") @@ -59,8 +59,9 @@ def test_backup_management(sys_db, bad_db, enterprise): assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} # Test upload backup. + backup_id = backup_id_foo if cluster else backup_id_bar result = sys_db.backup.upload( - backup_id=backup_id_foo, + backup_id=backup_id, repository="local://tmp/backups", config={"local": {"type": "local"}}, ) @@ -79,7 +80,7 @@ def test_backup_management(sys_db, bad_db, enterprise): # Test download backup. result = sys_db.backup.download( - backup_id=backup_id_bar, + backup_id=backup_id_foo, repository="local://tmp/backups", config={"local": {"type": "local"}}, ) @@ -112,4 +113,7 @@ def test_backup_management(sys_db, bad_db, enterprise): # Test delete missing backup. with assert_raises(BackupDeleteError) as err: sys_db.backup.delete(backup_id_foo) - assert err.value.error_code == FILE_NOT_FOUND + if cluster: + assert err.value.error_code == HTTP_NOT_FOUND + else: + assert err.value.error_code == FILE_NOT_FOUND diff --git a/tests/test_batch.py b/tests/test_batch.py index 990adb98..a2267502 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -1,15 +1,7 @@ -import json - -import mock import pytest from arango.database import BatchDatabase -from arango.exceptions import ( - BatchExecuteError, - BatchJobResultError, - BatchStateError, - DocumentInsertError, -) +from arango.exceptions import BatchJobResultError, BatchStateError, DocumentInsertError from arango.job import BatchJob from tests.helpers import clean_doc, extract @@ -74,10 +66,11 @@ def test_batch_execute_with_result(db, col, docs): # Test successful results assert job1.result()["_key"] == docs[0]["_key"] - assert job2.result()["_key"] == docs[1]["_key"] # Test insert error result + # job2 and job3 are concurrent, either one can fail with pytest.raises(DocumentInsertError) as err: + job2.result() job3.result() assert err.value.error_code == 1210 @@ -122,13 +115,9 @@ def test_batch_action_after_commit(db, col): def test_batch_execute_error(bad_db, col, docs): batch_db = bad_db.begin_batch_execution(return_result=True) job = batch_db.collection(col.name).insert_many(docs) - - # Test batch execute with bad database - with pytest.raises(BatchExecuteError) as err: - batch_db.commit() - assert err.value.error_code in {11, 1228} + batch_db.commit() assert len(col) == 0 - assert job.status() == "pending" + assert job.status() == "done" def test_batch_job_result_not_ready(db, col, docs): @@ -144,28 +133,3 @@ def test_batch_job_result_not_ready(db, col, docs): assert batch_db.commit() == [job] assert len(job.result()) == len(docs) assert extract("_key", col.all()) == extract("_key", docs) - - -def test_batch_bad_state(db, col, docs): - batch_db = db.begin_batch_execution() - batch_col = batch_db.collection(col.name) - batch_col.insert(docs[0]) - batch_col.insert(docs[1]) - batch_col.insert(docs[2]) - - # Monkey patch the connection object - mock_resp = mock.MagicMock() - mock_resp.is_success = True - mock_resp.raw_body = "" - mock_send_request = mock.MagicMock() - mock_send_request.return_value = mock_resp - mock_connection = mock.MagicMock() - mock_connection.send_request = mock_send_request - mock_connection.serialize = json.dumps - mock_connection.deserialize = json.loads - batch_db._executor._conn = mock_connection - - # Test commit with invalid batch state - with pytest.raises(BatchStateError) as err: - batch_db.commit() - assert "expecting 3 parts in batch response but got 0" in str(err.value) diff --git a/tests/test_client.py b/tests/test_client.py index 042779ff..a196a8fd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,25 +1,29 @@ import json +import pickle +from typing import Union +import importlib_metadata import pytest -from pkg_resources import get_distribution +from requests import Session from arango.client import ArangoClient from arango.database import StandardDatabase -from arango.exceptions import ServerConnectionError -from arango.http import DefaultHTTPClient -from arango.resolver import ( - RandomHostResolver, - RoundRobinHostResolver, - SingleHostResolver, +from arango.exceptions import ArangoClientError, ServerConnectionError +from arango.http import DefaultHTTPClient, DeflateRequestCompression +from arango.resolver import FallbackHostResolver, RandomHostResolver, SingleHostResolver +from tests.helpers import ( + generate_col_name, + generate_db_name, + generate_string, + generate_username, ) -from tests.helpers import generate_db_name, generate_string, generate_username def test_client_attributes(): http_client = DefaultHTTPClient() client = ArangoClient(hosts="http://127.0.0.1:8529", http_client=http_client) - assert client.version == get_distribution("python-arango").version + assert client.version == importlib_metadata.version("python-arango") assert client.hosts == ["http://127.0.0.1:8529"] assert repr(client) == "" @@ -34,10 +38,10 @@ def test_client_attributes(): serializer=json.dumps, deserializer=json.loads, ) - assert client.version == get_distribution("python-arango").version + assert client.version == importlib_metadata.version("python-arango") assert client.hosts == client_hosts assert repr(client) == client_repr - assert isinstance(client._host_resolver, RoundRobinHostResolver) + assert isinstance(client._host_resolver, FallbackHostResolver) client = ArangoClient( hosts=client_hosts, @@ -46,11 +50,14 @@ def test_client_attributes(): serializer=json.dumps, deserializer=json.loads, ) - assert client.version == get_distribution("python-arango").version + assert client.version == importlib_metadata.version("python-arango") assert client.hosts == client_hosts assert repr(client) == client_repr assert isinstance(client._host_resolver, RandomHostResolver) + client = ArangoClient(hosts=client_hosts, request_timeout=120) + assert client.request_timeout == client._http.request_timeout == 120 + def test_client_good_connection(db, username, password): client = ArangoClient(hosts="http://127.0.0.1:8529") @@ -82,13 +89,29 @@ def test_client_bad_connection(db, username, password, cluster): # Test connection with invalid host URL client = ArangoClient(hosts="http://127.0.0.1:8500") - with pytest.raises(ServerConnectionError) as err: + with pytest.raises(ArangoClientError) as err: client.db(db.name, username, password, verify=True) assert "bad connection" in str(err.value) -def test_client_custom_http_client(db, username, password): +def test_client_http_client_attributes(db, username, password): + http_client = DefaultHTTPClient( + request_timeout=80, + retry_attempts=5, + backoff_factor=1.0, + pool_connections=16, + pool_maxsize=12, + pool_timeout=120, + ) + client = ArangoClient( + hosts="http://127.0.0.1:8529", http_client=http_client, request_timeout=30 + ) + client.db(db.name, username, password, verify=True) + assert http_client.request_timeout == 80 + assert client.request_timeout == http_client.request_timeout + +def test_client_custom_http_client(db, username, password): # Define custom HTTP client which increments the counter on any API call. class MyHTTPClient(DefaultHTTPClient): def __init__(self) -> None: @@ -108,3 +131,114 @@ def send_request( # Set verify to True to send a test API call on initialization. client.db(db.name, username, password, verify=True) assert http_client.counter == 1 + + +def test_client_override_validate() -> None: + # custom http client that manipulates the underlying session + class MyHTTPClient(DefaultHTTPClient): + def __init__(self, verify: Union[None, bool, str]) -> None: + super().__init__() + self.verify = verify + + def create_session(self, host: str) -> Session: + session = super().create_session(host) + session.verify = self.verify + return session + + def assert_verify( + http_client_verify: Union[None, str, bool], + arango_override: Union[None, str, bool], + expected_result: Union[None, str, bool], + ) -> None: + http_client = MyHTTPClient(verify=http_client_verify) + client = ArangoClient( + hosts="http://127.0.0.1:8529", + http_client=http_client, + verify_override=arango_override, + ) + # make sure there is at least 1 session + assert len(client._sessions) > 0 + for session in client._sessions: + # make sure the final session.verify has expected value + assert session.verify == expected_result + + assert_verify(None, None, None) + assert_verify(None, True, True) + assert_verify(None, False, False) + assert_verify(None, "test", "test") + + assert_verify(True, None, True) + assert_verify(True, True, True) + assert_verify(True, "test", "test") + assert_verify(True, False, False) + + assert_verify(False, None, False) + assert_verify(False, True, True) + assert_verify(False, "test", "test") + assert_verify(False, False, False) + + assert_verify("test", None, "test") + assert_verify("test", True, True) + assert_verify("test", False, False) + assert_verify("test", False, False) + assert_verify("test", "foo", "foo") + + +def test_can_serialize_deserialize_client() -> None: + client = ArangoClient(hosts="http://127.0.0.1:8529") + client_pstr = pickle.dumps(client) + client2 = pickle.loads(client_pstr) + assert len(client2._sessions) > 0 + + +def test_client_compression(db, username, password): + class CheckCompression: + def __init__(self, should_compress: bool): + self.should_compress = should_compress + + def check(self, headers): + if self.should_compress: + assert headers["content-encoding"] == "deflate" + else: + assert "content-encoding" not in headers + + class MyHTTPClient(DefaultHTTPClient): + def __init__(self, compression_checker: CheckCompression) -> None: + super().__init__() + self.checker = compression_checker + + def send_request( + self, session, method, url, headers=None, params=None, data=None, auth=None + ): + self.checker.check(headers) + return super().send_request( + session, method, url, headers, params, data, auth + ) + + checker = CheckCompression(should_compress=False) + + # should not compress, as threshold is 0 + client = ArangoClient( + hosts="http://127.0.0.1:8529", + http_client=MyHTTPClient(compression_checker=checker), + response_compression="gzip", + ) + db = client.db(db.name, username, password) + col = db.create_collection(generate_col_name()) + col.insert({"_key": "1"}) + + # should not compress, as size of payload is less than threshold + checker = CheckCompression(should_compress=False) + client = ArangoClient( + hosts="http://127.0.0.1:8529", + http_client=MyHTTPClient(compression_checker=checker), + request_compression=DeflateRequestCompression(250, level=7), + response_compression="deflate", + ) + db = client.db(db.name, username, password) + col = db.create_collection(generate_col_name()) + col.insert({"_key": "2"}) + + # should compress + checker.should_compress = True + col.insert({"_key": "3" * 250}) diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 3534895a..0a4cd19f 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,16 +1,23 @@ +import time +import warnings + import pytest +from packaging import version from arango.errno import DATABASE_NOT_FOUND, FORBIDDEN from arango.exceptions import ( ClusterEndpointsError, ClusterHealthError, ClusterMaintenanceModeError, + ClusterRebalanceError, ClusterServerCountError, ClusterServerEngineError, ClusterServerIDError, + ClusterServerModeError, ClusterServerRoleError, ClusterServerStatisticsError, ClusterServerVersionError, + ClusterVpackSortMigrationError, ) from tests.helpers import assert_raises @@ -39,6 +46,18 @@ def test_cluster_server_role(sys_db, bad_db, cluster): assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} +def test_cluster_server_mode(sys_db, bad_db, cluster): + if not cluster: + pytest.skip("Only tested in a cluster setup") + + result = sys_db.cluster.server_mode() + assert result == "default" + + with assert_raises(ClusterServerModeError) as err: + bad_db.cluster.server_mode() + assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} + + def test_cluster_health(sys_db, bad_db, cluster): if not cluster: pytest.skip("Only tested in a cluster setup") @@ -95,6 +114,37 @@ def test_cluster_server_statistics(sys_db, bad_db, cluster): assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} +def test_cluster_server_maintenance_mode(sys_db, bad_db, cluster): + if not cluster: + pytest.skip("Only tested in a cluster setup") + + # Must be a DBServer + health = sys_db.cluster.health() + server_id = None + for server_id, info in health["Health"].items(): + if info["Role"] == "DBServer": + server_id = server_id + break + if server_id is None: + pytest.skip("No DBServer found in cluster") + + result = sys_db.cluster.server_maintenance_mode(server_id) + assert result == {} + + with assert_raises(ClusterMaintenanceModeError) as err: + bad_db.cluster.server_maintenance_mode(server_id) + assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} + + sys_db.cluster.toggle_server_maintenance_mode(server_id, "maintenance", timeout=2) + result = sys_db.cluster.server_maintenance_mode(server_id) + assert "Mode" in result + assert "Until" in result + + time.sleep(5) + result = sys_db.cluster.server_maintenance_mode(server_id) + assert result == {} + + def test_cluster_toggle_maintenance_mode(sys_db, bad_db, cluster): if not cluster: pytest.skip("Only tested in a cluster setup") @@ -134,3 +184,78 @@ def test_cluster_server_count(db, bad_db, cluster): with assert_raises(ClusterServerCountError) as err: bad_db.cluster.server_count() assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} + + +def test_cluster_rebalance(sys_db, bad_db, cluster): + if not cluster: + pytest.skip("Only tested in a cluster setup") + + # Test imbalance retrieval + imbalance = sys_db.cluster.calculate_imbalance() + assert "leader" in imbalance + assert "shards" in imbalance + assert imbalance["pendingMoveShards"] == 0 + assert imbalance["todoMoveShards"] == 0 + + with assert_raises(ClusterRebalanceError) as err: + bad_db.cluster.calculate_imbalance() + assert err.value.error_code == FORBIDDEN + + # Test rebalance computation + rebalance = sys_db.cluster.calculate_rebalance_plan( + max_moves=3, + leader_changes=True, + move_leaders=True, + move_followers=True, + pi_factor=1234.5, + databases_excluded=["_system"], + ) + assert "imbalanceBefore" in rebalance + assert "imbalanceAfter" in rebalance + assert "moves" in rebalance + + with assert_raises(ClusterRebalanceError) as err: + bad_db.cluster.calculate_rebalance_plan() + assert err.value.error_code == FORBIDDEN + + # Test rebalance execution + if sys_db.cluster.execute_rebalance_plan(rebalance["moves"]) is False: + warnings.warn( + "Rebalance plan was not executed." + "This may happen independent of the driver." + ) + with assert_raises(ClusterRebalanceError) as err: + bad_db.cluster.execute_rebalance_plan(rebalance["moves"]) + assert err.value.error_code == FORBIDDEN + + # Rebalance cluster in one go + rebalance = sys_db.cluster.rebalance() + assert "imbalanceBefore" in rebalance + assert "imbalanceAfter" in rebalance + assert "moves" in rebalance + + with assert_raises(ClusterRebalanceError) as err: + bad_db.cluster.rebalance() + assert err.value.error_code == FORBIDDEN + + +def test_vpack_sort_migration(sys_db, bad_db, db_version, cluster): + if not cluster: + pytest.skip("Only tested in a cluster setup") + if db_version < version.parse("3.12.2"): + pytest.skip("vpackSortMigration is only tested in 3.12.2+") + + sys_db.cluster.vpack_sort_migration_status() + with assert_raises(ClusterVpackSortMigrationError) as err: + bad_db.cluster.vpack_sort_migration_status() + assert err.value.error_code == FORBIDDEN + + sys_db.cluster.vpack_sort_migration_index_check() + with assert_raises(ClusterVpackSortMigrationError) as err: + bad_db.cluster.vpack_sort_migration_index_check() + assert err.value.error_code == FORBIDDEN + + sys_db.cluster.migrate_vpack_sorting() + with assert_raises(ClusterVpackSortMigrationError) as err: + bad_db.cluster.migrate_vpack_sorting() + assert err.value.error_code == FORBIDDEN diff --git a/tests/test_collection.py b/tests/test_collection.py index 4f081688..7ab72800 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -1,3 +1,6 @@ +import pytest + +from arango.client import ArangoClient from arango.collection import StandardCollection from arango.exceptions import ( CollectionChecksumError, @@ -13,8 +16,15 @@ CollectionStatisticsError, CollectionTruncateError, CollectionUnloadError, + DatabaseDeleteError, +) +from tests.helpers import ( + assert_raises, + extract, + generate_col_name, + generate_string, + generate_username, ) -from tests.helpers import assert_raises, extract, generate_col_name def test_collection_attributes(db, col, username): @@ -38,10 +48,27 @@ def test_collection_misc_methods(col, bad_col, cluster): # Test configure properties prev_sync = properties["sync"] - properties = col.configure(sync=not prev_sync, schema={}) + + computed_values = [ + { + "name": "foo", + "expression": "RETURN 1", + "computeOn": ["insert", "update", "replace"], + "overwrite": True, + "failOnWarning": False, + "keepNull": True, + } + ] + + properties = col.configure( + sync=not prev_sync, schema={}, computed_values=computed_values + ) + assert properties["name"] == col.name assert properties["system"] is False assert properties["sync"] is not prev_sync + assert properties["computedValues"] == computed_values + col.configure(computed_values=[]) # Test configure properties with bad collection with assert_raises(CollectionConfigureError) as err: @@ -123,6 +150,16 @@ def test_collection_misc_methods(col, bad_col, cluster): bad_col.recalculate_count() assert err.value.error_code in {11, 1228} + # Test collection info + info = col.info() + assert set(info.keys()) == {"id", "name", "system", "type", "status", "global_id"} + assert info["name"] == col.name + assert info["system"] is False + + # Test collection compact + result = col.compact() + assert result == info + def test_collection_management(db, bad_db, cluster): # Test create collection @@ -133,42 +170,78 @@ def test_collection_management(db, bad_db, cluster): "rule": { "type": "object", "properties": { - "test_attr": {"type": "string"}, + "test_attr:": {"type": "string"}, }, "required": ["test_attr"], }, "level": "moderate", "message": "Schema Validation Failed.", + "type": "json", } + computed_values = [ + { + "name": "foo", + "expression": "RETURN 1", + "computeOn": ["insert", "update", "replace"], + "overwrite": True, + "failOnWarning": False, + "keepNull": True, + } + ] + + col = db.create_collection( + name=col_name, key_generator="autoincrement", key_increment=9, key_offset=100 + ) + key_options = col.properties()["key_options"] + assert key_options["key_generator"] == "autoincrement" + assert key_options["key_increment"] == 9 + assert key_options["key_offset"] == 100 + db.delete_collection(col_name) + col = db.create_collection( name=col_name, sync=True, system=False, key_generator="traditional", user_keys=False, - key_increment=9, - key_offset=100, edge=True, shard_count=2, - shard_fields=["test_attr"], + shard_fields=["test_attr:"], replication_factor=1, - shard_like="", sync_replication=False, enforce_replication_factor=False, sharding_strategy="community-compat", - smart_join_attribute="test", + smart_join_attribute="test_attr", write_concern=1, schema=schema, + computedValues=computed_values, ) assert db.has_collection(col_name) is True + if cluster: + for details in (False, True): + shards = col.shards(details=details) + assert shards["name"] == col_name + assert shards["system"] is False + assert len(shards["shards"]) == 2 + properties = col.properties() assert "key_options" in properties assert properties["schema"] == schema assert properties["name"] == col_name assert properties["sync"] is True assert properties["system"] is False + assert properties["computedValues"] == computed_values + col.configure(computed_values=[]) + + if cluster: + # Create distribute-shards-like collection + shard_like_name = col_name + "_shards_like" + shard_like_col = db.create_collection(name=shard_like_name, shard_like=col_name) + assert shard_like_col.properties()["shard_like"] == col_name + assert db.has_collection(shard_like_name) is True + assert db.delete_collection(shard_like_name, system=False) is True # Test create duplicate collection with assert_raises(CollectionCreateError) as err: @@ -227,3 +300,79 @@ def test_collection_management(db, bad_db, cluster): with assert_raises(CollectionRenameError) as err: bad_db.collection(new_name).rename(new_name) assert err.value.error_code in {11, 1228} + + +@pytest.fixture +def special_collection_names(db): + names = ["abc123", "maçã", "mötör", "😀", "ﻚﻠﺑ ﻞﻄﻴﻓ", "かわいい犬"] + + yield names + + for name in names: + try: + db.delete_collection(name) + except CollectionDeleteError: + pass + + +# Code duplication from `test_database.py`... +@pytest.fixture +def special_db_names(sys_db): + names = ["abc123", "maçã", "mötör", "😀", "ﻚﻠﺑ ﻞﻄﻴﻓ", "かわいい犬"] + + yield names + + for name in names: + try: + sys_db.delete_database(name) + except DatabaseDeleteError: + pass + + +def test_collection_utf8(db, special_collection_names): + for name in special_collection_names: + create_and_delete_collection(db, name) + + +# Not sure if this belongs in here or in `test_database.py`... +def test_database_and_collection_utf8( + sys_db, special_collection_names, special_db_names +): + client = ArangoClient(hosts="http://127.0.0.1:8529") + for db_name in special_db_names: + username = generate_username() + password = generate_string() + + assert sys_db.create_database( + name=db_name, + users=[ + { + "active": True, + "username": username, + "password": password, + } + ], + ) + + assert sys_db.has_database(db_name) + + db = client.db(db_name, username, password, verify=True) + + for col_name in special_collection_names: + create_and_delete_collection(db, col_name) + + assert sys_db.delete_database(db_name) + + +def create_and_delete_collection(db, name): + col = db.create_collection(name) + assert col.name == name + assert db.has_collection(name) is True + + persistent_index = {"type": "persistent", "fields": ["foo"]} + index_id = col.add_index(persistent_index)["name"] + assert index_id == col.indexes()[-1]["name"] + assert col.delete_index(index_id) is True + + assert db.delete_collection(name) is True + assert db.has_collection(name) is False diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 3dba809d..184d7ed8 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -22,7 +22,7 @@ def test_cursor_from_execute_query(db, col, docs): batch_size=2, ttl=1000, optimizer_rules=["+all"], - profile=True, + profile=2, ) cursor_id = cursor.id assert "Cursor" in repr(cursor) @@ -41,12 +41,27 @@ def test_cursor_from_execute_query(db, col, docs): assert "http_requests" in statistics assert "scanned_full" in statistics assert "scanned_index" in statistics + assert "nodes" in statistics + assert cursor.warnings() == [] profile = cursor.profile() assert profile["initializing"] > 0 assert profile["parsing"] > 0 + plan = cursor.plan() + expected_keys = { + "nodes", + "rules", + "collections", + "variables", + "estimatedCost", + "estimatedNrItems", + "isModificationQuery", + } + for key in expected_keys: + assert key in plan + assert clean_doc(cursor.next()) == docs[0] assert cursor.id == cursor_id assert cursor.has_more() is True @@ -106,7 +121,7 @@ def test_cursor_write_query(db, col, docs): batch_size=1, ttl=1000, optimizer_rules=["+all"], - profile=True, + profile=1, max_runtime=0.0, ) cursor_id = cursor.id @@ -262,6 +277,87 @@ def test_cursor_manual_fetch_and_pop(db, col, docs): assert err.value.message == "current batch is empty" +def test_cursor_retry_disabled(db, col, docs): + cursor = db.aql.execute( + f"FOR d IN {col.name} SORT d._key RETURN d", + count=True, + batch_size=1, + ttl=1000, + optimizer_rules=["+all"], + profile=True, + allow_retry=False, + ) + result = cursor.fetch() + assert result["id"] == cursor.id + + while not cursor.empty(): + cursor.pop() + + # The next batch ID should have no effect + cursor._next_batch_id = "2" + result = cursor.fetch() + assert result["next_batch_id"] == "4" + doc = cursor.pop() + assert clean_doc(doc) == docs[2] + + assert cursor.close(ignore_missing=True) + + +def test_cursor_retry(db, col, docs, db_version): + cursor = db.aql.execute( + f"FOR d IN {col.name} SORT d._key RETURN d", + count=True, + batch_size=1, + ttl=1000, + optimizer_rules=["+all"], + profile=True, + allow_retry=True, + ) + + assert cursor.count() == len(docs) + doc = cursor.pop() + assert clean_doc(doc) == docs[0] + assert cursor.has_more() + + result = cursor.fetch() + assert result["id"] == cursor.id + assert result["next_batch_id"] == "3" + doc = cursor.pop() + assert clean_doc(doc) == docs[1] + assert cursor.empty() + + # Decrease the next batch ID as if the previous fetch failed + cursor._next_batch_id = "2" + result = cursor.fetch() + assert result["id"] == cursor.id + assert result["next_batch_id"] == "3" + doc = cursor.pop() + assert clean_doc(doc) == docs[1] + assert cursor.empty() + + # Fetch the next batches normally + for batch in range(2, 5): + result = cursor.fetch() + assert result["id"] == cursor.id + assert result["next_batch_id"] == str(batch + 2) + doc = cursor.pop() + assert clean_doc(doc) == docs[batch] + + result = cursor.fetch() + assert not cursor.has_more() + assert "id" not in result + assert "next_batch_id" not in result + doc = cursor.pop() + assert clean_doc(doc) == docs[-1] + + # We should be able to fetch the last batch again + cursor.fetch() + doc = cursor.pop() + assert clean_doc(doc) == docs[-1] + + assert cursor.close() + + def test_cursor_no_count(db, col): cursor = db.aql.execute( f"FOR d IN {col.name} SORT d._key RETURN d", diff --git a/tests/test_database.py b/tests/test_database.py index f56122c0..0b1d9752 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,8 @@ from datetime import datetime +import pytest +from packaging import version + from arango.aql import AQL from arango.backup import Backup from arango.cluster import Cluster @@ -10,16 +13,21 @@ USE_SYSTEM_DATABASE, ) from arango.exceptions import ( + DatabaseCompactError, DatabaseCreateError, DatabaseDeleteError, DatabaseListError, DatabasePropertiesError, + DatabaseSupportInfoError, ServerDetailsError, ServerEchoError, ServerEngineError, + ServerLicenseSetError, ServerLogLevelError, + ServerLogLevelResetError, ServerLogLevelSetError, ServerMetricsError, + ServerModeSetError, ServerReadLogError, ServerReloadRoutingError, ServerRequiredDBVersionError, @@ -33,7 +41,12 @@ from arango.pregel import Pregel from arango.replication import Replication from arango.wal import WAL -from tests.helpers import assert_raises, generate_db_name +from tests.helpers import ( + assert_raises, + generate_col_name, + generate_db_name, + generate_jwt, +) def test_database_attributes(db, username): @@ -53,7 +66,7 @@ def test_database_attributes(db, username): assert isinstance(db.wal, WAL) -def test_database_misc_methods(sys_db, db, bad_db): +def test_database_misc_methods(client, sys_db, db, bad_db, cluster, secret, db_version): # Test get properties properties = db.properties() assert "id" in properties @@ -68,6 +81,7 @@ def test_database_misc_methods(sys_db, db, bad_db): # Test get server version assert isinstance(db.version(), str) + assert isinstance(db.version(details=True), dict) # Test get server version with bad database with assert_raises(ServerVersionError) as err: @@ -85,8 +99,8 @@ def test_database_misc_methods(sys_db, db, bad_db): assert err.value.error_code in {11, 1228} # Test get server required database version - version = db.required_db_version() - assert isinstance(version, str) + required_version = db.required_db_version() + assert isinstance(required_version, str) # Test get server target version with bad database with assert_raises(ServerRequiredDBVersionError): @@ -127,6 +141,19 @@ def test_database_misc_methods(sys_db, db, bad_db): bad_db.role() assert err.value.error_code in {11, 1228} + # Test get/set server mode + assert sys_db.mode() == "default" + with assert_raises(ServerModeSetError): + sys_db.set_mode("badmode") + assert err.value.error_code in {11, 1228} + + with assert_raises(ServerModeSetError): + db.set_mode("readonly") + assert err.value.error_code in {11, 1228} + + result = sys_db.set_mode("default") + assert result == {"mode": "default"} + # Test get server status status = db.status() assert "host" in status @@ -161,32 +188,59 @@ def test_database_misc_methods(sys_db, db, bad_db): bad_db.echo() assert err.value.error_code in {11, 1228} + # Test echo (forward request) + body = "request goes here" + echo = db.echo(body) + assert isinstance(echo, dict) + assert echo["requestBody"] == body + # Test read_log with default parameters + # Deprecated in 3.8.0 + # TODO: Remove in future release log = sys_db.read_log(upto="fatal") assert "lid" in log assert "level" in log assert "text" in log assert "total_amount" in log + log_entry = sys_db.read_log_entries(upto="fatal") + assert "total" in log_entry + assert "messages" in log_entry + + kwargs = { + "level": "error", + "start": 0, + "size": 100000, + "offset": 0, + "search": "test", + "sort": "desc", + } + # Test read_log with specific parameters - log = sys_db.read_log( - level="error", - start=0, - size=100000, - offset=0, - search="test", - sort="desc", - ) + # Deprecated in 3.8.0 + # TODO: Remove in future release + log = sys_db.read_log(**kwargs) assert "lid" in log assert "level" in log assert "text" in log assert "total_amount" in log + log_entry = sys_db.read_log_entries(**kwargs) + assert "total" in log_entry + assert "messages" in log_entry + # Test read_log with bad database + # Deprecated in 3.8.0 + # TODO: Remove in future release with assert_raises(ServerReadLogError) as err: bad_db.read_log() assert err.value.error_code in {11, 1228} + # Test read_log_entries with bad database + with assert_raises(ServerReadLogError) as err: + bad_db.read_log_entries() + assert err.value.error_code in {11, 1228} + # Test reload routing assert isinstance(db.reload_routing(), bool) @@ -196,7 +250,11 @@ def test_database_misc_methods(sys_db, db, bad_db): assert err.value.error_code in {11, 1228} # Test get log levels - assert isinstance(sys_db.log_levels(), dict) + default_log_levels = sys_db.log_levels() + assert isinstance(default_log_levels, dict) + if db_version >= version.parse("3.12.2"): + log_levels_with_appenders = sys_db.log_levels(with_appenders=True) + assert isinstance(log_levels_with_appenders, dict) # Test get log levels with bad database with assert_raises(ServerLogLevelError) as err: @@ -204,17 +262,56 @@ def test_database_misc_methods(sys_db, db, bad_db): assert err.value.error_code in {11, 1228} # Test set log levels - new_levels = {"agency": "DEBUG", "collector": "INFO", "threads": "WARNING"} + new_levels = {"agency": "DEBUG", "engines": "INFO", "threads": "WARNING"} result = sys_db.set_log_levels(**new_levels) for key, value in new_levels.items(): assert result[key] == value for key, value in sys_db.log_levels().items(): assert result[key] == value + if cluster: + # Test get log levels (with server_id) + server_id = sys_db.cluster.server_id() + assert isinstance(sys_db.log_levels(server_id), dict) + + # Test set log levels (with server_id) + result = sys_db.set_log_levels(server_id, **new_levels) + for key, value in new_levels.items(): + assert result[key] == value + for key, value in sys_db.log_levels(server_id).items(): + assert result[key] == value + # Test set log levels with bad database with assert_raises(ServerLogLevelSetError): bad_db.set_log_levels(**new_levels) + # Test Log Settings + result_1 = sys_db.set_log_settings(database=True, url=True, username=True) + result_2 = sys_db.log_settings() + assert isinstance(result_1, dict) + assert "database" in result_1 + assert "url" in result_1 + assert "username" in result_1 + assert result_1 == result_2 + + result_1 = sys_db.set_log_settings(database=True, username=False) + result_2 = sys_db.log_settings() + assert "database" in result_1 + assert "url" in result_1 + assert "username" not in result_1 + assert result_1 == result_2 + + # Reset Log Settings + if db.version() >= "3.12.1": + if cluster: + server_id = sys_db.cluster.server_id() + assert isinstance(sys_db.reset_log_levels(server_id), dict) + + result = sys_db.reset_log_levels() + assert result == default_log_levels + with assert_raises(ServerLogLevelResetError): + bad_db.reset_log_levels() + # Test get storage engine engine = db.engine() assert engine["name"] in ["rocksdb"] @@ -225,16 +322,56 @@ def test_database_misc_methods(sys_db, db, bad_db): bad_db.engine() assert err.value.error_code in {11, 1228} + with assert_raises(DatabaseSupportInfoError) as err: + db.support_info() + + info = sys_db.support_info() + assert isinstance(info, dict) + assert "deployment" in info + assert "date" in info + + # Test execute JavaScript code + assert db.execute(1) is None + assert db.execute(None) == {"error": False, "code": 200} + assert db.execute("") == {"error": False, "code": 200} + assert db.execute("return 1") == 1 + + # Test database compact + with assert_raises(DatabaseCompactError) as err: + db.compact() + + collection = db.create_collection(generate_col_name()) + collection.insert({"foo": "bar"}) + + token = generate_jwt(secret) + db_superuser = client.db(db.name, superuser_token=token) + result = db_superuser.compact() + assert result == {} + def test_database_management(db, sys_db, bad_db): # Test list databases result = sys_db.databases() assert "_system" in result + assert db.name in result + + # Test list databases accesible to root user + result = sys_db.databases_accessible_to_user() + assert "_system" in result + assert db.name in result + + # Test list databases accessible to user + result = db.databases_accessible_to_user() + assert result == [db.name] # Test list databases with bad database with assert_raises(DatabaseListError): bad_db.databases() + # Test list accessible databases with bad database + with assert_raises(DatabaseListError): + bad_db.databases_accessible_to_user() + # Test create database db_name = generate_db_name() assert sys_db.has_database(db_name) is False @@ -280,3 +417,50 @@ def test_database_management(db, sys_db, bad_db): sys_db.delete_database(db_name) assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} assert sys_db.delete_database(db_name, ignore_missing=True) is False + + +@pytest.fixture +def special_db_names(sys_db): + names = ["abc123", "maçã", "mötör", "😀", "ﻚﻠﺑ ﻞﻄﻴﻓ", "かわいい犬"] + + yield names + + for name in names: + try: + sys_db.delete_database(name) + except DatabaseDeleteError: + pass + + +def test_database_utf8(sys_db, special_db_names): + for name in special_db_names: + assert sys_db.create_database(name) + assert sys_db.has_database(name) + assert sys_db.delete_database(name) + + +def test_license(sys_db, enterprise): + license = sys_db.license() + assert isinstance(license, dict) + + if enterprise: + assert set(license.keys()) == { + "upgrading", + "features", + "license", + "version", + "status", + } + else: + assert license == {"license": "none"} + with pytest.raises(ServerLicenseSetError): + sys_db.set_license('"abc"') + + +def test_options(sys_db, db_version): + # Skip if below 3.12 + if db_version < version.parse("3.12.0"): + pytest.skip("Database options require ArangoDB 3.12+") + + assert sys_db.options() + assert sys_db.options_available() diff --git a/tests/test_document.py b/tests/test_document.py index 95446da1..7cb0a435 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,8 +1,7 @@ import pytest +from packaging import version from arango.exceptions import ( - CursorCloseError, - CursorNextError, DocumentCountError, DocumentDeleteError, DocumentGetError, @@ -14,6 +13,8 @@ DocumentReplaceError, DocumentRevisionError, DocumentUpdateError, + IndexGetError, + IndexMissingError, ) from tests.helpers import ( assert_raises, @@ -142,6 +143,13 @@ def test_document_insert(col, docs): ) assert err.value.error_code == 1210 + # Test insert with cache refilling + empty_collection(col) + doc = docs[0] + assert col.insert(doc, refill_index_caches=True) + assert col[doc["_key"]]["_key"] == doc["_key"] + assert col[doc["_key"]]["val"] == doc["val"] + def test_document_insert_many(col, bad_col, docs): # Test insert_many with default options @@ -236,6 +244,70 @@ def test_document_insert_many(col, bad_col, docs): bad_col.insert_many(docs) assert err.value.error_code in {11, 1228} + # test overwrite_mode: replace + extra_docs = [ + {"_key": docs[0]["_key"], "foo": {"qux": 2}}, + {"_key": "20", "foo": "t"}, + ] + results = col.insert_many( + documents=extra_docs, + return_old=True, + return_new=True, + overwrite_mode="replace", + keep_none=False, + merge=True, + ) + + assert "old" in results[0] + assert "new" in results[0] + assert "old" not in results[1] + assert "new" in results[1] + + # test overwrite_mode: update + extra_docs = [ + {"_key": docs[0]["_key"], "foo": {"t": 1}}, + {"_key": "20", "foo": None}, + ] + results = col.insert_many( + documents=extra_docs, + return_old=True, + return_new=True, + overwrite_mode="update", + keep_none=False, + merge=True, + ) + assert "old" in results[0] + assert "new" in results[0] + assert results[0]["new"]["foo"] == {"qux": 2, "t": 1} + assert "old" in results[1] + assert "new" in results[1] + assert "foo" not in results[1]["new"] + + extra_docs = [ + {"_key": docs[0]["_key"], "foo": {"t": 1}}, + {"_key": "21", "foo": None}, + ] + + # Test insert conflict + results = col.insert_many( + documents=extra_docs, + return_old=True, + return_new=True, + overwrite_mode="conflict", + keep_none=True, + merge=True, + ) + assert isinstance(results[0], DocumentInsertError) + assert results[0].error_code == 1210 + assert "new" in results[1] + + # Test insert with cache refilling + empty_collection(col) + assert col.insert_many(docs, refill_index_caches=True) + for doc in docs: + assert col[doc["_key"]]["_key"] == doc["_key"] + assert col[doc["_key"]]["val"] == doc["val"] + def test_document_update(col, docs): doc = docs[0] @@ -367,6 +439,11 @@ def test_document_update(col, docs): assert col.update(doc, silent=True) is True assert col[doc["_key"]]["val"] == 8 + # Test update with cache refilling + doc["val"] = 9 + assert col.update(doc, refill_index_caches=True, check_rev=False) + assert col[doc["_key"]]["val"] == 9 + def test_document_update_many(col, bad_col, docs): col.insert_many(docs) @@ -545,11 +622,29 @@ def test_document_update_many(col, bad_col, docs): for doc in docs: assert col[doc["_key"]]["val"] == 8 + # Test update_many with cache refilling + for doc in docs: + doc["val"] = 9 + assert col.update_many(docs, refill_index_caches=True, check_rev=False) + for doc in docs: + assert col[doc["_key"]]["val"] == 9 + # Test update_many with bad documents with assert_raises(DocumentParseError) as err: bad_col.update_many([{}]) assert str(err.value) == 'field "_key" or "_id" required' + # Test update_many with raise on document error (revision) + with assert_raises(DocumentRevisionError) as err: + # Raises: [HTTP 202][ERR 1200] conflict, _rev values do not match + col.update_many(docs, raise_on_document_error=True) + + # Test update_many with raise on document error (update) + with assert_raises(DocumentUpdateError) as err: + # Raises: [HTTP 202][ERR 1202] document not found + bad_docs = [{"_key": "unknown_doc", "foo": "bar"}] + col.update_many(bad_docs, raise_on_document_error=True) + def test_document_update_match(col, bad_col, docs): # Set up test documents @@ -679,6 +774,11 @@ def test_document_replace(col, docs): assert col.replace(doc, silent=True) is True assert col[doc["_key"]]["val"] == 8 + # Test replace with cache refilling + doc["val"] = 9 + assert col.replace(doc, refill_index_caches=True, check_rev=False) + assert col[doc["_key"]]["val"] == 9 + def test_document_replace_many(col, bad_col, docs): col.insert_many(docs) @@ -762,8 +862,8 @@ def test_document_replace_many(col, bad_col, docs): assert "foo" not in doc assert doc["baz"] == 4 - # Test replace_many with check_rev set to False - results = col.replace_many(docs, check_rev=False) + # Test replace_many with check_rev set to False and cache refilling + results = col.replace_many(docs, check_rev=False, refill_index_caches=True) for result, doc in zip(results, docs): doc_key = doc["_key"] assert result["_id"] == f"{col.name}/{doc_key}" @@ -910,9 +1010,9 @@ def test_document_delete(col, docs): if col.context != "transaction": assert col.delete(bad_key, ignore_missing=True) is False - # Test delete (document) with silent set to True + # Test delete (document) with silent set to True and cache refilling doc = docs[5] - assert col.delete(doc, silent=True) is True + assert col.delete(doc, silent=True, refill_index_caches=True) is True assert doc["_key"] not in col assert len(col) == 1 @@ -974,9 +1074,9 @@ def test_document_delete_many(col, bad_col, docs): old_revs[doc_key] = result["_rev"] assert len(col) == 0 - # Test delete_many with silent set to True + # Test delete_many with silent set to True and cache refilling col.import_bulk(docs) - assert col.delete_many(docs, silent=True) is True + assert col.delete_many(docs, silent=True, refill_index_caches=True) is True assert len(col) == 0 # Test delete_many (documents) with check_rev set to True @@ -1062,11 +1162,51 @@ def test_document_find(col, bad_col, docs): # Set up test documents col.import_bulk(docs) + # Test find with sort expression (single field) + found = list(col.find({}, sort=[{"sort_by": "text", "sort_order": "ASC"}])) + assert len(found) == 6 + assert found[0]["text"] == "bar" + assert found[-1]["text"] == "foo" + + # Test find with sort expression (multiple fields) + found = list( + col.find( + {}, + sort=[ + {"sort_by": "text", "sort_order": "ASC"}, + {"sort_by": "val", "sort_order": "DESC"}, + ], + ) + ) + assert len(found) == 6 + assert found[0]["val"] == 6 + assert found[-1]["val"] == 1 + # Test find (single match) with default options found = list(col.find({"val": 2})) assert len(found) == 1 assert found[0]["_key"] == "2" + # Test find with text value + found = list(col.find({"text": "foo"})) + assert len(found) == 3 + + # Test find with list value + found = list(col.find({"loc": [1, 1]})) + assert len(found) == 1 + assert found[0]["_key"] == "1" + + # Test find with dict value with None + data = {"dict": {"foo": "bar", "foo_2": None}} + col.insert(data) + found = list(col.find(data)) + assert len(found) == 1 + + # Test find with multiple conditions + found = list(col.find({"val": 2, "text": "foo"})) + assert len(found) == 1 + assert found[0]["_key"] == "2" + # Test find (multiple matches) with default options col.update_match({"val": 2}, {"val": 1}) found = list(col.find({"val": 1})) @@ -1117,6 +1257,14 @@ def test_document_find(col, bad_col, docs): bad_col.find({"val": 1}) assert err.value.error_code in {11, 1228} + # Test find by attribute with a space + col.insert({"foo bar": "baz"}) + assert len(list(col.find({"foo bar": "baz"}))) == 1 + + # Test find by nested attribute + col.insert({"foo": {"bar": "baz"}}) + assert len(list(col.find({"foo.bar": "baz"}))) == 1 + def test_document_find_near(col, bad_col, docs): col.import_bulk(docs) @@ -1234,7 +1382,7 @@ def test_document_find_in_radius(col, bad_col): assert err.value.error_code in {11, 1228} -def test_document_find_in_box(col, bad_col, geo, cluster): +def test_document_find_in_box(db, col, bad_col, geo, cluster): if cluster: pytest.skip("Not tested in a cluster setup") @@ -1251,6 +1399,10 @@ def test_document_find_in_box(col, bad_col, geo, cluster): ) assert clean_doc(result) == [doc1, doc3] + # Test find_in_box with no index provided + result = col.find_in_box(latitude1=0, longitude1=0, latitude2=6, longitude2=3) + assert clean_doc(result) == [doc1, doc3] + # Test find_in_box with limit of -1 with assert_raises(AssertionError) as err: col.find_in_box( @@ -1267,21 +1419,22 @@ def test_document_find_in_box(col, bad_col, geo, cluster): result = col.find_in_box( latitude1=0, longitude1=0, latitude2=6, longitude2=3, limit=0, index=geo["id"] ) - assert clean_doc(result) == [doc1, doc3] + assert clean_doc(result) == [] # Test find_in_box with limit of 1 result = col.find_in_box( - latitude1=0, + latitude1=2, longitude1=0, latitude2=6, longitude2=3, limit=1, + index=geo["id"], ) assert clean_doc(result) == [doc3] # Test find_in_box with limit of 4 result = col.find_in_box( - latitude1=0, longitude1=0, latitude2=10, longitude2=10, limit=4 + latitude1=0, longitude1=0, latitude2=10, longitude2=10, limit=4, index=geo["id"] ) assert clean_doc(result) == [doc1, doc2, doc3, doc4] @@ -1293,6 +1446,7 @@ def test_document_find_in_box(col, bad_col, geo, cluster): latitude2=6, longitude2=3, skip=-1, + index=geo["id"], ) assert "skip must be a non-negative int" == str(err.value) @@ -1303,22 +1457,56 @@ def test_document_find_in_box(col, bad_col, geo, cluster): latitude2=6, longitude2=3, skip=1, + index=geo["id"], ) - assert clean_doc(result) == [doc1] + assert clean_doc(result) in [[doc1], [doc3]] # Test find_in_box with skip 3 result = col.find_in_box( - latitude1=0, longitude1=0, latitude2=10, longitude2=10, skip=2 + latitude1=0, longitude1=0, latitude2=10, longitude2=10, skip=3, index=geo["id"] ) - assert clean_doc(result) == [doc1, doc2] + assert clean_doc(result) in [[doc1], [doc2], [doc3], [doc4]] + + # Test find_in_box with missing index in collection + empty_col = db.create_collection(generate_col_name()) + with assert_raises(IndexMissingError) as err: + empty_col.find_in_box( + latitude1=0, + longitude1=0, + latitude2=6, + longitude2=3, + ) + + # Test find_in_box with non-existing index + with assert_raises(IndexGetError) as err: + col.find_in_box( + latitude1=0, + longitude1=0, + latitude2=6, + longitude2=3, + index="abc", + ) + + # Test find_in_box with non-geo index + persistent_index = {"type": "persistent", "fields": ["loc"]} + non_geo = col.add_index(persistent_index) + with assert_raises(ValueError) as err: + col.find_in_box( + latitude1=0, + longitude1=0, + latitude2=6, + longitude2=3, + index=non_geo["id"], + ) # Test find_in_box with bad collection - with assert_raises(DocumentGetError) as err: + with assert_raises(IndexGetError) as err: bad_col.find_in_box( latitude1=0, longitude1=0, latitude2=6, longitude2=3, + index=geo["id"], ) assert err.value.error_code in {11, 1228} @@ -1390,7 +1578,7 @@ def test_document_has(col, bad_col, docs): with assert_raises(DocumentRevisionError) as err: col.has(doc_input, rev=bad_rev, check_rev=True) - assert err.value.error_code == 1200 + assert err.value.error_code == 412 # Test existing documents with bad revision for doc_input in [ @@ -1400,15 +1588,15 @@ def test_document_has(col, bad_col, docs): ]: with assert_raises(DocumentRevisionError) as err: col.has(doc_input) - assert err.value.error_code == 1200 + assert err.value.error_code == 412 with assert_raises(DocumentRevisionError) as err: col.has(doc_input, rev=bad_rev) - assert err.value.error_code == 1200 + assert err.value.error_code == 412 with assert_raises(DocumentRevisionError) as err: col.has(doc_input, rev=bad_rev, check_rev=True) - assert err.value.error_code == 1200 + assert err.value.error_code == 412 assert doc_input in col assert col.has(doc_input, rev=rev, check_rev=True) is True @@ -1487,12 +1675,12 @@ def test_document_has(col, bad_col, docs): # Test get with bad database with assert_raises(DocumentInError) as err: bad_col.has(doc_key) - assert err.value.error_code in {11, 1228} + assert err.value.error_code == 401 # Test contains with bad database with assert_raises(DocumentInError) as err: assert doc_key in bad_col - assert err.value.error_code in {11, 1228} + assert err.value.error_code == 401 def test_document_get(col, bad_col, docs): @@ -1567,7 +1755,10 @@ def test_document_get_many(col, bad_col, docs): # Test get_many in empty collection empty_collection(col) - assert col.get_many([]) == [] + + # sending an empty list returns internal error + # assert col.get_many([]) == [] + assert col.get_many(docs[:1]) == [] assert col.get_many(docs[:3]) == [] @@ -1676,76 +1867,6 @@ def test_document_keys(col, bad_col, docs): assert err.value.error_code in {11, 1228} -def test_document_export(col, bad_col, docs, cluster): - if cluster: - pytest.skip("Not tested in a cluster setup") - - # Set up test documents - col.insert_many(docs) - - # Test export with flush set to True and flush_wait set to 1 - cursor = col.export(flush=True, flush_wait=1) - assert clean_doc(cursor) == docs - assert cursor.type == "export" - - # Test export with count - cursor = col.export(flush=False, count=True) - assert cursor.count() == len(docs) - assert clean_doc(cursor) == docs - - # Test export with batch size - cursor = col.export(flush=False, count=True, batch_size=1) - assert cursor.count() == len(docs) - assert clean_doc(cursor) == docs - - # Test export with time-to-live - cursor = col.export(flush=False, count=True, ttl=10) - assert cursor.count() == len(docs) - assert clean_doc(cursor) == docs - - # Test export with filters - cursor = col.export( - count=True, flush=False, filter_fields=["text"], filter_type="exclude" - ) - assert cursor.count() == len(docs) - assert all(["text" not in d for d in cursor]) - - # Test export with a limit of 0 - cursor = col.export(flush=False, count=True, limit=0) - assert cursor.count() == 0 - assert clean_doc(cursor) == [] - - # Test export with a limit of 1 - cursor = col.export(flush=False, count=True, limit=1) - assert cursor.count() == 1 - assert len(list(cursor)) == 1 - all([clean_doc(d) in docs for d in cursor]) - - # Test export with a limit of 3 - cursor = col.export(flush=False, count=True, limit=3) - assert cursor.count() == 3 - assert len(list(cursor)) == 3 - all([clean_doc(d) in docs for d in cursor]) - - # Test export with bad database - with assert_raises(DocumentGetError): - bad_col.export() - - # Test closing export cursor - cursor = col.export(flush=False, count=True, batch_size=1) - assert cursor.close(ignore_missing=False) is True - assert cursor.close(ignore_missing=True) is False - - assert clean_doc(cursor.next()) in docs - with assert_raises(CursorNextError): - cursor.next() - with assert_raises(CursorCloseError): - cursor.close(ignore_missing=False) - - cursor = col.export(flush=False, count=True) - assert cursor.close(ignore_missing=True) is None - - def test_document_random(col, bad_col, docs): # Set up test documents col.import_bulk(docs) @@ -1832,6 +1953,21 @@ def test_document_import_bulk(col, bad_col, docs): assert col[doc_key]["loc"] == doc["loc"] empty_collection(col) + # Test import bulk with batch_size + results = col.import_bulk(docs, batch_size=len(docs) // 2) + assert type(results) is list + assert len(results) == 2 + empty_collection(col) + + result = col.import_bulk(docs, batch_size=len(docs) * 2) + assert type(result) is list + assert len(result) == 1 + empty_collection(col) + + # Test import bulk with overwrite and batch_size + with pytest.raises(ValueError): + col.import_bulk(docs, overwrite=True, batch_size=1) + # Test import bulk on_duplicate actions doc = docs[0] doc_key = doc["_key"] @@ -1957,3 +2093,74 @@ def test_document_management_via_db(db, col): assert result["_id"] == doc1_id assert doc1_id not in col assert len(col) == 2 + + +def test_version_attributes_update_many(col, db_version): + if db_version < version.parse("3.12.0"): + pytest.skip("Version attributes is tested in 3.12.0+") + + col.insert_many( + [ + {"_key": "test1", "version": 0}, + {"_key": "test2", "version": 0}, + {"_key": "test3", "version": 0}, + ] + ) + + docs = [ + {"_key": "test1", "version": 2}, + {"_key": "test1", "version": 3}, + {"_key": "test1", "version": 1}, + {"_key": "test2", "version": 1}, + {"_key": "test2", "version": 9}, + {"_key": "test2", "version": 42}, + {"_key": "test2", "version": 0}, + {"_key": "test3"}, + {"_key": "test3", "version": 5}, + {"_key": "test3", "version": 4}, + {"_key": "test3", "value": 2}, + ] + + col.update_many(docs, version_attribute="version") + assert col["test1"]["version"] == 3 + assert col["test2"]["version"] == 42 + assert col["test3"]["version"] == 5 + + docs = [ + {"_key": "test1", "version": 2}, + {"_key": "test1", "version": 3}, + {"_key": "test1", "version": 5}, + {"_key": "test2", "version": 1}, + {"_key": "test2", "version": 9}, + {"_key": "test2", "version": 42}, + {"_key": "test2", "version": 0}, + {"_key": "test3", "version": 5}, + {"_key": "test3", "version": 6}, + ] + + col.replace_many(docs, version_attribute="version") + assert col["test1"]["version"] == 5 + assert col["test2"]["version"] == 42 + assert col["test3"]["version"] == 6 + + docs = [ + {"_key": "test1", "version": 0}, + {"_key": "test2", "version": 0}, + {"_key": "test3", "version": 0}, + ] + + col.insert_many(docs, overwrite_mode="update", version_attribute="version") + assert col["test1"]["version"] == 5 + assert col["test2"]["version"] == 42 + assert col["test3"]["version"] == 6 + + docs = [ + {"_key": "test1", "version": 43}, + {"_key": "test2", "version": 41}, + {"_key": "test3", "version": 43}, + ] + + col.insert_many(docs, overwrite_mode="replace", version_attribute="version") + assert col["test1"]["version"] == 43 + assert col["test2"]["version"] == 42 + assert col["test3"]["version"] == 43 diff --git a/tests/test_foxx.py b/tests/test_foxx.py index 83a7f966..b096d2e8 100644 --- a/tests/test_foxx.py +++ b/tests/test_foxx.py @@ -29,7 +29,7 @@ from arango.foxx import Foxx from tests.helpers import assert_raises, extract, generate_service_mount -service_file = "/tmp/service.zip" +service_file = "/tests/static/service.zip" service_name = "test" diff --git a/tests/test_graph.py b/tests/test_graph.py index de440642..4d2588cb 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -1,3 +1,6 @@ +import pytest +from packaging import version + from arango.collection import EdgeCollection from arango.exceptions import ( DocumentDeleteError, @@ -330,6 +333,7 @@ def test_create_graph_with_edge_definition(db): orphan_collections=[ovcol_name], ) assert edge_definition in new_graph.edge_definitions() + assert ovcol_name in new_graph.vertex_collections() def test_vertex_management(fvcol, bad_fvcol, fvdocs): @@ -1070,7 +1074,10 @@ def test_edge_management_via_graph(graph, ecol, fvcol, fvdocs, tvcol, tvdocs): assert len(ecol) == 1 -def test_traverse(db): +def test_traverse(db, db_version): + if db_version >= version.parse("3.12.0"): + pytest.skip("Traversal API is no longer available for ArangoDB 3.12+") + # Create test graph, vertex and edge collections school = db.create_graph(generate_graph_name()) profs = school.create_vertex_collection(generate_col_name()) diff --git a/tests/test_index.py b/tests/test_index.py index 85171338..a5d0f5eb 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1,6 +1,10 @@ +import pytest +from packaging import version + from arango.exceptions import ( IndexCreateError, IndexDeleteError, + IndexGetError, IndexListError, IndexLoadError, ) @@ -23,23 +27,45 @@ def test_list_indexes(icol, bad_col): assert err.value.error_code in {11, 1228} +def test_get_index(icol, bad_col): + indexes = icol.indexes() + for index in indexes: + retrieved_index = icol.get_index(index["id"]) + assert retrieved_index["id"] == index["id"] + assert retrieved_index["name"] == index["name"] + assert retrieved_index["type"] == index["type"] + assert retrieved_index["fields"] == index["fields"] + assert retrieved_index["sparse"] == index["sparse"] + assert retrieved_index["unique"] == index["unique"] + # TODO: Revisit + # assert retrieved_index["selectivity"] == index["selectivity"] + + with assert_raises(IndexGetError) as err: + icol.get_index("bad_index") + + assert err.value.error_code == 1212 + + def test_add_hash_index(icol): icol = icol fields = ["attr1", "attr2"] - result = icol.add_hash_index( - fields=fields, - unique=True, - sparse=True, - deduplicate=True, - name="hash_index", - in_background=False, + result = icol.add_index( + { + "type": "hash", + "fields": fields, + "unique": True, + "sparse": True, + "deduplicate": True, + "name": "hash_index", + "inBackground": False, + } ) expected_index = { "sparse": True, "type": "hash", - "fields": ["attr1", "attr2"], + "fields": fields, "unique": True, "deduplicate": True, "name": "hash_index", @@ -55,13 +81,16 @@ def test_add_hash_index(icol): def test_add_skiplist_index(icol): fields = ["attr1", "attr2"] - result = icol.add_skiplist_index( - fields=fields, - unique=True, - sparse=True, - deduplicate=True, - name="skiplist_index", - in_background=False, + result = icol.add_index( + { + "type": "skiplist", + "fields": fields, + "unique": True, + "sparse": True, + "deduplicate": True, + "name": "skiplist_index", + "inBackground": False, + } ) expected_index = { @@ -83,8 +112,14 @@ def test_add_skiplist_index(icol): def test_add_geo_index(icol): # Test add geo index with one attribute - result = icol.add_geo_index( - fields=["attr1"], ordered=False, name="geo_index", in_background=True + result = icol.add_index( + { + "type": "geo", + "fields": ["attr1"], + "geoJson": True, + "name": "geo_index", + "inBackground": True, + } ) expected_index = { @@ -92,18 +127,21 @@ def test_add_geo_index(icol): "type": "geo", "fields": ["attr1"], "unique": False, - "geo_json": False, + "geoJson": True, "name": "geo_index", } for key, value in expected_index.items(): - assert result[key] == value + assert result[key] == value, (key, value, result[key]) assert result["id"] in extract("id", icol.indexes()) # Test add geo index with two attributes - result = icol.add_geo_index( - fields=["attr1", "attr2"], - ordered=False, + result = icol.add_index( + { + "type": "geo", + "fields": ["attr1", "attr2"], + "geoJson": False, + } ) expected_index = { "sparse": True, @@ -118,7 +156,7 @@ def test_add_geo_index(icol): # Test add geo index with more than two attributes (should fail) with assert_raises(IndexCreateError) as err: - icol.add_geo_index(fields=["attr1", "attr2", "attr3"]) + icol.add_index({"type": "geo", "fields": ["attr1", "attr2", "attr3"]}) assert err.value.error_code == 10 # Clean up the index @@ -127,14 +165,20 @@ def test_add_geo_index(icol): def test_add_fulltext_index(icol): # Test add fulltext index with one attributes - result = icol.add_fulltext_index( - fields=["attr1"], min_length=10, name="fulltext_index", in_background=True + result = icol.add_index( + { + "type": "fulltext", + "fields": ["attr1"], + "minLength": 10, + "name": "fulltext_index", + "inBackground": True, + } ) expected_index = { "sparse": True, "type": "fulltext", "fields": ["attr1"], - "min_length": 10, + "minLength": 10, "unique": False, "name": "fulltext_index", } @@ -145,7 +189,7 @@ def test_add_fulltext_index(icol): # Test add fulltext index with two attributes (should fail) with assert_raises(IndexCreateError) as err: - icol.add_fulltext_index(fields=["attr1", "attr2"]) + icol.add_index({"type": "fulltext", "fields": ["attr1", "attr2"]}) assert err.value.error_code == 10 # Clean up the index @@ -154,12 +198,15 @@ def test_add_fulltext_index(icol): def test_add_persistent_index(icol): # Test add persistent index with two attributes - result = icol.add_persistent_index( - fields=["attr1", "attr2"], - unique=True, - sparse=True, - name="persistent_index", - in_background=True, + result = icol.add_index( + { + "type": "persistent", + "fields": ["attr1", "attr2"], + "unique": True, + "sparse": True, + "name": "persistent_index", + "inBackground": True, + } ) expected_index = { "sparse": True, @@ -179,13 +226,19 @@ def test_add_persistent_index(icol): def test_add_ttl_index(icol): # Test add persistent index with two attributes - result = icol.add_ttl_index( - fields=["attr1"], expiry_time=1000, name="ttl_index", in_background=True + result = icol.add_index( + { + "type": "ttl", + "fields": ["attr1"], + "expireAfter": 1000, + "name": "ttl_index", + "inBackground": True, + } ) expected_index = { "type": "ttl", "fields": ["attr1"], - "expiry_time": 1000, + "expireAfter": 1000, "name": "ttl_index", } for key, value in expected_index.items(): @@ -197,11 +250,114 @@ def test_add_ttl_index(icol): icol.delete_index(result["id"]) +def test_add_inverted_index(icol, enterprise): + parameters = dict( + fields=[{"name": "attr1", "cache": True}], + name="c0_cached", + storedValues=[{"fields": ["a"], "compression": "lz4", "cache": True}], + includeAllFields=True, + analyzer="identity", + primarySort={"cache": True, "fields": [{"field": "a", "direction": "asc"}]}, + ) + expected_keys = ["primarySort", "analyzer", "includeAllFields", "searchField"] + + if enterprise: + parameters["cache"] = True + parameters["primaryKeyCache"] = True + expected_keys.extend(["cache", "primaryKeyCache"]) + + result = icol.add_index({"type": "inverted", **parameters}) + assert result["id"] in extract("id", icol.indexes()) + + for key in expected_keys: + assert key in result + + icol.delete_index(result["id"]) + + +def test_add_zkd_index(icol, db_version): + result = icol.add_index( + { + "type": "zkd", + "fields": ["x", "y", "z"], + "fieldValueTypes": "double", + "name": "zkd_index", + "inBackground": False, + "unique": False, + } + ) + + expected_index = { + "name": "zkd_index", + "type": "zkd", + "fields": ["x", "y", "z"], + "isNewlyCreated": True, + "unique": False, + } + + for key, value in expected_index.items(): + assert result[key] == value + + assert result["id"] in extract("id", icol.indexes()) + + with assert_raises(IndexCreateError) as err: + icol.add_index( + {"type": "zkd", "fieldValueTypes": "integer", "fields": ["x", "y", "z"]} + ) + assert err.value.error_code == 10 + + icol.delete_index(result["id"]) + + +def test_add_mdi_index(icol, db_version): + if db_version < version.parse("3.12.0"): + pytest.skip("MDI indexes are usable with 3.12+ only") + + result = icol.add_index( + { + "type": "mdi", + "fields": ["x", "y", "z"], + "fieldValueTypes": "double", + "name": "mdi_index", + "inBackground": False, + "unique": True, + } + ) + + expected_index = { + "name": "mdi_index", + "type": "mdi", + "fields": ["x", "y", "z"], + "isNewlyCreated": True, + "unique": True, + } + + for key, value in expected_index.items(): + assert result[key] == value + + assert result["id"] in extract("id", icol.indexes()) + + with assert_raises(IndexCreateError) as err: + icol.add_index( + { + "type": "mdi", + "fieldValueTypes": "integer", + "fields": ["x", "y", "z"], + } + ) + assert err.value.error_code == 10 + + icol.delete_index(result["id"]) + + def test_delete_index(icol, bad_col): old_indexes = set(extract("id", icol.indexes())) - icol.add_hash_index(["attr3", "attr4"], unique=True) - icol.add_skiplist_index(["attr3", "attr4"], unique=True) - icol.add_fulltext_index(fields=["attr3"], min_length=10) + hash_index = {"type": "hash", "fields": ["attr1", "attr2"], "unique": True} + icol.add_index(hash_index) + skiplist_Index = {"type": "skiplist", "fields": ["attr3", "attr4"], "unique": True} + icol.add_index(skiplist_Index) + fulltext_index = {"type": "fulltext", "fields": ["attr5"], "min_length": 10} + icol.add_index(fulltext_index) new_indexes = set(extract("id", icol.indexes())) assert new_indexes.issuperset(old_indexes) diff --git a/tests/test_overload.py b/tests/test_overload.py new file mode 100644 index 00000000..d3d93a7f --- /dev/null +++ b/tests/test_overload.py @@ -0,0 +1,49 @@ +import warnings + +from arango import errno +from arango.exceptions import OverloadControlExecutorError + + +def flood_with_requests(controlled_db, async_db): + """ + Flood the database with requests. + It is impossible to predict what the last recorded queue time will be. + We can only try and make it as large as possible. However, if the system + is fast enough, it may still be 0. + """ + controlled_db.aql.execute("RETURN SLEEP(0.5)", count=True) + for _ in range(3): + for _ in range(500): + async_db.aql.execute("RETURN SLEEP(0.5)", count=True) + controlled_db.aql.execute("RETURN SLEEP(0.5)", count=True) + if controlled_db.last_queue_time >= 0: + break + + +def test_overload_control(db): + controlled_db = db.begin_controlled_execution(100) + assert controlled_db.max_queue_time == 100 + + async_db = db.begin_async_execution(return_result=True) + + flood_with_requests(controlled_db, async_db) + assert controlled_db.last_queue_time >= 0 + + # We can only emit a warning here. The test will still pass. + if controlled_db.last_queue_time == 0: + warnings.warn( + f"last_queue_time of {controlled_db} is 0, test may be unreliable" + ) + + controlled_db.adjust_max_queue_time(0.0001) + try: + flood_with_requests(controlled_db, async_db) + assert controlled_db.last_queue_time >= 0 + except OverloadControlExecutorError as e: + assert e.http_code == errno.HTTP_PRECONDITION_FAILED + assert e.error_code == errno.QUEUE_TIME_REQUIREMENT_VIOLATED + else: + warnings.warn( + f"last_queue_time of {controlled_db} is {controlled_db.last_queue_time}," + f"test may be unreliable" + ) diff --git a/tests/test_pregel.py b/tests/test_pregel.py index a99bddce..2be8d5f0 100644 --- a/tests/test_pregel.py +++ b/tests/test_pregel.py @@ -1,23 +1,26 @@ import time import pytest +from packaging import version -from arango.exceptions import ( - PregelJobCreateError, - PregelJobDeleteError, - PregelJobGetError, -) +from arango.exceptions import PregelJobCreateError, PregelJobDeleteError from tests.helpers import assert_raises, generate_string -def test_pregel_attributes(db, username): +def test_pregel_attributes(db, db_version, username): + if db_version >= version.parse("3.12.0"): + pytest.skip("Pregel is not tested in 3.12.0+") + assert db.pregel.context in ["default", "async", "batch", "transaction"] assert db.pregel.username == username assert db.pregel.db_name == db.name assert repr(db.pregel) == f"" -def test_pregel_management(db, graph, cluster): +def test_pregel_management(db, db_version, graph, cluster): + if db_version >= version.parse("3.12.0"): + pytest.skip("Pregel is not tested in 3.12.0+") + if cluster: pytest.skip("Not tested in a cluster setup") @@ -51,9 +54,8 @@ def test_pregel_management(db, graph, cluster): # Test delete existing pregel job assert db.pregel.delete_job(job_id) is True time.sleep(0.2) - with assert_raises(PregelJobGetError) as err: - db.pregel.job(job_id) - assert err.value.error_code in {4, 10, 1600} + job = db.pregel.job(job_id) + assert job["state"] == "canceled" # Test delete missing pregel job with assert_raises(PregelJobDeleteError) as err: diff --git a/tests/test_replication.py b/tests/test_replication.py index bf03991c..2136b97a 100644 --- a/tests/test_replication.py +++ b/tests/test_replication.py @@ -150,9 +150,12 @@ def test_replication_applier(sys_db, bad_db, url, cluster): bad_db.replication.stop_applier() assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} + # We need a tcp endpoint + tcp_endpoint = url.replace("http", "tcp") + # Test replication set applier config result = sys_db.replication.set_applier_config( - endpoint=url, + endpoint=tcp_endpoint, database="_system", username="root", password="passwd", @@ -174,7 +177,7 @@ def test_replication_applier(sys_db, bad_db, url, cluster): restrict_type="exclude", restrict_collections=["students"], ) - assert result["endpoint"] == url + assert result["endpoint"] == tcp_endpoint assert result["database"] == "_system" assert result["username"] == "root" assert result["max_connect_retries"] == 120 @@ -196,7 +199,7 @@ def test_replication_applier(sys_db, bad_db, url, cluster): assert result["restrict_collections"] == ["students"] with assert_raises(ReplicationApplierConfigSetError) as err: - bad_db.replication.set_applier_config(url) + bad_db.replication.set_applier_config(tcp_endpoint) assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} # Test replication start applier diff --git a/tests/test_request.py b/tests/test_request.py index 616a388c..256c9a68 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -1,7 +1,7 @@ from arango.request import Request -def test_request_no_data(): +def test_request_no_data() -> None: request = Request( method="post", endpoint="/_api/test", @@ -11,15 +11,13 @@ def test_request_no_data(): assert request.method == "post" assert request.endpoint == "/_api/test" assert request.params == {"bool": "1"} - assert request.headers == { - "charset": "utf-8", - "content-type": "application/json", - "foo": "bar", - } + assert request.headers["charset"] == "utf-8" + assert request.headers["content-type"] == "application/json" + assert request.headers["foo"] == "bar" assert request.data is None -def test_request_string_data(): +def test_request_string_data() -> None: request = Request( method="post", endpoint="/_api/test", @@ -30,15 +28,13 @@ def test_request_string_data(): assert request.method == "post" assert request.endpoint == "/_api/test" assert request.params == {"bool": "1"} - assert request.headers == { - "charset": "utf-8", - "content-type": "application/json", - "foo": "bar", - } + assert request.headers["charset"] == "utf-8" + assert request.headers["content-type"] == "application/json" + assert request.headers["foo"] == "bar" assert request.data == "test" -def test_request_json_data(): +def test_request_json_data() -> None: request = Request( method="post", endpoint="/_api/test", @@ -49,15 +45,13 @@ def test_request_json_data(): assert request.method == "post" assert request.endpoint == "/_api/test" assert request.params == {"bool": "1"} - assert request.headers == { - "charset": "utf-8", - "content-type": "application/json", - "foo": "bar", - } + assert request.headers["charset"] == "utf-8" + assert request.headers["content-type"] == "application/json" + assert request.headers["foo"] == "bar" assert request.data == {"baz": "qux"} -def test_request_transaction_data(): +def test_request_transaction_data() -> None: request = Request( method="post", endpoint="/_api/test", @@ -68,9 +62,7 @@ def test_request_transaction_data(): assert request.method == "post" assert request.endpoint == "/_api/test" assert request.params == {"bool": "1"} - assert request.headers == { - "charset": "utf-8", - "content-type": "application/json", - "foo": "bar", - } + assert request.headers["charset"] == "utf-8" + assert request.headers["content-type"] == "application/json" + assert request.headers["foo"] == "bar" assert request.data == {"baz": "qux"} diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 0598addd..af58bb99 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,10 +1,21 @@ +from typing import Set + +import pytest + from arango.resolver import ( + FallbackHostResolver, + PeriodicHostResolver, RandomHostResolver, RoundRobinHostResolver, SingleHostResolver, ) +def test_bad_resolver(): + with pytest.raises(ValueError): + RandomHostResolver(3, 2) + + def test_resolver_single_host(): resolver = SingleHostResolver() for _ in range(20): @@ -16,6 +27,21 @@ def test_resolver_random_host(): for _ in range(20): assert 0 <= resolver.get_host_index() < 10 + resolver = RandomHostResolver(3) + indexes_to_filter: Set[int] = set() + + index_a = resolver.get_host_index() + indexes_to_filter.add(index_a) + + index_b = resolver.get_host_index(indexes_to_filter) + indexes_to_filter.add(index_b) + assert index_b != index_a + + index_c = resolver.get_host_index(indexes_to_filter) + indexes_to_filter.clear() + indexes_to_filter.add(index_c) + assert index_c not in [index_a, index_b] + def test_resolver_round_robin(): resolver = RoundRobinHostResolver(10) @@ -30,3 +56,30 @@ def test_resolver_round_robin(): assert resolver.get_host_index() == 8 assert resolver.get_host_index() == 9 assert resolver.get_host_index() == 0 + + +def test_resolver_periodic(): + resolver = PeriodicHostResolver(3, requests_period=3) + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index() == 1 + + +def test_resolver_fallback(): + resolver = FallbackHostResolver(4) + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index({0, 1, 3}) == 2 + assert resolver.get_host_index({1, 2, 3}) == 0 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index() == 1 diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 4308e4c6..75ec28a2 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -1,28 +1,31 @@ import pytest +from packaging import version from arango.database import TransactionDatabase from arango.exceptions import ( TransactionAbortError, TransactionCommitError, TransactionExecuteError, + TransactionFetchError, TransactionInitError, TransactionStatusError, ) -from tests.helpers import extract +from tests.helpers import extract, generate_db_name def test_transaction_execute_raw(db, col, docs): # Test execute raw transaction doc = docs[0] key = doc["_key"] - result = db.execute_transaction( - command=f""" + command = f""" function (params) {{ var db = require('internal').db; db.{col.name}.save({{'_key': params.key, 'val': 1}}); return true; }} - """, + """ # noqa: E702 E231 E272 E202 + result = db.execute_transaction( + command=command, params={"key": key}, write=[col.name], read=[col.name], @@ -42,7 +45,7 @@ def test_transaction_execute_raw(db, col, docs): assert err.value.error_code == 10 -def test_transaction_init(db, bad_db, col, username): +def test_transaction_init(db, db_version, bad_db, col, username): txn_db = db.begin_transaction() assert isinstance(txn_db, TransactionDatabase) @@ -67,6 +70,22 @@ def test_transaction_init(db, bad_db, col, username): bad_db.begin_transaction() assert err.value.error_code in {11, 1228} + # Test all options + kwargs = dict( + read=col.name, + write=col.name, + exclusive=[], + sync=True, + allow_implicit=False, + lock_timeout=1000, + max_size=1024 * 1024, + ) + if db_version >= version.parse("3.12.1"): + kwargs["skip_fast_lock_round"] = True + txn_db = db.begin_transaction(**kwargs) + assert isinstance(txn_db, TransactionDatabase) + assert txn_db.transaction_id is not None + def test_transaction_status(db, col, docs): txn_db = db.begin_transaction(read=col.name) @@ -96,7 +115,7 @@ def test_transaction_commit(db, col, docs): sync=True, allow_implicit=False, lock_timeout=1000, - max_size=10000, + max_size=1024 * 1024, # 1MB ) txn_col = txn_db.collection(col.name) @@ -117,6 +136,38 @@ def test_transaction_commit(db, col, docs): assert err.value.error_code in {10, 1655} +def test_transaction_fetch_existing(db, col, docs): + original_txn = db.begin_transaction( + read=col.name, + write=col.name, + exclusive=[], + sync=True, + allow_implicit=False, + lock_timeout=1000, + max_size=1024 * 1024, # 1MB + ) + txn_col = original_txn.collection(col.name) + + assert "_rev" in txn_col.insert(docs[0]) + assert "_rev" in txn_col.delete(docs[0]) + + txn_db = db.fetch_transaction(transaction_id=original_txn.transaction_id) + + txn_col = txn_db.collection(col.name) + assert "_rev" in txn_col.insert(docs[1]) + assert "_rev" in txn_col.delete(docs[1]) + + txn_db.commit_transaction() + assert txn_db.transaction_status() == "committed" + assert original_txn.transaction_status() == "committed" + assert txn_db.transaction_id == original_txn.transaction_id + + # Test fetch transaction that does not exist + with pytest.raises(TransactionFetchError) as err: + db.fetch_transaction(transaction_id="illegal") + assert err.value.error_code in {10, 1655} + + def test_transaction_abort(db, col, docs): txn_db = db.begin_transaction(write=col.name) txn_col = txn_db.collection(col.name) @@ -149,3 +200,34 @@ def test_transaction_graph(db, graph, fvcol, fvdocs): assert len(vcol) == 0 txn_db.commit_transaction() + + +def test_transaction_list(client, sys_db, username, password): + db_name = generate_db_name() + + sys_db.create_database( + name=db_name, + users=[{"username": username, "password": password, "active": True}], + ) + + db = client.db(db_name, username, password) + + assert db.list_transactions() == [] + + txn_db = db.begin_transaction() + txn_db.aql.execute("RETURN 1") + + txn_db_2 = db.begin_transaction() + txn_db_2.aql.execute("RETURN 1") + + assert len(db.list_transactions()) == 2 + + txn_db.commit_transaction() + + assert len(db.list_transactions()) == 1 + + txn_db_2.commit_transaction() + + assert db.list_transactions() == [] + + sys_db.delete_database(db_name) diff --git a/tests/test_user.py b/tests/test_user.py index 2e48448f..6bf8439a 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -79,7 +79,7 @@ def test_user_management(sys_db, bad_db): ) assert new_user["username"] == username assert new_user["active"] is False - assert new_user["extra"] == {"bar": "baz"} + assert new_user["extra"] == {"foo": "bar", "bar": "baz"} assert sys_db.user(username) == new_user # Update missing user diff --git a/tests/test_view.py b/tests/test_view.py index 8f610fd6..778f87e6 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -1,3 +1,5 @@ +from packaging import version + from arango.exceptions import ( ViewCreateError, ViewDeleteError, @@ -10,7 +12,7 @@ from tests.helpers import assert_raises, generate_view_name -def test_view_management(db, bad_db, col, cluster): +def test_view_management(db, bad_db, col, cluster, db_version, enterprise): # Test create view view_name = generate_view_name() bad_view_name = generate_view_name() @@ -47,18 +49,28 @@ def test_view_management(db, bad_db, col, cluster): bad_db.views() assert err.value.error_code in {11, 1228} - # Test get view + # Test get view (properties) view = db.view(view_name) assert view["id"] == view_id assert view["name"] == view_name assert view["type"] == view_type assert view["consolidation_interval_msec"] == 50000 + # Test get view (info) + view_info = db.view_info(view_name) + assert view_info["id"] == view_id + assert view_info["name"] == view_name + assert view_info["type"] == view_type + # Test get missing view with assert_raises(ViewGetError) as err: db.view(bad_view_name) assert err.value.error_code == 1203 + with assert_raises(ViewGetError) as err: + db.view_info(bad_view_name) + assert err.value.error_code == 1203 + # Test update view view = db.update_view(view_name, {"consolidationIntervalMsec": 70000}) assert view["id"] == view_id @@ -112,6 +124,20 @@ def test_view_management(db, bad_db, col, cluster): # Test delete missing view with ignore_missing set to True assert db.delete_view(view_name, ignore_missing=True) is False + if enterprise and db_version >= version.parse("3.12"): + res = db.create_view( + view_name, + view_type, + properties={ + "links": {col.name: {"fields": {}}}, + "optimizeTopK": [ + "BM25(@doc) DESC", + ], + }, + ) + assert "optimizeTopK" in res + db.delete_view(view_name) + def test_arangosearch_view_management(db, bad_db, cluster): # Test create arangosearch view @@ -166,3 +192,37 @@ def test_arangosearch_view_management(db, bad_db, cluster): # Test delete arangosearch view assert db.delete_view(view_name, ignore_missing=False) is True + + +def test_arangosearch_view_properties(db, col, enterprise): + view_name = generate_view_name() + params = {"consolidationIntervalMsec": 50000} + + if enterprise: + params.update( + { + "links": { + col.name: { + "fields": { + "value": { + "nested": {"nested_1": {"nested": {"nested_2": {}}}} + } + } + } + } + } + ) + + params.update({"primarySortCache": True, "primaryKeyCache": True}) + params.update({"storedValues": ["attr1", "attr2"]}) + + result = db.create_arangosearch_view(view_name, properties=params) + assert "id" in result + assert result["name"] == view_name + assert result["type"].lower() == "arangosearch" + + if enterprise: + assert "links" in result + assert col.name in result["links"] + + assert db.delete_view(view_name) is True