pyproject.toml

[tool.poetry]
name = "dlt"
version = "1.8.0a0"
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
authors = ["dltHub Inc. <services@dlthub.com>"]
maintainers = [ "Marcin Rudolf <marcin@dlthub.com>", "Adrian Brudaru <adrian@dlthub.com>", "Anton Burnashev <anton@dlthub.com>", "David Scharf <david@dlthub.com>" ]
readme = "README.md"
license = "Apache-2.0"
homepage = "https://github.com/dlt-hub"
repository = "https://github.com/dlt-hub/dlt"
classifiers = [
"Development Status :: 5 - Production/Stable",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Topic :: Software Development :: Libraries",
    "Typing :: Typed",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Operating System :: MacOS :: MacOS X",
    "Operating System :: POSIX :: Linux",
    "Operating System :: Microsoft :: Windows",]
keywords = [ "etl" ]
include = [ "LICENSE.txt", "README.md", "dlt/sources/pipeline_templates/.gitignore", "dlt/sources/pipeline_templates/.dlt/config.toml" ]
packages = [
    { include = "dlt" },
]

[tool.poetry.dependencies]
python = ">=3.9,<3.14"
requests = ">=2.26.0"
pendulum = {version = ">=2.1.2", markers = "python_version < '3.13'"}
# NOTE: revert to normal pendulum once it supports python 3.13
dlt-pendulum = [
    {version=">=3.0.2", markers = "python_version >= '3.13'"}
]
simplejson = ">=3.17.5"
PyYAML = ">=5.4.1"
semver = ">=3.0.0"
hexbytes = ">=0.2.2"
tzdata = ">=2022.1"
tomlkit = ">=0.11.3"
pathvalidate = ">=2.5.2"
typing-extensions = ">=4.8.0"
makefun = ">=1.15.0"
click = ">=7.1"
requirements-parser = ">=0.5.0"
setuptools = ">=65.6.0"
humanize = ">=4.4.0"
gitpython = ">=3.1.29"
pytz = ">=2022.6"
giturlparse = ">=0.10.0"
# exclude some versions because of segfault bugs in orjson
orjson = {version = ">=3.6.7,<4,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,!=3.10.1", markers="platform_python_implementation != 'PyPy'"}
tenacity = ">=8.0.2"
jsonpath-ng = ">=1.5.3"
fsspec = ">=2022.4.0"
packaging = ">=21.1"
pluggy = ">=1.3.0"
win-precise-time = {version = ">=1.4.2", markers="os_name == 'nt'"}

psycopg2-binary = {version = ">=2.9.1", optional = true}
# use this dependency as the current version of psycopg2cffi does not have sql module
# psycopg2cffi = {git = "https://github.com/chtd/psycopg2cffi.git", optional = true, markers="platform_python_implementation == 'PyPy'"}
psycopg2cffi = {version = ">=2.9.0", optional = true, markers="platform_python_implementation == 'PyPy'"}
grpcio = {version = ">=1.50.0", optional = true}
google-cloud-bigquery = {version = ">=2.26.0", optional = true}
pyarrow = [
    {version = ">=12.0.0,<18", markers = "python_version >= '3.9' and python_version < '3.13'", optional = true},
    {version = ">=18.0.0", markers = "python_version >= '3.13'", optional = true}
]
duckdb = {version = ">=0.9", optional = true}
# keep per-python version dependency as a reference
# duckdb = [
#     {version = ">=0.6.1,<0.10.0", python = ">=3.9,<3.12", optional = true},
#     {version = ">=0.10.0,<0.11.0", python = ">=3.12", optional = true}
# ]
s3fs = {version = ">=2022.4.0", optional = true}
gcsfs = {version = ">=2022.4.0", optional = true}
botocore = {version = ">=1.28", optional = true}
snowflake-connector-python = {version = ">=3.5.0", optional = true}
cron-descriptor = {version = ">=1.2.32", optional = true}
pipdeptree = {version = ">=2.9.0,<2.10", optional = true}
# pip is used by pipdeptree but not listed in its dependencies
pip = {version = ">=23.0.0", optional = true}
pyathena = {version = ">=2.9.6", optional = true}
weaviate-client = {version = ">=3.22", optional = true}
adlfs = {version = ">=2024.7.0", optional = true}
pyodbc = {version = ">=4.0.39", optional = true}
qdrant-client = {version = ">=1.8", optional = true, extras = ["fastembed"]}
databricks-sql-connector = [
    {version = ">=2.9.3,<4", optional = true, markers = "python_version <= '3.12'"},
    {version = ">=3.6.0", optional = true, markers = "python_version >= '3.13'"},
]
clickhouse-driver = { version = ">=0.2.7", optional = true }
clickhouse-connect = { version = ">=0.7.7", optional = true }
lancedb = { version = ">=0.8.2", optional = true, markers = "python_version < '3.13'", allow-prereleases = true }
tantivy = { version = ">= 0.22.0", optional = true }
deltalake = { version = ">=0.21.0", optional = true }
sqlalchemy = { version = ">=1.4", optional = true }
alembic = {version = ">1.10.0", optional = true}
paramiko = {version = ">=3.3.0", optional = true}
sqlglot = {version = ">=20.0.0", optional = true}
db-dtypes = { version = ">=1.2.0", optional = true }
# `sql-sqlite` extra leads to dependency conflict with `apache-airflow` because `apache-airflow`
# requires `sqlalchemy<2.0.0` while the extra requires `sqlalchemy>=2.0.18`
# https://github.com/apache/airflow/issues/28723
# pyiceberg = { version = ">=0.7.1", optional = true, extras = ["sql-sqlite"] }
# we will rely on manual installation of `sqlalchemy>=2.0.18` instead
pyiceberg = { version = ">=0.8.1", optional = true }
databricks-sdk = {version = ">=0.38.0", optional = true}
pywin32 = {version = ">=306", optional = true, platform = "win32"}
rich-argparse = "^1.6.0"

[tool.poetry.extras]
gcp = ["grpcio", "google-cloud-bigquery", "db-dtypes", "gcsfs"]
# bigquery is alias on gcp extras
bigquery = ["grpcio", "google-cloud-bigquery", "pyarrow", "gcsfs", "db-dtypes"]
postgres = ["psycopg2-binary", "psycopg2cffi"]
redshift = ["psycopg2-binary", "psycopg2cffi"]
parquet = ["pyarrow"]
duckdb = ["duckdb"]
filesystem = ["s3fs", "botocore", "sqlglot"]
s3 = ["s3fs", "botocore"]
gs = ["gcsfs"]
az = ["adlfs"]
sftp = ["paramiko"]
snowflake = ["snowflake-connector-python"]
motherduck = ["duckdb", "pyarrow"]
cli = ["pipdeptree", "cron-descriptor", "pip"]
athena = ["pyathena", "pyarrow", "s3fs", "botocore"]
weaviate = ["weaviate-client"]
mssql = ["pyodbc"]
synapse = ["pyodbc", "adlfs", "pyarrow"]
qdrant = ["qdrant-client"]
databricks = ["databricks-sql-connector", "databricks-sdk"]
clickhouse = ["clickhouse-driver", "clickhouse-connect", "s3fs", "gcsfs", "adlfs", "pyarrow"]
dremio = ["pyarrow"]
lancedb = ["lancedb", "pyarrow", "tantivy"]
deltalake = ["deltalake", "pyarrow"]
sql_database = ["sqlalchemy"]
sqlalchemy = ["sqlalchemy", "alembic"]
pyiceberg = ["pyiceberg", "pyarrow", "sqlalchemy"]
postgis = ["psycopg2-binary", "psycopg2cffi"]

[tool.poetry.scripts]
dlt = "dlt.cli._dlt:_main"

[tool.poetry.group.dev.dependencies]
cffi = ">=1.16"
greenlet = ">=3.1"
regex = ">=2024.10"
sqlalchemy = "<2"
requests-mock = "^1.10.0"
types-click = "^7.1.8"
sqlfluff = "^2.3.2"
types-deprecated = "^1.2.9.2"
pytest-console-scripts = "^1.4.1"
pytest = "^7.0.0"
mypy = ">=1.11.0,<1.13.0"
flake8 = "^7.0.0"
bandit = "^1.7.0"
black = "^23.7.0"
isort = "^5.12.0"
flake8-bugbear = "^22.0.0"
pytest-order = ">=1.0.0"
pytest-cases = ">=3.6.9"
pytest-forked = ">=1.3.0"
types-PyYAML = ">=6.0.7"
types-cachetools = ">=4.2.9"
types-protobuf = ">=3.19.8"
types-simplejson = ">=3.17.0"
types-requests = ">=2.25.6"
types-python-dateutil = ">=2.8.15"
flake8-tidy-imports = ">=4.8.0"
flake8-encodings = { git = "https://github.com/dlt-hub/flake8-encodings.git", branch = "disable_jedi_support" }
flake8-builtins = "^1.5.3"
boto3-stubs = "^1.28.28"
types-tqdm = "^4.66.0.2"
types-psutil = "^5.9.5.16"
types-psycopg2 = "^2.9.21.14"
cryptography = "^41.0.7"
google-api-python-client = ">=1.7.11"
pytest-asyncio = "^0.23.5"
types-sqlalchemy = "^1.4.53.38"
types-pytz = ">=2024.1.0.20240203"
ruff = "^0.3.2"
pyjwt = "^2.8.0"
pytest-mock = "^3.14.0"
types-regex = "^2024.5.15.20240519"
flake8-print = "^5.0.0"
mimesis = "^7.0.0"
shapely = ">=2.0.6"
pip = ">=24.0.0"
# keep duckdb compatible with motherduck
duckdb = "<1.3"

[tool.poetry.group.sources]
optional = true
[tool.poetry.group.sources.dependencies]
connectorx = [
    {version = ">=0.3.3", python = ">=3.9"},
    {version = ">=0.4.0", python = ">=3.10"}
]
pymysql = "^1.1.0"
openpyxl = "^3"

[tool.poetry.group.pipeline]
optional = true

[tool.poetry.group.pipeline.dependencies]
google-auth-oauthlib = "^1.0.0"
tqdm = "^4.65.0"
enlighten = "^1.11.2"
alive-progress = "^3.1.1"
pydantic = ">=2.10"
numpy = [
    { version = ">=1.21", python = ">=3.9,<3.12" },
    { version = ">=1.26", python = ">=3.12" },
    { version = ">=2.0.0", python = ">=3.13" }
]
pandas = [
    {version = ">2.1", markers = "python_version >= '3.12'"},
    {version = "<2.1", markers = "python_version < '3.12'"}
]
sqlglot = {version = ">=20.0.0"}

[tool.poetry.group.airflow]
optional = true

[tool.poetry.group.airflow.dependencies]
apache-airflow = {version = "^2.8.0", markers = "python_version < '3.12'"}

# TODO: restore when ibis allows pyarrow 18
# [tool.poetry.group.ibis]
#optional = true

#[tool.poetry.group.ibis.dependencies]
# ibis-framework = { version = ">=9.0.0,<10.0.0", markers = "python_version >= '3.10'", extras = ["duckdb", "postgres", "bigquery", "snowflake", "mssql", "clickhouse"]}

[tool.poetry.group.providers]
optional = true

[tool.poetry.group.providers.dependencies]
google-api-python-client = "^2.86.0"

[tool.poetry.group.sentry-sdk]
optional = true

[tool.poetry.group.sentry-sdk.dependencies]
sentry-sdk = "^2.0.0"

[tool.poetry.group.docs]
optional = true

[tool.poetry.group.dbt]
optional = true
[tool.poetry.group.dbt.dependencies]
dbt-core = ">=1.5.0"
dbt-redshift = ">=1.5.0"
dbt-bigquery = ">=1.5.0"
dbt-duckdb = ">=1.5.0"
dbt-snowflake = ">=1.5.0"
dbt-athena-community = ">=1.5.0"
dbt-sqlserver = ">=1.5.0"
# dbt-databricks = {version = ">=1.7.3", optional = true}

[tool.poetry.group.docs.dependencies]
SQLAlchemy = ">=1.4.0"
pymysql = "^1.1.0"
pypdf2 = "^3.0.1"
pydoc-markdown = "^4.8.2"
dbt-core = ">=1.5.0"
dbt-duckdb = ">=1.5.0"
pymongo = ">=4.3.3"
pandas = ">2"
alive-progress = ">=3.0.1"
pyarrow = [
    {version = ">=17.0.0", markers = "python_version < '3.13'"},
    {version = ">=18.0.0", markers = "python_version >= '3.13'"}
]
psycopg2-binary = ">=2.9"
lancedb = [
    { version = ">=0.8.2", markers = "python_version < '3.13'", allow-prereleases = true }
]
openai = ">=1.45"
connectorx = { version = ">=0.3.2" }
modal = ">=0.64.170"

[tool.black] # https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-via-a-file
line-length = 100
preview = true

[tool.isort] # https://pycqa.github.io/isort/docs/configuration/options.html
color_output = true
line_length = 100
profile = "black"
src_paths = ["dlt"]
multi_line_output = 3

[build-system]
requires = ["poetry-core>=1.0.8"]
build-backend = "poetry.core.masonry.api"