diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d11052c..235c160 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -465,8 +465,17 @@ jobs: # PYSEC-2023-121 (CVE-2022-4899 / GHSA-5c9c-6x87-f9vm) # * incorrectly flags zstd versions >= 1.5.4.0 as vulnerable to CVE-2022-4899 # * see https://github.com/pypa/advisory-database/blob/main/vulns/zstd/PYSEC-2023-121.yaml (our version ranges are outside of reported versions) + # CVE-2026-0994 (protobuf DoS via nested Any messages in ParseDict) + # * no fix version available yet + # * low risk: only affects json_format.ParseDict() with untrusted input + # PYSEC-2024-161 (pyarrow deserialization vulnerability) + # * false positive: only affects Arrow R package, not PyArrow + # * see CVE description: "This vulnerability only affects the arrow R package, not other Apache Arrow implementations" + # * databricks-sqlalchemy 1.x caps pyarrow<17, but upgrading requires SQLAlchemy 2.x (which is not possible for some Python versions) ignore-vulns: &ignore-vulns | PYSEC-2023-121 + CVE-2026-0994 + PYSEC-2024-161 audit-all: name: Audit - All diff --git a/deepnote_toolkit/sql/sql_execution.py b/deepnote_toolkit/sql/sql_execution.py index 77ea872..3b8e96e 100644 --- a/deepnote_toolkit/sql/sql_execution.py +++ b/deepnote_toolkit/sql/sql_execution.py @@ -440,6 +440,27 @@ def _execute_sql_with_caching( ) +@contextlib.contextmanager +def suppress_third_party_deprecation_warnings(): + """Suppress known deprecation warnings from third-party SQL packages. + + These warnings are caused by internal implementation details of upstream packages + and cannot be fixed in deepnote-toolkit. We suppress them to avoid cluttering + user output with warnings they cannot act upon. + + Suppressed warnings: + - databricks-sqlalchemy: '_user_agent_entry' parameter deprecated + https://github.com/databricks/databricks-sqlalchemy/issues/36 + """ + with warnings.catch_warnings(): + # databricks-sqlalchemy uses deprecated '_user_agent_entry' parameter + warnings.filterwarnings( + "ignore", + message=r"Parameter '_user_agent_entry' is deprecated", + ) + yield + + def _query_data_source( query, bind_params, @@ -454,7 +475,10 @@ def _query_data_source( if url is None: url = sql_alchemy_dict["url"] - engine = create_engine(url, **sql_alchemy_dict["params"], pool_pre_ping=True) + with suppress_third_party_deprecation_warnings(): + engine = create_engine( + url, **sql_alchemy_dict["params"], pool_pre_ping=True + ) try: dataframe = _execute_sql_on_engine(engine, query, bind_params) diff --git a/poetry.lock b/poetry.lock index bea8255..92953ad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -335,7 +335,7 @@ description = "A asyncio driver for ClickHouse with native tcp protocol" optional = false python-versions = "<4.0,>=3.7" groups = ["main"] -markers = "python_version < \"3.12\"" +markers = "python_version <= \"3.12\"" files = [ {file = "asynch-0.2.4-py3-none-any.whl", hash = "sha256:7286a88da4060ecd23a3b4caff6c25e9e6f26561c1b3d61853f2d6d88d57c33d"}, {file = "asynch-0.2.4.tar.gz", hash = "sha256:15ef9517bb093dfd5a0f1f31cc31e0da84543eead0269345052922cf399d280e"}, @@ -359,7 +359,7 @@ description = "An asyncio driver for ClickHouse with native TCP support" optional = false python-versions = "<4.0,>=3.9" groups = ["main"] -markers = "python_version >= \"3.12\"" +markers = "python_version == \"3.13\"" files = [ {file = "asynch-0.2.5-py3-none-any.whl", hash = "sha256:b14a4d4f49c3ed2f77a44722ce33a48b8049acb1ba24f425792ffdb3d1ef2829"}, {file = "asynch-0.2.5.tar.gz", hash = "sha256:e8e1656c81a1a156df20334e4c08d5832b02f38166dffc979235336309608543"}, @@ -1028,7 +1028,7 @@ description = "Simple ClickHouse SQLAlchemy Dialect" optional = false python-versions = "<4,>=3.7" groups = ["main"] -markers = "python_version < \"3.12\"" +markers = "python_version <= \"3.12\"" files = [ {file = "clickhouse_sqlalchemy-0.2.9.tar.gz", hash = "sha256:64b1c79edf3f277c229ccdc519e9ab3f7acc28e49dc4919f9dd280ea331728be"}, ] @@ -1046,7 +1046,7 @@ description = "Simple ClickHouse SQLAlchemy Dialect" optional = false python-versions = "<4,>=3.7" groups = ["main"] -markers = "python_version >= \"3.12\"" +markers = "python_version == \"3.13\"" files = [ {file = "clickhouse-sqlalchemy-0.3.2.tar.gz", hash = "sha256:267f3a9a1d0d186eb99a41895a684922d31125cea21702cd7dc73af1ccdd10e7"}, ] @@ -1288,6 +1288,72 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.3)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "databricks-sql-connector" +version = "4.2.4" +description = "Databricks SQL Connector for Python" +optional = false +python-versions = "<4.0.0,>=3.8.0" +groups = ["main"] +files = [ + {file = "databricks_sql_connector-4.2.4-py3-none-any.whl", hash = "sha256:e9ea625df4ee3a8a4a7855fd7e0b5799a029c8965b2ff37694fff5e8294734df"}, + {file = "databricks_sql_connector-4.2.4.tar.gz", hash = "sha256:e8ce4257ada2b6274ee1c17d4e29831c76cd9acc994c243bb9eb22314dac74ee"}, +] + +[package.dependencies] +lz4 = {version = ">=4.0.2,<5.0.0", markers = "python_version >= \"3.8\" and python_version < \"3.14\""} +oauthlib = ">=3.1.0,<4.0.0" +openpyxl = ">=3.0.10,<4.0.0" +pandas = [ + {version = ">=1.2.5,<2.4.0", markers = "python_version >= \"3.8\" and python_version < \"3.13\""}, + {version = ">=2.2.3,<2.4.0", markers = "python_version >= \"3.13\""}, +] +pybreaker = ">=1.0.0,<2.0.0" +pyjwt = ">=2.0.0,<3.0.0" +python-dateutil = ">=2.8.0,<3.0.0" +requests = ">=2.18.1,<3.0.0" +thrift = ">=0.16.0,<0.21.0" +urllib3 = ">=1.26" + +[package.extras] +pyarrow = ["pyarrow (>=14.0.1) ; python_version >= \"3.8\" and python_version < \"3.13\"", "pyarrow (>=18.0.0) ; python_version == \"3.13\"", "pyarrow (>=22.0.0) ; python_version >= \"3.14\""] + +[[package]] +name = "databricks-sqlalchemy" +version = "1.0.5" +description = "Databricks SQLAlchemy plugin for Python" +optional = false +python-versions = "<4.0.0,>=3.8.0" +groups = ["main"] +markers = "python_version <= \"3.12\"" +files = [ + {file = "databricks_sqlalchemy-1.0.5-py3-none-any.whl", hash = "sha256:397657392e76a73bb9dfecee99c1ce267f1c5dd456b6e2d961e09697ed4d328f"}, + {file = "databricks_sqlalchemy-1.0.5.tar.gz", hash = "sha256:e0b33f5803393e4baf9c466cd38b75fa4b154e23d6a588ef172d534ce5e3914b"}, +] + +[package.dependencies] +databricks_sql_connector = ">=4.0.0" +pyarrow = ">=14.0.1,<17" +sqlalchemy = ">=1.3.24,<2.0.0" + +[[package]] +name = "databricks-sqlalchemy" +version = "2.0.8" +description = "Databricks SQLAlchemy plugin for Python" +optional = false +python-versions = "<4.0.0,>=3.8.0" +groups = ["main"] +markers = "python_version == \"3.13\"" +files = [ + {file = "databricks_sqlalchemy-2.0.8-py3-none-any.whl", hash = "sha256:755b6f9dc2aaabfdc1243064f4054b8dc325da342bc9939b84988e00938b3c56"}, + {file = "databricks_sqlalchemy-2.0.8.tar.gz", hash = "sha256:2d3e3c2f3336d98d5e11aa289bdc94e69b4df288dcc776a4af06c0fddbd59f15"}, +] + +[package.dependencies] +databricks_sql_connector = ">=4.0.0" +pyarrow = ">=14.0.1" +sqlalchemy = ">=2.0.21" + [[package]] name = "datafusion" version = "49.0.0" @@ -1681,6 +1747,18 @@ files = [ [package.dependencies] packaging = ">=20.9" +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + [[package]] name = "exceptiongroup" version = "1.2.2" @@ -3576,6 +3654,38 @@ files = [ {file = "numpy-2.3.4.tar.gz", hash = "sha256:a7d018bfedb375a8d979ac758b120ba846a7fe764911a64465fd87b8729f4a6a"}, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1"}, + {file = "oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "overrides" version = "7.7.0" @@ -4281,57 +4391,54 @@ files = [ [[package]] name = "pyarrow" -version = "17.0.0" +version = "16.1.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "python_version < \"3.12\"" +markers = "python_version <= \"3.12\"" files = [ - {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"}, - {file = "pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655"}, - {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545"}, - {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2"}, - {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8"}, - {file = "pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047"}, - {file = "pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087"}, - {file = "pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977"}, - {file = "pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3"}, - {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15"}, - {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597"}, - {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420"}, - {file = "pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4"}, - {file = "pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03"}, - {file = "pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22"}, - {file = "pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053"}, - {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a"}, - {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc"}, - {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a"}, - {file = "pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b"}, - {file = "pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7"}, - {file = "pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204"}, - {file = "pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8"}, - {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155"}, - {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145"}, - {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c"}, - {file = "pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c"}, - {file = "pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca"}, - {file = "pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb"}, - {file = "pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df"}, - {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687"}, - {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b"}, - {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5"}, - {file = "pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda"}, - {file = "pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204"}, - {file = "pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28"}, + {file = "pyarrow-16.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9"}, + {file = "pyarrow-16.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd"}, + {file = "pyarrow-16.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff"}, + {file = "pyarrow-16.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c"}, + {file = "pyarrow-16.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b"}, + {file = "pyarrow-16.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b"}, + {file = "pyarrow-16.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f"}, + {file = "pyarrow-16.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7"}, + {file = "pyarrow-16.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444"}, + {file = "pyarrow-16.1.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f"}, + {file = "pyarrow-16.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed"}, + {file = "pyarrow-16.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710"}, + {file = "pyarrow-16.1.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55"}, + {file = "pyarrow-16.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3"}, + {file = "pyarrow-16.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a"}, + {file = "pyarrow-16.1.0.tar.gz", hash = "sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315"}, ] [package.dependencies] numpy = ">=1.16.6" -[package.extras] -test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] - [[package]] name = "pyarrow" version = "21.0.0" @@ -4339,7 +4446,7 @@ description = "Python library for Apache Arrow" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version >= \"3.12\"" +markers = "python_version == \"3.13\"" files = [ {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26"}, {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79"}, @@ -4442,6 +4549,21 @@ fastparquet = ["fastparquet (>=0.4.0)"] pandas = ["pandas (>=1.3.0)"] sqlalchemy = ["sqlalchemy (>=1.0.0)"] +[[package]] +name = "pybreaker" +version = "1.4.1" +description = "Python implementation of the Circuit Breaker pattern" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pybreaker-1.4.1-py3-none-any.whl", hash = "sha256:b4dab4a05195b7f2a64a6c1a6c4ba7a96534ef56ea7210e6bcb59f28897160e0"}, + {file = "pybreaker-1.4.1.tar.gz", hash = "sha256:8df2d245c73ba40c8242c56ffb4f12138fbadc23e296224740c2028ea9dc1178"}, +] + +[package.extras] +test = ["fakeredis", "mock", "pytest", "redis", "tornado", "types-mock", "types-redis"] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -6013,7 +6135,7 @@ description = "Database Abstraction Library" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" groups = ["main"] -markers = "python_version < \"3.12\"" +markers = "python_version <= \"3.12\"" files = [ {file = "SQLAlchemy-1.4.54-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:af00236fe21c4d4f4c227b6ccc19b44c594160cc3ff28d104cdce85855369277"}, {file = "SQLAlchemy-1.4.54-cp310-cp310-manylinux1_x86_64.manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_5_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1183599e25fa38a1a322294b949da02b4f0da13dbc2688ef9dbe746df573f8a6"}, @@ -6092,7 +6214,7 @@ description = "Database Abstraction Library" optional = false python-versions = ">=3.7" groups = ["main"] -markers = "python_version >= \"3.12\"" +markers = "python_version == \"3.13\"" files = [ {file = "SQLAlchemy-2.0.40-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ae9597cab738e7cc823f04a704fb754a9249f0b6695a6aeb63b74055cd417a96"}, {file = "SQLAlchemy-2.0.40-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37a5c21ab099a83d669ebb251fddf8f5cee4d75ea40a5a1653d9c43d60e20867"}, @@ -6403,6 +6525,25 @@ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] +[[package]] +name = "thrift" +version = "0.20.0" +description = "Python bindings for the Apache Thrift RPC system" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "thrift-0.20.0.tar.gz", hash = "sha256:4dd662eadf6b8aebe8a41729527bd69adf6ceaa2a8681cbef64d1273b3e8feba"}, +] + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + [[package]] name = "tinycss2" version = "1.3.0" @@ -7143,4 +7284,4 @@ server = ["deepnote-python-lsp-server", "jupyter-resource-usage", "jupyter-serve [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<3.14" -content-hash = "ea3de36fb8236900cf63a3dcd2b77b830b4f93ab4e97df2a7606a3ff420c90c1" +content-hash = "4cc899866682fe2768d833511f73116cfe44a101daa6fbc251a8617ed6888b1d" diff --git a/pyproject.toml b/pyproject.toml index ea7bf10..05b13c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,8 @@ dependencies = [ "sqlalchemy-dremio>=3.0.3,<3.1", "dill>=0.3.8,<=0.4; python_version <= '3.12'", "dill>=0.3.9,<=0.4; python_version >= '3.13'", + "databricks-sqlalchemy>=2.0.0,<3.0.0; python_version >= '3.13'", + "databricks-sqlalchemy>=1.0.5,<2.0.0; python_version < '3.13'", # SQLAlchemy 1.x compatibility # AlloyDB "pg8000==1.31.5", @@ -176,6 +178,13 @@ server = [ generate-constraints = "dockerfiles.builder.constraintsgen:main" deepnote-toolkit = "deepnote_toolkit.cli.main:main" +[project.entry-points."sqlalchemy.dialects"] +# Backward-compatible dialect alias for Databricks. +# The integration generates URLs with 'databricks+connector://' scheme (for the old +# sqlalchemy-databricks package), but we now use databricks-sqlalchemy which only +# registers the 'databricks' dialect. This alias ensures the old URL format still works. +"databricks.connector" = "databricks.sqlalchemy:DatabricksDialect" + [dependency-groups] dev = [ "pylint>=3.3.0,<4.0.0", diff --git a/tests/unit/test_deepnote_toolkit_cli.py b/tests/unit/test_deepnote_toolkit_cli.py index e0c41d6..35b41d5 100644 --- a/tests/unit/test_deepnote_toolkit_cli.py +++ b/tests/unit/test_deepnote_toolkit_cli.py @@ -235,7 +235,7 @@ def test_process_cleanup_on_interrupt( # Mock the process manager mock_manager = mock.MagicMock() mock_manager.processes = [mock_proc1, mock_proc2] - mock_manager.check_processes.return_value = [] + mock_manager.check_processes.side_effect = KeyboardInterrupt mock_context.return_value.__enter__.return_value = mock_manager args = argparse.Namespace( @@ -247,13 +247,7 @@ def test_process_cleanup_on_interrupt( python_kernel_only=None, ) - # Simulate interrupt after process checks - # Need to mock: 2x time.sleep(0.1) for process checks, then 1x time.sleep(1) before interrupt - with mock.patch( - "deepnote_toolkit.cli.server.time.sleep", - side_effect=[None, None, None, KeyboardInterrupt], - ): - ret = run_server_command(args) + ret = run_server_command(args) assert ret == 0 @@ -265,7 +259,7 @@ def test_process_cleanup_on_interrupt( mock_manager.add_process.assert_any_call(mock_proc1) mock_manager.add_process.assert_any_call(mock_proc2) - # Verify monitoring occurred + # Verify monitoring was attempted (interrupt occurred during check) mock_manager.check_processes.assert_called() diff --git a/tests/unit/test_sql_execution.py b/tests/unit/test_sql_execution.py index de46ba4..e0caaab 100644 --- a/tests/unit/test_sql_execution.py +++ b/tests/unit/test_sql_execution.py @@ -6,6 +6,7 @@ import os import secrets import unittest +import warnings from unittest import TestCase, mock import duckdb @@ -940,3 +941,81 @@ def test_federated_auth_params_bigquery_missing_params( # Verify the dict was not modified self.assertEqual(sql_alchemy_dict, original_dict) + + +class TestSuppressThirdPartyDeprecationWarnings(TestCase): + """Test suppression of known third-party deprecation warnings.""" + + def test_suppresses_databricks_user_agent_warning(self): + """Verify databricks-sqlalchemy '_user_agent_entry' warning is suppressed. + + See: https://github.com/databricks/databricks-sqlalchemy/issues/36 + """ + from deepnote_toolkit.sql.sql_execution import ( + suppress_third_party_deprecation_warnings, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + + with suppress_third_party_deprecation_warnings(): + # Simulate the warning that databricks-sqlalchemy emits + warnings.warn( + "Parameter '_user_agent_entry' is deprecated; " + "use 'user_agent_entry' instead.", + DeprecationWarning, + ) + + # Warning should be suppressed + databricks_warnings = [ + w for w in caught_warnings if "_user_agent_entry" in str(w.message) + ] + self.assertEqual(len(databricks_warnings), 0) + + def test_does_not_suppress_unrelated_warnings(self): + """Verify unrelated warnings are not suppressed.""" + from deepnote_toolkit.sql.sql_execution import ( + suppress_third_party_deprecation_warnings, + ) + + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + + with suppress_third_party_deprecation_warnings(): + warnings.warn("Some other deprecation", DeprecationWarning) + + # Unrelated warning should NOT be suppressed + self.assertEqual(len(caught_warnings), 1) + self.assertIn("Some other deprecation", str(caught_warnings[0].message)) + + +class TestSqlAlchemyDialectRegistration(TestCase): + """Test that SQL integration dialects are properly installed and registered.""" + + def test_databricks_dialect_is_registered(self): + """Verify databricks-sqlalchemy package is installed and dialect is loadable.""" + from sqlalchemy.engine.url import make_url + + # databricks-sqlalchemy 1.x and 2.x registers as 'databricks' dialect + url = make_url("databricks://token:test@host:443") + dialect_cls = url.get_dialect() + + self.assertEqual(url.drivername, "databricks") + self.assertIsNotNone(dialect_cls) + + def test_databricks_connector_dialect_alias_is_registered(self): + """Verify backward-compatible 'databricks+connector://' URL format works. + + The integration generates URLs with 'databricks+connector://' scheme + (originally for the old sqlalchemy-databricks package). We now use + databricks-sqlalchemy which only registers the 'databricks' dialect. + deepnote-toolkit registers 'databricks.connector' as an alias via entry points + in pyproject.toml to ensure the old URL format still works. + """ + from sqlalchemy.engine.url import make_url + + url = make_url("databricks+connector://token:test@host:443") + dialect_cls = url.get_dialect() + + self.assertEqual(url.drivername, "databricks+connector") + self.assertIsNotNone(dialect_cls)