diff --git a/Cargo.lock b/Cargo.lock index f425373ceea5..934cae590c1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -242,6 +242,8 @@ version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -390,6 +392,7 @@ dependencies = [ "tree-sitter", "tree-sitter-json", "tree-sitter-ql", + "zstd", ] [[package]] @@ -983,6 +986,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "jod-thread" version = "0.1.2" @@ -1334,6 +1346,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "portable-atomic" version = "1.11.0" @@ -3027,3 +3045,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/MODULE.bazel b/MODULE.bazel index 49ea49975bb9..764eb6abe72f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -124,6 +124,7 @@ use_repo( "vendor_ts__tree-sitter-ruby-0.23.1", "vendor_ts__triomphe-0.1.14", "vendor_ts__ungrammar-1.16.1", + "vendor_ts__zstd-0.13.3", ) http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index 1f97bfd967f4..52ebf159f017 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -672,3 +672,15 @@ alias( actual = "@vendor_ts__ungrammar-1.16.1//:ungrammar", tags = ["manual"], ) + +alias( + name = "zstd-0.13.3", + actual = "@vendor_ts__zstd-0.13.3//:zstd", + tags = ["manual"], +) + +alias( + name = "zstd", + actual = "@vendor_ts__zstd-0.13.3//:zstd", + tags = ["manual"], +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cc-1.2.7.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cc-1.2.7.bazel index 56be5f5153be..54aab54c9ca5 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cc-1.2.7.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cc-1.2.7.bazel @@ -28,6 +28,9 @@ rust_library( "WORKSPACE.bazel", ], ), + crate_features = [ + "parallel", + ], crate_root = "src/lib.rs", edition = "2018", rustc_flags = [ @@ -81,6 +84,42 @@ rust_library( }), version = "1.2.7", deps = [ + "@vendor_ts__jobserver-0.1.32//:jobserver", "@vendor_ts__shlex-1.3.0//:shlex", - ], + ] + select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [ + "@vendor_ts__libc-0.2.171//:libc", # aarch64-apple-darwin + ], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # aarch64-unknown-linux-gnu + ], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # aarch64-unknown-linux-gnu, aarch64-unknown-nixos-gnu + ], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [ + "@vendor_ts__libc-0.2.171//:libc", # arm-unknown-linux-gnueabi + ], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # i686-unknown-linux-gnu + ], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # powerpc-unknown-linux-gnu + ], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # s390x-unknown-linux-gnu + ], + "@rules_rust//rust/platform:x86_64-apple-darwin": [ + "@vendor_ts__libc-0.2.171//:libc", # x86_64-apple-darwin + ], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [ + "@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-freebsd + ], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-linux-gnu + ], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # x86_64-unknown-linux-gnu, x86_64-unknown-nixos-gnu + ], + "//conditions:default": [], + }), ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.jobserver-0.1.32.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.jobserver-0.1.32.bazel new file mode 100644 index 000000000000..7e041e35b8c0 --- /dev/null +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.jobserver-0.1.32.bazel @@ -0,0 +1,158 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors +############################################################################### + +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "jobserver", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2021", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=jobserver", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.1.32", + deps = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-apple-ios": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-linux-android": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:armv7-linux-androideabi": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:i686-apple-darwin": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:i686-linux-android": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:i686-unknown-freebsd": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-apple-darwin": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-apple-ios": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-linux-android": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [ + "@vendor_ts__libc-0.2.171//:libc", # cfg(unix) + ], + "//conditions:default": [], + }), +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.pkg-config-0.3.32.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.pkg-config-0.3.32.bazel new file mode 100644 index 000000000000..8fb981887a7a --- /dev/null +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.pkg-config-0.3.32.bazel @@ -0,0 +1,83 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors +############################################################################### + +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "pkg_config", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=pkg-config", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.3.32", +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-0.13.3.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-0.13.3.bazel new file mode 100644 index 000000000000..2ba80632d93b --- /dev/null +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-0.13.3.bazel @@ -0,0 +1,92 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors +############################################################################### + +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "zstd", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "arrays", + "default", + "legacy", + "zdict_builder", + ], + crate_root = "src/lib.rs", + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=zstd", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.13.3", + deps = [ + "@vendor_ts__zstd-safe-7.2.4//:zstd_safe", + ], +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-safe-7.2.4.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-safe-7.2.4.bazel new file mode 100644 index 000000000000..265b6514b701 --- /dev/null +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-safe-7.2.4.bazel @@ -0,0 +1,158 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_build_script") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "zstd_safe", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "arrays", + "legacy", + "std", + "zdict_builder", + ], + crate_root = "src/lib.rs", + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=zstd-safe", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "7.2.4", + deps = [ + "@vendor_ts__zstd-safe-7.2.4//:build_script_build", + "@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:zstd_sys", + ], +) + +cargo_build_script( + name = "_bs", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + "**/*.rs", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "arrays", + "legacy", + "std", + "zdict_builder", + ], + crate_name = "build_script_build", + crate_root = "build.rs", + data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + edition = "2018", + link_deps = [ + "@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:zstd_sys", + ], + pkg_name = "zstd-safe", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=zstd-safe", + "manual", + "noclippy", + "norustfmt", + ], + version = "7.2.4", + visibility = ["//visibility:private"], +) + +alias( + name = "build_script_build", + actual = ":_bs", + tags = ["manual"], +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel new file mode 100644 index 000000000000..7db2ad80d024 --- /dev/null +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel @@ -0,0 +1,157 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//misc/bazel/3rdparty:vendor_tree_sitter_extractors +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_build_script") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "zstd_sys", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "legacy", + "std", + "zdict_builder", + ], + crate_root = "src/lib.rs", + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=zstd-sys", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "2.0.15+zstd.1.5.7", + deps = [ + "@vendor_ts__zstd-sys-2.0.15-zstd.1.5.7//:build_script_build", + ], +) + +cargo_build_script( + name = "_bs", + srcs = glob( + include = ["**/*.rs"], + allow_empty = True, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + "**/*.rs", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "legacy", + "std", + "zdict_builder", + ], + crate_name = "build_script_build", + crate_root = "build.rs", + data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + edition = "2018", + links = "zstd", + pkg_name = "zstd-sys", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=zstd-sys", + "manual", + "noclippy", + "norustfmt", + ], + version = "2.0.15+zstd.1.5.7", + visibility = ["//visibility:private"], + deps = [ + "@vendor_ts__cc-1.2.7//:cc", + "@vendor_ts__pkg-config-0.3.32//:pkg_config", + ], +) + +alias( + name = "build_script_build", + actual = ":_bs", + tags = ["manual"], +) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index ee2044468e73..0247ee36d9f5 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -381,6 +381,7 @@ _NORMAL_DEPENDENCIES = { "tracing": Label("@vendor_ts__tracing-0.1.41//:tracing"), "tracing-subscriber": Label("@vendor_ts__tracing-subscriber-0.3.19//:tracing_subscriber"), "tree-sitter": Label("@vendor_ts__tree-sitter-0.24.6//:tree_sitter"), + "zstd": Label("@vendor_ts__zstd-0.13.3//:zstd"), }, }, } @@ -1658,6 +1659,16 @@ def crate_repositories(): build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.itoa-1.0.15.bazel"), ) + maybe( + http_archive, + name = "vendor_ts__jobserver-0.1.32", + sha256 = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0", + type = "tar.gz", + urls = ["https://static.crates.io/crates/jobserver/0.1.32/download"], + strip_prefix = "jobserver-0.1.32", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.jobserver-0.1.32.bazel"), + ) + maybe( http_archive, name = "vendor_ts__jod-thread-0.1.2", @@ -2048,6 +2059,16 @@ def crate_repositories(): build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.pin-project-lite-0.2.16.bazel"), ) + maybe( + http_archive, + name = "vendor_ts__pkg-config-0.3.32", + sha256 = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c", + type = "tar.gz", + urls = ["https://static.crates.io/crates/pkg-config/0.3.32/download"], + strip_prefix = "pkg-config-0.3.32", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.pkg-config-0.3.32.bazel"), + ) + maybe( http_archive, name = "vendor_ts__portable-atomic-1.11.0", @@ -3647,6 +3668,36 @@ def crate_repositories(): build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zerocopy-derive-0.8.20.bazel"), ) + maybe( + http_archive, + name = "vendor_ts__zstd-0.13.3", + sha256 = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a", + type = "tar.gz", + urls = ["https://static.crates.io/crates/zstd/0.13.3/download"], + strip_prefix = "zstd-0.13.3", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-0.13.3.bazel"), + ) + + maybe( + http_archive, + name = "vendor_ts__zstd-safe-7.2.4", + sha256 = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d", + type = "tar.gz", + urls = ["https://static.crates.io/crates/zstd-safe/7.2.4/download"], + strip_prefix = "zstd-safe-7.2.4", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-safe-7.2.4.bazel"), + ) + + maybe( + http_archive, + name = "vendor_ts__zstd-sys-2.0.15-zstd.1.5.7", + sha256 = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237", + type = "tar.gz", + urls = ["https://static.crates.io/crates/zstd-sys/2.0.15+zstd.1.5.7/download"], + strip_prefix = "zstd-sys-2.0.15+zstd.1.5.7", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.zstd-sys-2.0.15+zstd.1.5.7.bazel"), + ) + return [ struct(repo = "vendor_ts__anyhow-1.0.97", is_dev_dep = False), struct(repo = "vendor_ts__argfile-0.2.1", is_dev_dep = False), @@ -3698,6 +3749,7 @@ def crate_repositories(): struct(repo = "vendor_ts__tree-sitter-ruby-0.23.1", is_dev_dep = False), struct(repo = "vendor_ts__triomphe-0.1.14", is_dev_dep = False), struct(repo = "vendor_ts__ungrammar-1.16.1", is_dev_dep = False), + struct(repo = "vendor_ts__zstd-0.13.3", is_dev_dep = False), struct(repo = "vendor_ts__rand-0.9.0", is_dev_dep = True), struct(repo = "vendor_ts__tree-sitter-json-0.24.8", is_dev_dep = True), struct(repo = "vendor_ts__tree-sitter-ql-0.23.1", is_dev_dep = True), diff --git a/ruby/codeql-extractor.yml b/ruby/codeql-extractor.yml index 9b949a1a4abb..abb50db2a295 100644 --- a/ruby/codeql-extractor.yml +++ b/ruby/codeql-extractor.yml @@ -27,7 +27,7 @@ options: title: Controls compression for the TRAP files written by the extractor. description: > This option is only intended for use in debugging the extractor. Accepted - values are 'gzip' (the default, to write gzip-compressed TRAP) and 'none' - (to write uncompressed TRAP). + values are 'gzip' (the default, to write gzip-compressed TRAP) 'zstd' (to + write Zstandard-compressed TRAP) and 'none' (to write uncompressed TRAP). type: string - pattern: "^(none|gzip)$" + pattern: "^(none|gzip|zstd)$" diff --git a/ruby/ql/integration-tests/compression/methods.expected b/ruby/ql/integration-tests/compression/methods.expected new file mode 100644 index 000000000000..2bdc091c80d9 --- /dev/null +++ b/ruby/ql/integration-tests/compression/methods.expected @@ -0,0 +1 @@ +| source.rb:1:1:3:3 | f | diff --git a/ruby/ql/integration-tests/compression/methods.ql b/ruby/ql/integration-tests/compression/methods.ql new file mode 100644 index 000000000000..208d26dee633 --- /dev/null +++ b/ruby/ql/integration-tests/compression/methods.ql @@ -0,0 +1,4 @@ +import codeql.ruby.AST + +from Method m +select m diff --git a/ruby/ql/integration-tests/compression/source.rb b/ruby/ql/integration-tests/compression/source.rb new file mode 100644 index 000000000000..360eabdab756 --- /dev/null +++ b/ruby/ql/integration-tests/compression/source.rb @@ -0,0 +1,3 @@ +def f + puts "hello" +end diff --git a/ruby/ql/integration-tests/compression/test.py b/ruby/ql/integration-tests/compression/test.py new file mode 100644 index 000000000000..6976b063d3e3 --- /dev/null +++ b/ruby/ql/integration-tests/compression/test.py @@ -0,0 +1,26 @@ +import pytest + + +@pytest.mark.parametrize(("compression", "suffix"), [ + pytest.param("none", [], id="none"), + pytest.param("gzip", [".gz"], id="gzip"), + pytest.param("zstd", [".zst"], id="zstd"), +]) +def test(codeql, ruby, compression, suffix, cwd): + codeql.database.create( + _env={ + "CODEQL_EXTRACTOR_RUBY_OPTION_TRAP_COMPRESSION": compression, + } + ) + trap_files = [*(cwd / "test-db" / "trap").rglob("*.trap*")] + assert trap_files, "No trap files found" + expected_suffixes = [".trap"] + suffix + + def is_of_expected_format(file): + return file.name == "metadata.trap.gz" or \ + file.suffixes[-len(expected_suffixes):] == expected_suffixes + + files_with_wrong_format = [ + f for f in trap_files if not is_of_expected_format(f) + ] + assert not files_with_wrong_format, f"Found trap files with wrong format" diff --git a/rust/codeql-extractor.yml b/rust/codeql-extractor.yml index 0ba77ee88d13..ce876e8d3fff 100644 --- a/rust/codeql-extractor.yml +++ b/rust/codeql-extractor.yml @@ -23,10 +23,11 @@ options: title: Controls compression for the TRAP files written by the extractor. description: > This option is only intended for use in debugging the extractor. Accepted - values are 'gzip' (to write gzip-compressed TRAP) and 'none' - (currently the default, to write uncompressed TRAP). + values are 'gzip' (to write gzip-compressed TRAP) 'zstd' (to write + Zstandard-compressed TRAP) and 'none' (the default, to write uncompressed + TRAP). type: string - pattern: "^(none|gzip)$" + pattern: "^(none|gzip|zstd)$" cargo_target_dir: title: Directory to use for cargo output files. description: > diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 3f6b86d1f1f9..8124f825da3c 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -29,6 +29,7 @@ pub enum Compression { #[default] // TODO make gzip default None, Gzip, + Zstd, } impl From for trap::Compression { @@ -36,6 +37,7 @@ impl From for trap::Compression { match val { Compression::None => Self::None, Compression::Gzip => Self::Gzip, + Compression::Zstd => Self::Zstd, } } } @@ -55,7 +57,7 @@ pub struct Config { pub cargo_all_targets: bool, pub logging_flamegraph: Option, pub logging_verbosity: Option, - pub compression: Compression, + pub trap_compression: Compression, pub inputs: Vec, pub qltest: bool, pub qltest_cargo_check: bool, diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index ce739e067f0a..2206c4c067b2 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -1,4 +1,3 @@ -use crate::config::Compression; use crate::{config, generated}; use codeql_extractor::{extractor, file_paths, trap}; use ra_ap_ide_db::line_index::LineCol; @@ -9,7 +8,7 @@ use std::path::{Path, PathBuf}; use tracing::debug; pub use trap::Label as UntypedLabel; -pub use trap::Writer; +pub use trap::{Compression, Writer}; pub trait AsTrapKeyPart { fn as_key_part(&self) -> String; @@ -245,8 +244,7 @@ impl TrapFile { pub fn commit(&self) -> std::io::Result<()> { std::fs::create_dir_all(self.path.parent().unwrap())?; - self.writer - .write_to_file(&self.path, self.compression.into()) + self.writer.write_to_file(&self.path, self.compression) } } @@ -261,12 +259,16 @@ impl TrapFileProvider { std::fs::create_dir_all(&trap_dir)?; Ok(TrapFileProvider { trap_dir, - compression: cfg.compression, + compression: cfg.trap_compression.into(), }) } pub fn create(&self, category: &str, key: impl AsRef) -> TrapFile { - let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap"); + let path = file_paths::path_for( + &self.trap_dir.join(category), + key.as_ref(), + self.compression.extension(), + ); debug!("creating trap file {}", path.display()); let mut writer = trap::Writer::new(); extractor::populate_empty_location(&mut writer); diff --git a/rust/ql/integration-tests/hello-project/test_project.py b/rust/ql/integration-tests/hello-project/test_project.py index 5509bc3a93d8..7f53e78f7d49 100644 --- a/rust/ql/integration-tests/hello-project/test_project.py +++ b/rust/ql/integration-tests/hello-project/test_project.py @@ -10,7 +10,34 @@ def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_che codeql.database.create() @pytest.mark.ql_test(None) -def test_do_not_print_env(codeql, rust, cargo, check_env_not_dumped, rust_check_diagnostics): +# parametrizing `rust_edition` allows us to skip the default parametrization over all editions +@pytest.mark.parametrize("rust_edition", [2024]) +def test_do_not_print_env(codeql, rust, rust_edition, cargo, check_env_not_dumped, rust_check_diagnostics): codeql.database.create(_env={ "CODEQL_EXTRACTOR_RUST_VERBOSE": "2", }) + +@pytest.mark.ql_test("steps.ql", expected=".cargo.expected") +@pytest.mark.parametrize(("rust_edition", "compression", "suffix"), [ + pytest.param(2024, "none", [], id="none"), + pytest.param(2024, "gzip", [".gz"], id="gzip"), + pytest.param(2024, "zstd", [".zst"], id="zstd"), +]) +def test_compression(codeql, rust, rust_edition, compression, suffix, cargo, rust_check_diagnostics, cwd): + codeql.database.create( + _env={ + "CODEQL_EXTRACTOR_RUST_OPTION_TRAP_COMPRESSION": compression, + } + ) + trap_files = [*(cwd / "test-db" / "trap").rglob("*.trap*")] + assert trap_files, "No trap files found" + expected_suffixes = [".trap"] + suffix + + def is_of_expected_format(file): + return file.name == "metadata.trap.gz" or \ + file.suffixes[-len(expected_suffixes):] == expected_suffixes + + files_with_wrong_format = [ + f for f in trap_files if not is_of_expected_format(f) + ] + assert not files_with_wrong_format, f"Found trap files with wrong format" diff --git a/shared/tree-sitter-extractor/Cargo.toml b/shared/tree-sitter-extractor/Cargo.toml index 6bbda6dc83b8..3bda73a774d2 100644 --- a/shared/tree-sitter-extractor/Cargo.toml +++ b/shared/tree-sitter-extractor/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" chrono = { version = "0.4.40", features = ["serde"] } num_cpus = "1.16.0" +zstd = "0.13.3" [dev-dependencies] tree-sitter-ql = "0.23.1" diff --git a/shared/tree-sitter-extractor/src/trap.rs b/shared/tree-sitter-extractor/src/trap.rs index 4ad1e48eb6ba..a636dc885510 100644 --- a/shared/tree-sitter-extractor/src/trap.rs +++ b/shared/tree-sitter-extractor/src/trap.rs @@ -96,10 +96,17 @@ impl Writer { self.write_trap_entries(&mut trap_file) } Compression::Gzip => { - let trap_file = GzEncoder::new(trap_file, flate2::Compression::fast()); + let trap_file = GzEncoder::new(trap_file, Compression::GZIP_LEVEL); let mut trap_file = BufWriter::new(trap_file); self.write_trap_entries(&mut trap_file) } + Compression::Zstd => { + let trap_file = zstd::stream::Encoder::new(trap_file, Compression::ZSTD_LEVEL)?; + let mut trap_file = BufWriter::new(trap_file); + self.write_trap_entries(&mut trap_file)?; + trap_file.into_inner()?.finish()?; + Ok(()) + } } } @@ -107,7 +114,7 @@ impl Writer { for trap_entry in &self.trap_output { writeln!(file, "{}", trap_entry)?; } - std::io::Result::Ok(()) + Ok(()) } } @@ -280,9 +287,13 @@ fn limit_string(string: &str, max_size: usize) -> &str { pub enum Compression { None, Gzip, + Zstd, } impl Compression { + pub const ZSTD_LEVEL: i32 = 2; + pub const GZIP_LEVEL: flate2::Compression = flate2::Compression::fast(); + pub fn from_env(var_name: &str) -> Result { match std::env::var(var_name) { Ok(method) => match Compression::from_string(&method) { @@ -298,6 +309,7 @@ impl Compression { match s.to_lowercase().as_ref() { "none" => Some(Compression::None), "gzip" => Some(Compression::Gzip), + "zstd" => Some(Compression::Zstd), _ => None, } } @@ -306,6 +318,7 @@ impl Compression { match self { Compression::None => "trap", Compression::Gzip => "trap.gz", + Compression::Zstd => "trap.zst", } } }