Skip to content

Commit bdcf725

Browse files
author
DocDBrown
committed
get the features
0 parents  commit bdcf725

File tree

7 files changed

+318
-0
lines changed

7 files changed

+318
-0
lines changed

‎Readme.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Shell Script to check crates.io features
2+
## Created so you can check what features there are for the crates you need
3+
4+
1. Create a shell script that accepts crate names as input, fetches each crate’s newest version from the crates.io API, retrieves that version’s metadata, and prints the crate name if the features map is non-empty
5+
2. /api/v1/crates/{name}/versions returns version objects that include a features map. Non-empty ⇒ activatable features.
6+
3. The per-crate endpoint exposes the newest version number, which you can query directly. If absent, list versions and pick the first
7+
4. Handle HTTP errors and timeouts. crates.io enforces polite crawling
8+
5. If max_version empty, call /api/v1/crates/${crate_name}/versions and pick the first non-yanked version’s .num.
9+
6. Ensure delay is not too short for crates.io’s crawler policy.
10+
11+
## BATS files verify Shell-Script passess all tests

‎crate_feature_finder.sh‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
3+
# Base URL for the crates.io API
4+
CRATES_IO_API_BASE="https://crates.io/api/v1"
5+
# Delay between API requests to be polite (in seconds)
6+
REQUEST_DELAY=0.2
7+
8+
# Function to fetch JSON from a URL and handle basic errors
9+
# Arguments:
10+
# $1: URL to fetch
11+
# Returns:
12+
# JSON string on success, empty string on failure
13+
# Returns 0 on success, 1 on curl/network failure, 0 for 404 (handled case)
14+
fetch_json() {
15+
local url="$1"
16+
local response_and_status
17+
local http_status
18+
local response_body
19+
20+
# Use -s for silent, --max-time for timeout
21+
# -w "\n%{http_code}" appends the HTTP status code on a new line to stdout.
22+
# This helps in reliably separating the body from the status code.
23+
response_and_status=$(curl -s --max-time 10 -w "\n%{http_code}" "$url" 2>/dev/null)
24+
local curl_exit=$?
25+
26+
# Check if curl command itself failed (e.g., network error, timeout before HTTP response)
27+
if [[ $curl_exit -ne 0 ]]; then
28+
echo "Error: curl command failed for $url" >&2
29+
return 1 # Indicate failure
30+
fi
31+
32+
# Ensure we have some output to parse
33+
if [[ -z "$response_and_status" ]]; then
34+
echo "Error: curl command failed for $url" >&2
35+
return 1
36+
fi
37+
38+
# Split the response into body and status code based on the newline added by -w
39+
# The last line is the status code, everything before it is the body.
40+
http_status=$(echo -n "$response_and_status" | tail -n 1)
41+
response_body=$(echo -n "$response_and_status" | head -n -1)
42+
43+
# Handle 2xx success, 404 as handled empty, and other statuses as errors.
44+
if [[ "$http_status" =~ ^2[0-9][0-9]$ ]]; then
45+
echo "$response_body"
46+
sleep "$REQUEST_DELAY" # Be polite
47+
return 0
48+
elif [[ "$http_status" == "404" ]]; then
49+
# Crate or version not found. This is a handled case, not necessarily an error for the script's logic.
50+
echo ""
51+
sleep "$REQUEST_DELAY"
52+
return 0
53+
else
54+
echo "Error: Failed to fetch $url. HTTP Status: $http_status. Response: ${response_body:0:100}..." >&2
55+
return 1 # Indicate failure
56+
fi
57+
}
58+
59+
# Only run main when executed directly, not when sourced by tests
60+
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
61+
# Main script logic
62+
if [[ "\$#" -eq 0 ]]; then
63+
echo "Usage: $0 <crate_name_1> [crate_name_2] ..." >&2
64+
exit 1
65+
fi
66+
67+
for crate_name in "\$@"; do
68+
# 1. Try to get max_version from /api/v1/crates/{name}
69+
CRATE_INFO_URL="${CRATES_IO_API_BASE}/crates/${crate_name}"
70+
crate_info_json=$(fetch_json "$CRATE_INFO_URL")
71+
72+
if [[ $? -ne 0 ]]; then # Check if fetch_json itself failed (e.g., curl error)
73+
echo "Skipping '$crate_name': Failed to retrieve crate info due to network/curl error." >&2
74+
continue
75+
fi
76+
77+
target_version=""
78+
if [[ -n "$crate_info_json" ]]; then
79+
max_version=$(echo "$crate_info_json" | jq -r '.crate.max_version // empty')
80+
if [[ -n "$max_version" ]]; then
81+
target_version="$max_version"
82+
fi
83+
fi
84+
85+
if [[ -z "$target_version" ]]; then
86+
# 2. If max_version is empty or not found, fetch all versions and pick the first non-yanked one
87+
VERSIONS_URL="${CRATES_IO_API_BASE}/crates/${crate_name}/versions"
88+
versions_json=$(fetch_json "$VERSIONS_URL")
89+
90+
if [[ $? -ne 0 ]]; then
91+
echo "Skipping '$crate_name': Failed to retrieve versions list due to network/curl error." >&2
92+
continue
93+
fi
94+
95+
if [[ -n "$versions_json" ]]; then
96+
# Find the first non-yanked version number
97+
target_version=$(echo "$versions_json" | jq -r '.versions[] | select(.yanked == false) | .num' | head -n 1)
98+
fi
99+
100+
if [[ -z "$target_version" ]]; then
101+
echo "Skipping '$crate_name': No non-yanked versions found or could not retrieve versions." >&2
102+
continue
103+
fi
104+
fi
105+
106+
# 3. Now that we have a target_version, fetch its metadata and check features
107+
VERSION_METADATA_URL="${CRATES_IO_API_BASE}/crates/${crate_name}/${target_version}"
108+
version_metadata_json=$(fetch_json "$VERSION_METADATA_URL")
109+
110+
if [[ $? -ne 0 ]]; then
111+
echo "Skipping '$crate_name': Failed to retrieve metadata for version '$target_version' due to network/curl error." >&2
112+
continue
113+
fi
114+
115+
if [[ -z "$version_metadata_json" ]]; then
116+
echo "Skipping '$crate_name': Could not retrieve metadata for version '$target_version' (empty response)." >&2
117+
continue
118+
fi
119+
120+
# Check if the features map is non-empty
121+
feature_count=$(echo "$version_metadata_json" | jq -r '.version.features | to_entries | length // 0')
122+
123+
if [[ "$feature_count" -gt 0 ]]; then
124+
echo "$crate_name"
125+
fi
126+
done
127+
fi

‎crate_feature_finder_batch_0_test.bats‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env bats
2+
3+
setup() {
4+
SCRIPT="${BATS_TEST_DIRNAME}/crate_feature_finder.sh"
5+
}
6+
7+
start_server_batch_0() {
8+
code="$1"
9+
body="$2"
10+
tmpdir=$(mktemp -d)
11+
server_py="$tmpdir/server.py"
12+
cat > "$server_py" <<'PY'
13+
import sys,http.server,socketserver
14+
CODE=int(sys.argv[1])
15+
BODY=sys.argv[2]
16+
class Handler(http.server.BaseHTTPRequestHandler):
17+
def do_GET(self):
18+
self.send_response(CODE)
19+
self.send_header("Content-Type","text/plain")
20+
self.end_headers()
21+
self.wfile.write(BODY.encode())
22+
def log_message(self, format, *args):
23+
return
24+
with socketserver.TCPServer(("127.0.0.1",0), Handler) as httpd:
25+
print(httpd.server_address[1])
26+
sys.stdout.flush()
27+
httpd.serve_forever()
28+
PY
29+
python3 "$server_py" "$code" "$body" >"$tmpdir/port" 2>"$tmpdir/err" &
30+
server_pid=$!
31+
for i in $(seq 1 50); do
32+
if [[ -s "$tmpdir/port" ]]; then break; fi
33+
sleep 0.01
34+
done
35+
port=$(cat "$tmpdir/port")
36+
echo "$port:$server_pid:$tmpdir"
37+
}
38+
39+
@test "fetch_json_returns_body_and_status_0_for_200_response" {
40+
result=$(start_server_batch_0 200 '{"ok":true}')
41+
IFS=: read port server_pid tmpdir <<< "$result"
42+
url="http://127.0.0.1:${port}/"
43+
run bash -c "source \"$SCRIPT\" && fetch_json \"$url\""
44+
[ "$status" -eq 0 ]
45+
[ "$output" = '{"ok":true}' ]
46+
kill "$server_pid"
47+
wait "$server_pid" 2>/dev/null || true
48+
rm -rf "$tmpdir"
49+
}
50+
51+
@test "fetch_json_returns_empty_string_and_status_0_for_404_response" {
52+
result=$(start_server_batch_0 404 'Not Found')
53+
IFS=: read port server_pid tmpdir <<< "$result"
54+
url="http://127.0.0.1:${port}/"
55+
run bash -c "source \"$SCRIPT\" && fetch_json \"$url\""
56+
[ "$status" -eq 0 ]
57+
[ -z "$output" ]
58+
kill "$server_pid"
59+
wait "$server_pid" 2>/dev/null || true
60+
rm -rf "$tmpdir"
61+
}
62+
63+
@test "fetch_json_prints_curl_error_and_returns_1_on_network_failure" {
64+
run bash -c "source \"$SCRIPT\" && fetch_json 'http://127.0.0.1:9/'"
65+
[ "$status" -eq 1 ]
66+
[[ "$output" == *"Error: curl command failed for http://127.0.0.1:9/"* ]]
67+
}
68+
69+
@test "fetch_json_prints_http_error_and_returns_1_for_500_response" {
70+
result=$(start_server_batch_0 500 'Internal Server Error')
71+
IFS=: read port server_pid tmpdir <<< "$result"
72+
url="http://127.0.0.1:${port}/"
73+
run bash -c "source \"$SCRIPT\" && fetch_json \"$url\""
74+
[ "$status" -eq 1 ]
75+
[[ "$output" == *"HTTP Status: 500"* ]]
76+
kill "$server_pid"
77+
wait "$server_pid" 2>/dev/null || true
78+
rm -rf "$tmpdir"
79+
}
80+
81+
@test "fetch_json_truncates_error_body_to_100_chars_for_non_2xx_non_404_response" {
82+
long_body=$(printf 'A%.0s' $(seq 1 150))
83+
result=$(start_server_batch_0 500 "$long_body")
84+
IFS=: read port server_pid tmpdir <<< "$result"
85+
url="http://127.0.0.1:${port}/"
86+
expected_prefix=$(printf 'A%.0s' $(seq 1 100))
87+
run bash -c "source \"$SCRIPT\" && fetch_json \"$url\""
88+
[ "$status" -eq 1 ]
89+
[[ "$output" == *"${expected_prefix}..."* ]]
90+
kill "$server_pid"
91+
wait "$server_pid" 2>/dev/null || true
92+
rm -rf "$tmpdir"
93+
}

‎crate_feature_finder_batch_1_test.bats‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env bats
2+
3+
setup() {
4+
TMPDIR=$(mktemp -d)
5+
}
6+
7+
teardown() {
8+
if [[ -n "$TMPDIR" && -d "$TMPDIR" ]]; then
9+
rm -rf "$TMPDIR"
10+
fi
11+
}
12+
13+
write_mock_curl_batch_1() {
14+
cat > "$TMPDIR/curl" <<'MOCK'
15+
#!/bin/bash
16+
if [[ "$MOCK_CURL_BEHAVIOR" == "success" ]]; then
17+
printf 'MOCKBODY\n200'
18+
exit 0
19+
elif [[ "$MOCK_CURL_BEHAVIOR" == "fail" ]]; then
20+
exit 7
21+
else
22+
printf 'ERRORBODY\n500'
23+
exit 0
24+
fi
25+
MOCK
26+
chmod +x "$TMPDIR/curl"
27+
}
28+
29+
@test "fetch_json_sleeps_after_successful_api_call" {
30+
write_mock_curl_batch_1
31+
PATH="$TMPDIR:$PATH"
32+
export MOCK_CURL_BEHAVIOR=success
33+
SCRIPT="${BATS_TEST_DIRNAME}/crate_feature_finder.sh"
34+
start_ns=$(date +%s%N)
35+
run bash -c "source \"$SCRIPT\" >/dev/null 2>&1; fetch_json 'http://example' >/dev/null 2>&1"
36+
status=$status
37+
end_ns=$(date +%s%N)
38+
elapsed_ms=$(( (end_ns - start_ns) / 1000000 ))
39+
[ "$status" -eq 0 ]
40+
if [ "$elapsed_ms" -lt 150 ]; then
41+
echo "fetch_json did not sleep long enough: ${elapsed_ms}ms" >&2
42+
false
43+
fi
44+
}
45+
46+
@test "fetch_json_does_not_sleep_on_error_exit" {
47+
write_mock_curl_batch_1
48+
PATH="$TMPDIR:$PATH"
49+
export MOCK_CURL_BEHAVIOR=fail
50+
SCRIPT="${BATS_TEST_DIRNAME}/crate_feature_finder.sh"
51+
start_ns=$(date +%s%N)
52+
run bash -c "source \"$SCRIPT\" >/dev/null 2>&1; fetch_json 'http://example' >/dev/null 2>&1"
53+
status=$status
54+
end_ns=$(date +%s%N)
55+
elapsed_ms=$(( (end_ns - start_ns) / 1000000 ))
56+
[ "$status" -ne 0 ]
57+
if [ "$elapsed_ms" -ge 100 ]; then
58+
echo "fetch_json slept on error path: ${elapsed_ms}ms" >&2
59+
false
60+
fi
61+
}

‎crate_feature_finder_test_names.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"all_test_names": [
3+
"fetch_json_returns_body_and_status_0_for_200_response",
4+
"fetch_json_returns_empty_string_and_status_0_for_404_response",
5+
"fetch_json_prints_curl_error_and_returns_1_on_network_failure",
6+
"fetch_json_prints_http_error_and_returns_1_for_500_response",
7+
"fetch_json_truncates_error_body_to_100_chars_for_non_2xx_non_404_response",
8+
"fetch_json_sleeps_after_successful_api_call",
9+
"fetch_json_does_not_sleep_on_error_exit"
10+
],
11+
"tests_per_file": {
12+
"crate_feature_finder_batch_0_test.bats": [
13+
"fetch_json_returns_body_and_status_0_for_200_response",
14+
"fetch_json_returns_empty_string_and_status_0_for_404_response",
15+
"fetch_json_prints_curl_error_and_returns_1_on_network_failure",
16+
"fetch_json_prints_http_error_and_returns_1_for_500_response",
17+
"fetch_json_truncates_error_body_to_100_chars_for_non_2xx_non_404_response"
18+
],
19+
"crate_feature_finder_batch_1_test.bats": [
20+
"fetch_json_sleeps_after_successful_api_call",
21+
"fetch_json_does_not_sleep_on_error_exit"
22+
]
23+
}
24+
}

‎test/libs/bats-assert‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit f1e9280eaae8f86cbe278a687e6ba755bc802c1a

‎test/libs/bats-support‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit 0954abb9925cad550424cebca2b99255d4eabe96

0 commit comments

Comments
 (0)