diff --git a/docs/api/query.rst b/docs/api/query.rst index 22616007..70a29bf3 100644 --- a/docs/api/query.rst +++ b/docs/api/query.rst @@ -231,3 +231,23 @@ MultiVectorQuery :inherited-members: :show-inheritance: :exclude-members: add_filter,get_args,highlight,return_field,summarize + + +SQLQuery +======== + +.. currentmodule:: redisvl.query + + +.. autoclass:: SQLQuery + :members: + :show-inheritance: + +.. note:: + SQLQuery requires the optional ``sql-redis`` package. Install with: + ``pip install redisvl[sql-redis]`` + +.. note:: + SQLQuery translates SQL SELECT statements into Redis FT.SEARCH or FT.AGGREGATE commands. + The SQL syntax supports WHERE clauses, field selection, ordering, and parameterized queries + for vector similarity searches. diff --git a/docs/user_guide/01_getting_started.ipynb b/docs/user_guide/01_getting_started.ipynb index 13bb7062..d7ae9ff7 100644 --- a/docs/user_guide/01_getting_started.ipynb +++ b/docs/user_guide/01_getting_started.ipynb @@ -81,14 +81,14 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "schema = {\n", " \"index\": {\n", " \"name\": \"user_simple\",\n", - " \"prefix\": \"user_simple_docs\",\n", + " \"prefix\": [\"prefix_1\", \"prefix_2\"],\n", " },\n", " \"fields\": [\n", " {\"name\": \"user\", \"type\": \"tag\"},\n", @@ -290,21 +290,21 @@ "\n", "\n", "Index Information:\n", - "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", - "\u2502 Index Name \u2502 Storage Type \u2502 Prefixes \u2502 Index Options \u2502 Indexing \u2502\n", - "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", + "╭──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────╮\n", + "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", + "├──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┤\n", "| user_simple | HASH | ['user_simple_docs'] | [] | 0 |\n", - "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n", + "╰──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────╯\n", "Index Fields:\n", - "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", - "\u2502 Name \u2502 Attribute \u2502 Type \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502\n", - "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", - "\u2502 user \u2502 user \u2502 TAG \u2502 SEPARATOR \u2502 , \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", - "\u2502 credit_score \u2502 credit_score \u2502 TAG \u2502 SEPARATOR \u2502 , \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", - "\u2502 job \u2502 job \u2502 TEXT \u2502 WEIGHT \u2502 1 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", - "\u2502 age \u2502 age \u2502 NUMERIC \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", - "\u2502 user_embedding \u2502 user_embedding \u2502 VECTOR \u2502 algorithm \u2502 FLAT \u2502 data_type \u2502 FLOAT32 \u2502 dim \u2502 3 \u2502 distance_metric \u2502 COSINE \u2502\n", - "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n" + "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", + "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", + "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", + "│ user │ user │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ job │ job │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ age │ age │ NUMERIC │ │ │ │ │ │ │ │ │\n", + "│ user_embedding │ user_embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 3 │ distance_metric │ COSINE │\n", + "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" ] } ], @@ -678,29 +678,29 @@ "text": [ "\n", "Statistics:\n", - "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", - "\u2502 Stat Key \u2502 Value \u2502\n", - "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", - "\u2502 num_docs \u2502 10 \u2502\n", - "\u2502 num_terms \u2502 0 \u2502\n", - "\u2502 max_doc_id \u2502 10 \u2502\n", - "\u2502 num_records \u2502 50 \u2502\n", - "\u2502 percent_indexed \u2502 1 \u2502\n", - "\u2502 hash_indexing_failures \u2502 0 \u2502\n", - "\u2502 number_of_uses \u2502 2 \u2502\n", - "\u2502 bytes_per_record_avg \u2502 19.5200004 \u2502\n", - "\u2502 doc_table_size_mb \u2502 0.00105857 \u2502\n", - "\u2502 inverted_sz_mb \u2502 9.30786132 \u2502\n", - "\u2502 key_table_size_mb \u2502 4.70161437 \u2502\n", - "\u2502 offset_bits_per_record_avg \u2502 nan \u2502\n", - "\u2502 offset_vectors_sz_mb \u2502 0 \u2502\n", - "\u2502 offsets_per_term_avg \u2502 0 \u2502\n", - "\u2502 records_per_doc_avg \u2502 5 \u2502\n", - "\u2502 sortable_values_size_mb \u2502 0 \u2502\n", - "\u2502 total_indexing_time \u2502 0.16899999 \u2502\n", - "\u2502 total_inverted_index_blocks \u2502 11 \u2502\n", - "\u2502 vector_index_sz_mb \u2502 0.23619842 \u2502\n", - "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n" + "╭─────────────────────────────┬────────────╮\n", + "│ Stat Key │ Value │\n", + "├─────────────────────────────┼────────────┤\n", + "│ num_docs │ 10 │\n", + "│ num_terms │ 0 │\n", + "│ max_doc_id │ 10 │\n", + "│ num_records │ 50 │\n", + "│ percent_indexed │ 1 │\n", + "│ hash_indexing_failures │ 0 │\n", + "│ number_of_uses │ 2 │\n", + "│ bytes_per_record_avg │ 19.5200004 │\n", + "│ doc_table_size_mb │ 0.00105857 │\n", + "│ inverted_sz_mb │ 9.30786132 │\n", + "│ key_table_size_mb │ 4.70161437 │\n", + "│ offset_bits_per_record_avg │ nan │\n", + "│ offset_vectors_sz_mb │ 0 │\n", + "│ offsets_per_term_avg │ 0 │\n", + "│ records_per_doc_avg │ 5 │\n", + "│ sortable_values_size_mb │ 0 │\n", + "│ total_indexing_time │ 0.16899999 │\n", + "│ total_inverted_index_blocks │ 11 │\n", + "│ vector_index_sz_mb │ 0.23619842 │\n", + "╰─────────────────────────────┴────────────╯\n" ] } ], @@ -800,4 +800,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/docs/user_guide/02_hybrid_queries.ipynb b/docs/user_guide/02_hybrid_queries.ipynb index e7f8d225..b76f0c51 100644 --- a/docs/user_guide/02_hybrid_queries.ipynb +++ b/docs/user_guide/02_hybrid_queries.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 94, "metadata": {}, "outputs": [ { @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 96, "metadata": {}, "outputs": [], "source": [ @@ -92,18 +92,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 52, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "13:00:56 [RedisVL] INFO Indices:\n", - "13:00:56 [RedisVL] INFO 1. user_queries\n" - ] - } - ], + "outputs": [], "source": [ "# use the CLI to see the created index\n", "!rvl index listall" @@ -111,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 97, "metadata": {}, "outputs": [], "source": [ @@ -121,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 98, "metadata": {}, "outputs": [ { @@ -130,7 +121,7 @@ "7" ] }, - "execution_count": 6, + "execution_count": 98, "metadata": {}, "output_type": "execute_result" } @@ -160,13 +151,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 99, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
" ], "text/plain": [ "" @@ -174,6 +165,16 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "'@credit_score:{high}=>[KNN 10 @user_embedding $vector AS vector_distance] RETURN 7 user credit_score age job office_location last_updated vector_distance SORTBY vector_distance ASC DIALECT 2 LIMIT 0 10'" + ] + }, + "execution_count": 99, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -190,18 +191,39 @@ ")\n", "\n", "results = index.query(v)\n", - "result_print(results)" + "result_print(results)\n", + "str(v)" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'@credit_score:{high}=>[KNN 10 @user_embedding $vector AS vector_distance]'" + ] + }, + "execution_count": 100, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "v.query_string()" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0derricklow14doctor-122.4194,37.77491741627789
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0derricklow14doctor-122.4194,37.77491741627789
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -242,13 +264,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 57, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -268,13 +290,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 58, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -305,13 +327,13 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 59, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -340,13 +362,13 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -367,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -393,13 +415,13 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -428,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 63, "metadata": {}, "outputs": [ { @@ -441,7 +463,7 @@ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -466,7 +488,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 64, "metadata": {}, "outputs": [ { @@ -479,7 +501,7 @@ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
" ], "text/plain": [ "" @@ -505,7 +527,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 65, "metadata": {}, "outputs": [ { @@ -518,7 +540,7 @@ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
" ], "text/plain": [ "" @@ -554,7 +576,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 66, "metadata": {}, "outputs": [ { @@ -582,13 +604,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -608,7 +630,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -634,7 +656,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 69, "metadata": {}, "outputs": [ { @@ -660,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 70, "metadata": {}, "outputs": [ { @@ -686,13 +708,13 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 71, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
vector_distanceusercredit_scoreagejoboffice_locationlast_updated
0johnhigh18engineer-122.4194,37.77491741627789
0derricklow14doctor-122.4194,37.77491741627789
0.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -719,14 +741,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 72, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'id': 'user_queries_docs:01JY4J5VC91SV4C91BM4D0FCV2',\n", - " 'score': 0.9090908893868948,\n", + "[{'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90PY',\n", + " 'score': 1.8181817787737895,\n", " 'vector_distance': '0',\n", " 'user': 'john',\n", " 'credit_score': 'high',\n", @@ -734,7 +756,7 @@ " 'job': 'engineer',\n", " 'office_location': '-122.4194,37.7749',\n", " 'last_updated': '1741627789'},\n", - " {'id': 'user_queries_docs:01JY4J5VC90DRSFJ0WKXXN49JT',\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90PZ',\n", " 'score': 0.0,\n", " 'vector_distance': '0',\n", " 'user': 'derrick',\n", @@ -743,8 +765,8 @@ " 'job': 'doctor',\n", " 'office_location': '-122.4194,37.7749',\n", " 'last_updated': '1741627789'},\n", - " {'id': 'user_queries_docs:01JY4J5VC9QTPMCD60YP40Q6PW',\n", - " 'score': 0.9090908893868948,\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q1',\n", + " 'score': 1.8181817787737895,\n", " 'vector_distance': '0.109129190445',\n", " 'user': 'tyler',\n", " 'credit_score': 'high',\n", @@ -752,25 +774,25 @@ " 'job': 'engineer',\n", " 'office_location': '-122.0839,37.3861',\n", " 'last_updated': '1742232589'},\n", - " {'id': 'user_queries_docs:01JY4J5VC9FW7QQNJKDJ4Z7PRG',\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q2',\n", " 'score': 0.0,\n", - " 'vector_distance': '0.158808946609',\n", + " 'vector_distance': '0.158808887005',\n", " 'user': 'tim',\n", " 'credit_score': 'high',\n", " 'age': '12',\n", " 'job': 'dermatologist',\n", " 'office_location': '-122.0839,37.3861',\n", " 'last_updated': '1739644189'},\n", - " {'id': 'user_queries_docs:01JY4J5VC940DJ9F47EJ6KN2MH',\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q3',\n", " 'score': 0.0,\n", - " 'vector_distance': '0.217882037163',\n", + " 'vector_distance': '0.217881977558',\n", " 'user': 'taimur',\n", " 'credit_score': 'low',\n", " 'age': '15',\n", " 'job': 'CEO',\n", " 'office_location': '-122.0839,37.3861',\n", " 'last_updated': '1742232589'},\n", - " {'id': 'user_queries_docs:01JY4J5VC9D53KQD7ZTRP14KCE',\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q0',\n", " 'score': 0.0,\n", " 'vector_distance': '0.266666650772',\n", " 'user': 'nancy',\n", @@ -779,7 +801,7 @@ " 'job': 'doctor',\n", " 'office_location': '-122.4194,37.7749',\n", " 'last_updated': '1710696589'},\n", - " {'id': 'user_queries_docs:01JY4J5VC9806MD90GBZNP0MNY',\n", + " {'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q4',\n", " 'score': 0.0,\n", " 'vector_distance': '0.653301358223',\n", " 'user': 'joe',\n", @@ -790,7 +812,7 @@ " 'last_updated': '1742232589'}]" ] }, - "execution_count": 24, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -813,7 +835,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -841,13 +863,13 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 74, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
scorevector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.45454544469344740johnhigh18engineer-122.4194,37.77491741627789
0.45454544469344740derricklow14doctor-122.4194,37.77491741627789
0.45454544469344740.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.45454544469344740.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.45454544469344740.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.45454544469344740.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.45454544469344740.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
scorevector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.45454544469344740johnhigh18engineer-122.4194,37.77491741627789
0.45454544469344740derricklow14doctor-122.4194,37.77491741627789
0.45454544469344740.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.45454544469344740.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.45454544469344740.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.45454544469344740.266666650772nancyhigh94doctor-122.4194,37.77491710696589
0.45454544469344740.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -867,13 +889,13 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 75, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
scorevector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.00.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.00.158808946609timhigh12dermatologist-122.0839,37.38611739644189
0.00.217882037163taimurlow15CEO-122.0839,37.38611742232589
0.00.653301358223joemedium35dentist-122.0839,37.38611742232589
" + "
scorevector_distanceusercredit_scoreagejoboffice_locationlast_updated
0.00.109129190445tylerhigh100engineer-122.0839,37.38611742232589
0.00.158808887005timhigh12dermatologist-122.0839,37.38611739644189
0.00.217881977558taimurlow15CEO-122.0839,37.38611742232589
0.00.653301358223joemedium35dentist-122.0839,37.38611742232589
" ], "text/plain": [ "" @@ -904,7 +926,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -948,13 +970,13 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 77, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_location
0derricklow14doctor-122.4194,37.7749
0.109129190445tylerhigh100engineer-122.0839,37.3861
0.158808946609timhigh12dermatologist-122.0839,37.3861
0.217882037163taimurlow15CEO-122.0839,37.3861
0.266666650772nancyhigh94doctor-122.4194,37.7749
" + "
vector_distanceusercredit_scoreagejoboffice_location
0derricklow14doctor-122.4194,37.7749
0.109129190445tylerhigh100engineer-122.0839,37.3861
0.158808887005timhigh12dermatologist-122.0839,37.3861
0.217881977558taimurlow15CEO-122.0839,37.3861
0.266666650772nancyhigh94doctor-122.4194,37.7749
" ], "text/plain": [ "" @@ -992,7 +1014,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 78, "metadata": {}, "outputs": [], "source": [ @@ -1007,7 +1029,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -1032,7 +1054,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -1057,7 +1079,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 81, "metadata": {}, "outputs": [ { @@ -1082,13 +1104,13 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 82, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejoboffice_location
0johnhigh18engineer-122.4194,37.7749
0derricklow14doctor-122.4194,37.7749
0.109129190445tylerhigh100engineer-122.0839,37.3861
0.158808946609timhigh12dermatologist-122.0839,37.3861
0.217882037163taimurlow15CEO-122.0839,37.3861
0.266666650772nancyhigh94doctor-122.4194,37.7749
0.653301358223joemedium35dentist-122.0839,37.3861
" + "
vector_distanceusercredit_scoreagejoboffice_location
0johnhigh18engineer-122.4194,37.7749
0derricklow14doctor-122.4194,37.7749
0.109129190445tylerhigh100engineer-122.0839,37.3861
0.158808887005timhigh12dermatologist-122.0839,37.3861
0.217881977558taimurlow15CEO-122.0839,37.3861
0.266666650772nancyhigh94doctor-122.4194,37.7749
0.653301358223joemedium35dentist-122.0839,37.3861
" ], "text/plain": [ "" @@ -1116,7 +1138,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 83, "metadata": {}, "outputs": [ { @@ -1158,7 +1180,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 84, "metadata": {}, "outputs": [ { @@ -1192,13 +1214,13 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 85, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
vector_distanceusercredit_scoreagejob
0johnhigh18engineer
0derricklow14doctor
0.109129190445tylerhigh100engineer
0.158808946609timhigh12dermatologist
" + "
vector_distanceusercredit_scoreagejob
0johnhigh18engineer
0derricklow14doctor
0.109129190445tylerhigh100engineer
0.158808887005timhigh12dermatologist
" ], "text/plain": [ "" @@ -1233,7 +1255,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 86, "metadata": {}, "outputs": [ { @@ -1264,7 +1286,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 87, "metadata": {}, "outputs": [ { @@ -1304,7 +1326,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 88, "metadata": {}, "outputs": [ { @@ -1345,7 +1367,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 89, "metadata": {}, "outputs": [ { @@ -1354,7 +1376,7 @@ "'@job:(\"engineer\")=>[KNN 5 @user_embedding $vector AS vector_distance] RETURN 6 user credit_score age job office_location vector_distance SORTBY age DESC DIALECT 3 LIMIT 0 5'" ] }, - "execution_count": 41, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -1366,7 +1388,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 90, "metadata": {}, "outputs": [ { @@ -1375,7 +1397,7 @@ "'@credit_score:{high}'" ] }, - "execution_count": 42, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -1388,7 +1410,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 91, "metadata": {}, "outputs": [ { @@ -1397,7 +1419,7 @@ "'((@credit_score:{high} @age:[18 +inf]) @age:[-inf 100])'" ] }, - "execution_count": 43, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" } @@ -1422,17 +1444,17 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 92, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{'id': 'user_queries_docs:01JY4J5VC91SV4C91BM4D0FCV2', 'payload': None, 'user': 'john', 'age': '18', 'job': 'engineer', 'credit_score': 'high', 'office_location': '-122.4194,37.7749', 'user_embedding': '==\\x00\\x00\\x00?', 'last_updated': '1741627789'}\n", - "{'id': 'user_queries_docs:01JY4J5VC9D53KQD7ZTRP14KCE', 'payload': None, 'user': 'nancy', 'age': '94', 'job': 'doctor', 'credit_score': 'high', 'office_location': '-122.4194,37.7749', 'user_embedding': '333?=\\x00\\x00\\x00?', 'last_updated': '1710696589'}\n", - "{'id': 'user_queries_docs:01JY4J5VC9QTPMCD60YP40Q6PW', 'payload': None, 'user': 'tyler', 'age': '100', 'job': 'engineer', 'credit_score': 'high', 'office_location': '-122.0839,37.3861', 'user_embedding': '=>\\x00\\x00\\x00?', 'last_updated': '1742232589'}\n", - "{'id': 'user_queries_docs:01JY4J5VC9FW7QQNJKDJ4Z7PRG', 'payload': None, 'user': 'tim', 'age': '12', 'job': 'dermatologist', 'credit_score': 'high', 'office_location': '-122.0839,37.3861', 'user_embedding': '>>\\x00\\x00\\x00?', 'last_updated': '1739644189'}\n" + "{'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90PY', 'payload': None, 'user': 'john', 'age': '18', 'job': 'engineer', 'credit_score': 'high', 'office_location': '-122.4194,37.7749', 'user_embedding': '==\\x00\\x00\\x00?', 'last_updated': '1741627789'}\n", + "{'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q0', 'payload': None, 'user': 'nancy', 'age': '94', 'job': 'doctor', 'credit_score': 'high', 'office_location': '-122.4194,37.7749', 'user_embedding': '333?=\\x00\\x00\\x00?', 'last_updated': '1710696589'}\n", + "{'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q1', 'payload': None, 'user': 'tyler', 'age': '100', 'job': 'engineer', 'credit_score': 'high', 'office_location': '-122.0839,37.3861', 'user_embedding': '=>\\x00\\x00\\x00?', 'last_updated': '1742232589'}\n", + "{'id': 'user_queries_docs:01KG0AV1K9NY8H2BBKRSRZ90Q2', 'payload': None, 'user': 'tim', 'age': '12', 'job': 'dermatologist', 'credit_score': 'high', 'office_location': '-122.0839,37.3861', 'user_embedding': '>>\\x00\\x00\\x00?', 'last_updated': '1739644189'}\n" ] } ], @@ -1444,7 +1466,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 93, "metadata": {}, "outputs": [], "source": [ @@ -1455,7 +1477,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "redisvl", "language": "python", "name": "python3" }, @@ -1469,10 +1491,10 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.8" + "version": "3.11.9" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/docs/user_guide/12_sql_to_redis_queries.ipynb b/docs/user_guide/12_sql_to_redis_queries.ipynb new file mode 100644 index 00000000..cf971de0 --- /dev/null +++ b/docs/user_guide/12_sql_to_redis_queries.ipynb @@ -0,0 +1,1074 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SQLQuery class\n", + "\n", + "It may arise that you want to use SQL-like queries to interact with your Redis vector database. While Redis does not natively support SQL, the `redisvl` library provides a `SQLQuery` class that allows you to write SQL-like queries that are automatically translated into Redis queries.\n", + "\n", + "The `SQLQuery` class is a wrapper around the [`sql-redis`](https://pypi.org/project/sql-redis/) package, which provides a SQL-to-Redis query translator. The `sql-redis` package is not installed by default with `redisvl`, so you will need to install with the optional syntax:\n", + "\n", + "`pip install \"redisvl[sql-redis]\"` or, if running locally, you can `uv sync --all-extras --all-groups`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create an index to search" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from redisvl.utils.vectorize import HFTextVectorizer\n", + "\n", + "hf = HFTextVectorizer()\n", + "\n", + "schema = {\n", + " \"index\": {\n", + " \"name\": \"user_simple\",\n", + " \"prefix\": \"user_simple_docs\",\n", + " \"storage_type\": \"json\",\n", + " \"prefix\": [\"prefix_1\", \"prefix_2\"]\n", + " },\n", + " \"fields\": [\n", + " {\"name\": \"user\", \"type\": \"tag\"},\n", + " {\"name\": \"region\", \"type\": \"tag\"},\n", + " {\"name\": \"job\", \"type\": \"tag\"},\n", + " {\"name\": \"job_description\", \"type\": \"text\"},\n", + " {\"name\": \"age\", \"type\": \"numeric\"},\n", + " {\n", + " \"name\": \"job_embedding\",\n", + " \"type\": \"vector\",\n", + " \"attrs\": {\n", + " \"dims\": len(hf.embed(\"get embed length\")),\n", + " \"distance_metric\": \"cosine\",\n", + " \"algorithm\": \"flat\",\n", + " \"datatype\": \"float32\"\n", + " }\n", + " }\n", + " ]\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create sample dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = [\n", + " {\n", + " 'user': 'john',\n", + " 'age': 34,\n", + " 'job': 'software engineer',\n", + " 'region': 'us-west',\n", + " 'job_description': 'Designs, develops, and maintains software applications and systems.'\n", + " },\n", + " {\n", + " 'user': 'bill',\n", + " 'age': 54,\n", + " 'job': 'engineer',\n", + " 'region': 'us-central',\n", + " 'job_description': 'Applies scientific and mathematical principles to solve technical problems.'\n", + " },\n", + " {\n", + " 'user': 'mary',\n", + " 'age': 24,\n", + " 'job': 'doctor',\n", + " 'region': 'us-central',\n", + " 'job_description': 'Diagnoses and treats illnesses, injuries, and other medical conditions in the healthcare field.'\n", + " },\n", + " {\n", + " 'user': 'joe',\n", + " 'age': 27,\n", + " 'job': 'dentist',\n", + " 'region': 'us-east',\n", + " 'job_description': 'Provides oral healthcare including diagnosing and treating teeth and gum issues.'\n", + " },\n", + " {\n", + " 'user': 'stacy',\n", + " 'age': 61,\n", + " 'job': 'project manager',\n", + " 'region': 'us-west',\n", + " 'job_description': 'Plans, organizes, and oversees projects from inception to completion.'\n", + " }\n", + "]\n", + "\n", + "data = [\n", + " { \n", + " **d,\n", + " \"job_embedding\": hf.embed(f\"{d['job_description']=} {d['job']=}\"),\n", + " } \n", + " for d in data\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_2 = [\n", + " {\n", + " 'prefix': 'prefix_2',\n", + " 'user': 'guy',\n", + " 'age': 34,\n", + " 'job': 'software engineer',\n", + " 'region': 'us-west',\n", + " 'job_description': 'Designs, develops, and maintains software applications and systems.'\n", + " }]\n", + "\n", + "data_2 = [\n", + " { \n", + " **d,\n", + " \"job_embedding\": hf.embed(f\"{d['job_description']=} {d['job']=}\"),\n", + " } \n", + " for d in data_2\n", + "]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a `SearchIndex`\n", + "\n", + "With the schema and sample dataset ready, create a `SearchIndex`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bring your own Redis connection instance\n", + "\n", + "This is ideal in scenarios where you have custom settings on the connection instance or if your application will share a connection pool:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "from redisvl.index import SearchIndex\n", + "from redis import Redis\n", + "\n", + "client = Redis.from_url(\"redis://localhost:6379\")\n", + "index = SearchIndex.from_dict(schema, redis_client=client, validate_on_load=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'OK'" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.execute_command(\n", + " \"FT.CREATE\", \"user_simple\",\n", + " \"ON\", \"JSON\",\n", + " \"PREFIX\", \"2\", \"prefix_1\", \"prefix_2\",\n", + " \"SCHEMA\",\n", + " \"$.user\", \"AS\", \"user\", \"TAG\",\n", + " \"$.region\", \"AS\", \"region\", \"TAG\",\n", + " \"$.job\", \"AS\", \"job\", \"TAG\",\n", + " \"$.job_description\", \"AS\", \"job_description\", \"TEXT\",\n", + " \"$.age\", \"AS\", \"age\", \"NUMERIC\",\n", + " \"$.job_embedding\", \"AS\", \"job_embedding\", \"VECTOR\", \"FLAT\", \"6\",\n", + " \"TYPE\", \"FLOAT32\",\n", + " \"DIM\", \"768\",\n", + " \"DISTANCE_METRIC\", \"COSINE\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Let the index manage the connection instance\n", + "\n", + "This is ideal for simple cases:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "index = SearchIndex.from_dict(schema, redis_url=\"redis://localhost:6379\", validate_on_load=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create the index\n", + "\n", + "Now that we are connected to Redis, we need to run the create command." + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "index = index.from_existing(\"user_simple\", redis_client=client)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Data to `SearchIndex`\n", + "\n", + "Load the sample dataset to Redis.\n", + "\n", + "### Validate data entries on load\n", + "RedisVL uses pydantic validation under the hood to ensure loaded data is valid and confirms to your schema. This setting is optional and can be configured in the `SearchIndex` class." + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['prefix_2:01KH5W16MNDEZ1HVN8D1Q9T0WE']\n" + ] + } + ], + "source": [ + "keys = index.load(data_2, keys=['prefix_2:01KH5W16MNDEZ1HVN8D1Q9T0WE'])\n", + "\n", + "print(keys)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a `SQLQuery` Object\n", + "\n", + "First, let's test a simple select statement such as the one below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from redisvl.query import SQLQuery\n", + "\n", + "sql_str = \"\"\"\n", + " SELECT user,\n", + " FROM user_simple\n", + " \"\"\"\n", + "\n", + "sql_query = SQLQuery(sql_str) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check the created query string" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'FT.SEARCH user_simple \"*\"'" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Executing the query" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'$': '{\"user\":\"john\",\"age\":34,\"job\":\"software engineer\",\"region\":\"us-west\",\"job_description\":\"Designs, develops, and maintains software applications and systems.\",\"job_embedding\":[0.04465870559215546,0.010587756521999836,-0.05353280529379845,-0.010134829208254814,0.005005690734833479,0.030759306624531742,-0.014902854338288307,-0.04698082059621811,-0.034925706684589386,-0.039096299558877945,0.04844668507575989,0.01839977689087391,-0.0014138900442048907,0.05128078907728195,-0.03908208757638931,-0.005652933847159147,0.044875431805849075,0.005909909959882498,-0.02784912846982479,0.05582724139094353,-0.04550224915146828,0.04076167196035385,-0.03107522800564766,0.07929772138595581,0.001851016888394952,-0.045421943068504333,0.024686742573976517,-0.012819361872971058,-0.0027404886204749346,0.008564073592424393,0.05924975499510765,-0.02189061790704727,0.018245821818709373,-0.03385121375322342,1.5249262332872604e-6,0.014445340260863304,0.008472551591694355,-0.018257101997733116,0.019804224371910095,-0.05231376364827156,0.01821916177868843,-0.03638678789138794,0.008458063006401062,-0.010556346736848354,0.005708886776119471,-0.012801719829440115,0.017545489594340324,-0.003853817703202367,0.00018880338757298887,0.019757159054279327,0.0144624887034297,-0.06486458331346512,-0.05547238886356354,0.0007543584797531366,-0.03300764784216881,0.005791503936052322,0.0706072449684143,0.028669003397226337,-0.01565077155828476,0.012256475165486336,0.005658522713929415,-0.02144368179142475,-0.06874477118253708,-0.004140299744904041,0.07985666394233704,0.02707027830183506,0.047376226633787155,-0.05364399775862694,0.07513970136642456,-0.03838520497083664,0.05580262094736099,-0.028208106756210327,-0.008463665843009949,0.005147140473127365,-0.006130254827439785,0.03338335454463959,0.01734927110373974,-0.026589877903461456,0.0042011612094938755,-0.03583930805325508,0.028226161375641823,0.04009611904621124,-0.00635960279032588,0.01978931576013565,-0.03259668126702309,0.04199458658695221,-0.017851844429969788,-0.004294633865356445,0.019615309312939644,-0.01458727102726698,0.06155366823077202,-0.004903111606836319,-0.007876947522163391,0.05156824737787247,-0.0200202614068985,-0.015131845138967035,-0.0420171320438385,-0.06212737411260605,0.003615464083850384,0.008403409272432327,-0.028165841475129128,-0.04172233119606972,0.023405209183692932,0.008914102800190449,0.01742619462311268,0.009811930358409882,0.06951268017292023,0.05322611704468727,-0.01027042791247368,-0.029340257868170735,-0.01377735286951065,-0.024378562346100807,-0.05883018299937248,-0.07888967543840408,0.043934427201747894,0.020339537411928177,0.030926089733839035,0.06548662483692169,-0.017526190727949142,0.02179715223610401,0.023180047050118446,-0.011735012754797935,-0.04570522904396057,0.06130192056298256,-0.05169547349214554,-0.05671132355928421,0.013352531008422376,0.04106322303414345,0.03835366666316986,0.0952172726392746,-0.02331560105085373,0.04318320006132126,0.04439179226756096,-0.07245053350925446,-0.0212740208953619,0.02844415046274662,0.07966526597738266,0.029575632885098457,0.0020792530849575996,-0.0512903593480587,0.010419570840895176,-0.025810690596699715,-0.007542443461716175,0.01621614582836628,-0.00010836944420589134,-0.005424370989203453,0.006950909737497568,-0.044698648154735565,0.0027721188962459564,0.032937370240688324,-0.008376521058380604,0.02040264941751957,-0.05888582766056061,0.022673219442367554,0.021120436489582065,-0.016786502674221992,-0.05020292103290558,-0.04792346432805061,-0.010282994247972964,0.06183501332998276,0.02124611847102642,-0.030474167317152023,0.032991476356983185,-0.03589514642953873,-0.05750162526965141,-0.03402270749211311,0.02953892946243286,0.04329780489206314,0.010238228365778925,-0.015392176806926727,0.015344868414103985,-0.08808453381061554,0.0014120446285232902,0.026648560538887978,0.01757194660604,0.05683029443025589,0.09528370946645735,-0.06084143742918968,0.0491555854678154,0.0533013753592968,0.015687769278883934,-0.058529552072286606,0.06815719604492188,0.019825482740998268,0.03856364265084267,0.026698391884565353,-0.1271713525056839,0.0198629442602396,0.032641589641571045,-0.0013364723417907951,0.02446918748319149,0.01197370607405901,0.02447534166276455,0.03078152798116207,0.028993841260671616,0.026045816019177437,-0.02668362855911255,0.042730774730443954,0.016784878447651863,0.01373843103647232,-0.011448456905782224,-0.000605617300607264,-0.02513749711215496,0.053134240210056305,-0.002671192865818739,-0.08087806403636932,-0.0152437100186944,-0.030810166150331497,-0.05792485550045967,0.02000865526497364,0.030052082613110542,-0.05141400545835495,-0.05317755043506622,0.04202063009142876,-0.013559513725340366,-0.05682005733251572,0.02087032794952393,-0.026059286668896675,0.05425924435257912,0.0014789339620620012,0.019972937181591988,-0.006503177806735039,-0.028391238301992416,0.03559108078479767,-0.00927132647484541,-0.030650366097688675,0.016098273918032646,0.018095768988132477,-0.04999242350459099,0.02510037831962109,-0.004824165254831314,0.023220734670758247,0.00495633389800787,0.055009014904499054,0.01661697030067444,-0.02695009671151638,0.015451953746378422,0.02166012115776539,-0.06815879046916962,0.0367053747177124,-0.0005299412296153605,0.030327554792165756,0.020127788186073303,0.03413009271025658,0.01899602822959423,-0.010855140164494514,-0.05183544009923935,0.028192434459924694,0.006386462599039078,0.007999991998076439,0.016479570418596268,-0.028493184596300125,0.04614035412669182,0.0490453876554966,-0.034517351537942886,-0.05584757402539253,-0.048515040427446365,0.02208630181849003,-0.07525543868541718,-0.007137464825063944,0.03509140759706497,-0.02124635875225067,0.014397354796528816,0.019801365211606026,0.002038734033703804,-0.015954433009028435,-0.04233185946941376,0.020379304885864254,-0.044618163257837296,0.027857569977641106,0.026164568960666656,0.026104288175702095,-0.011441050097346306,-0.041615501046180725,0.011017434298992155,-0.016569258645176888,-0.014039809815585612,-0.06655829399824142,-0.022975444793701172,0.048915307968854904,0.0383693166077137,-0.003366320626810193,-0.009515364654362202,0.0004410519322846085,-0.026810435578227043,0.011147692799568176,-0.02208589948713779,-0.04045959934592247,0.004336528480052948,-0.0044569396413862705,-0.020270539447665215,0.012656289152801037,0.0203036367893219,-0.009235053323209286,-0.033484891057014465,0.06517349928617477,-0.026231884956359863,0.015807827934622765,-0.005210281349718571,0.046363454312086105,0.036493875086307526,0.022376226261258125,0.04786619916558266,0.027203161269426342,-0.005276663228869438,0.08965267241001129,0.03998945653438568,0.003212028881534934,0.013152167201042175,0.009163230657577516,0.016631200909614563,-0.00746575091034174,-0.007537082303315401,0.021438689902424812,0.02376984991133213,-0.006901635322719812,-0.050480298697948456,-0.008092441596090794,0.010061741806566715,0.027116617187857628,-0.006690903566777706,0.0009932616958394649,0.024158434942364693,0.02275818958878517,0.04967658221721649,-0.06021817401051521,-0.07022630423307419,-0.02696800045669079,0.03406010940670967,-0.03273770213127136,0.027883876115083694,0.01337795238941908,-0.025057410821318623,0.013362097553908823,0.04701513797044754,0.003923701122403145,-0.04618452861905098,-0.03003666549921036,0.015678154304623604,-0.007023845333606005,-0.020407039672136307,0.008067491464316845,-0.04248598590493202,0.015716487541794777,-0.028392242267727852,0.03516009822487831,-0.007695518899708986,-0.021124351769685745,-0.07776573300361633,-0.005828561261296272,-0.025031480938196186,0.05122635141015053,0.040730927139520645,0.002972182584926486,-0.055967263877391815,-0.03269585594534874,0.07373689860105515,0.03529178723692894,-0.08661627769470215,0.030852768570184708,0.020371168851852417,-0.02736722491681576,-0.0648396760225296,0.014588996767997742,0.016788939014077187,-0.010172214359045029,-0.02986638061702251,-0.08333751559257507,0.011108599603176115,-0.017572185024619102,-0.08196686953306198,-0.025323400273919106,-0.010699030943214892,0.013932229951024055,-0.011705647222697737,0.02149447239935398,0.028941592201590535,0.03692883625626564,-0.13413920998573303,0.04255669564008713,0.038019899278879166,-0.06765256077051163,-0.006805430166423321,-0.003157504601404071,0.02068285644054413,0.0120023125782609,0.04418373852968216,-0.0350869856774807,0.03865808993577957,0.005781974643468857,0.014032726176083088,-0.02135581150650978,-0.034278467297554016,-0.06327281892299652,-0.022360803559422493,-0.031069112941622738,-0.04077891260385513,-0.003460371401160955,0.013886417262256144,0.008947626687586308,0.033127691596746445,0.05420679971575737,-0.05928004905581474,0.0677759200334549,0.05554516613483429,-0.05297881364822388,0.05733130127191544,0.025021331384778023,0.0015319109661504626,-0.021326791495084763,0.0016730449860915542,0.029583754017949104,-0.005570429842919111,0.03734691068530083,0.03247775137424469,-0.07861404865980148,-0.027241727337241173,0.0010142368264496326,-0.023824498057365417,-0.09941552579402924,-0.0013529404532164335,-0.007340687792748213,0.07179094105958939,0.03368861973285675,0.0418473556637764,-0.05536535754799843,-0.010485627688467504,0.00845787487924099,0.07209079712629318,-0.02944222278892994,0.010411819443106651,0.005573534872382879,0.001070575905032456,-0.04579661041498184,0.001051754574291408,-0.0015043772291392088,0.021390117704868317,-0.02959471568465233,0.008632016368210316,-0.035281289368867874,-0.051078833639621735,0.03796431049704552,-0.03065161593258381,-0.00018123297195415944,0.029030172154307365,0.03871051222085953,-0.011236165650188925,-0.08649012446403503,-0.006070153322070837,0.015582511201500893,0.0876385048031807,0.026383766904473305,-0.1041778326034546,-0.003999432548880577,-0.00015469823847524822,-0.04784208536148071,-0.07154097408056259,-0.0031742581631988287,0.008527224883437157,-0.0099567249417305,-0.03959545120596886,0.0184535663574934,-0.037420134991407394,-0.030903376638889313,0.0032852927688509226,-0.06925247609615326,0.05274089798331261,-0.02097955346107483,-0.02662389539182186,-0.03413928672671318,-0.005745565518736839,-0.012825817801058292,-0.01180932391434908,-0.004063298460096121,0.007535572163760662,-0.052773475646972656,-0.0353076308965683,0.06955849379301071,-0.012281234376132488,0.003093794919550419,0.027969347313046455,-0.023082371801137924,0.03956673666834831,-0.04250053316354752,-0.03760653734207153,0.005985671654343605,-0.03234819322824478,-0.007203991990536451,0.004373366013169289,-0.03956164792180061,-0.0010423494968563318,-0.06416654586791992,0.026362525299191475,0.04471899569034576,0.015243026427924631,0.01574576087296009,-0.044725965708494186,0.023194169625639915,0.015917260199785233,0.0048509095795452595,-0.0027987430803477764,-0.01413262728601694,-0.003690459532663226,-0.07854505628347397,-0.012334320694208143,0.013173775747418404,0.01886870339512825,-0.003687628312036395,-0.032245997339487076,0.013316662050783634,-0.004381058271974325,-0.018151583150029182,-0.040154702961444855,-0.015029806643724442,0.015661481767892838,-0.05530605837702751,-0.02742258831858635,-0.04182353988289833,-0.0027738912031054497,0.015038341283798218,-0.004403000697493553,-0.0474509634077549,0.01090835127979517,0.025810757651925087,0.015593734569847584,-0.025652870535850525,-0.007837526500225067,0.004585846792906523,-0.019243549555540085,0.0900978296995163,0.0058959838934242725,0.036284394562244415,-0.024973612278699875,-0.02088586613535881,-0.015253443270921707,0.005590340588241816,0.0528322272002697,0.07389015704393387,-0.010602890513837336,0.03244037553668022,-0.03009670414030552,-0.028876788914203644,-0.04536840692162514,-0.013469278812408447,0.05582284554839134,-0.005577081348747015,-0.052328675985336304,0.03389532491564751,-0.0010674572549760342,-0.005366648081690073,-0.020758986473083496,-0.034027453511953354,0.018306003883481026,0.06809838116168976,0.005569029599428177,-4.3393351920085665e-33,-0.00973324105143547,-0.04467969387769699,0.03002370148897171,-0.01250439416617155,0.016666710376739502,0.02318446896970272,0.04204557463526726,-0.03259154409170151,-0.04651488736271858,-0.007368916179984808,0.02794821374118328,-0.003272237256169319,0.017125636339187622,-0.005343317054212093,0.0024764370173215866,0.05281930789351463,0.005955635569989681,0.031859930604696274,0.0038711426313966513,-0.005840168334543705,0.003616162575781346,-0.03967035934329033,0.06066974252462387,-0.012133347801864147,0.033040232956409454,-0.02197360433638096,-0.022924693301320076,-0.011500217020511627,0.001590083003975451,0.028223734349012375,-0.024313317611813545,0.03297041729092598,-0.0021596101578325033,0.017391007393598557,-0.01773904636502266,-0.03350696340203285,-0.053558144718408585,0.021118270233273503,-0.05480914190411568,0.0191555954515934,0.03171834349632263,-0.007546309381723404,0.014071046374738216,0.04323604702949524,-0.01361356396228075,-0.03759201616048813,0.02297029085457325,-0.013601225800812244,0.02264777570962906,0.018254593014717106,-0.06235458701848984,-0.0062710861675441265,0.007994781248271465,0.0033081835135817528,-0.0029781123157590628,-0.01680983230471611,0.02104908972978592,0.015528872609138489,-0.01188796665519476,-0.005531811621040106,0.0063726031221449375,-0.020053302869200703,0.0489845909178257,0.04094725847244263,0.005458582658320665,0.008285298943519592,0.00833235401660204,0.08911659568548203,0.03960005193948746,-0.003904785728082061,-0.03730185702443123,0.037068650126457214,-0.04585558548569679,-0.05195619538426399,-0.001891149440780282,-0.038914501667022705,-0.024431433528661728,-0.011677077040076256,-0.009039110504090786,0.02113197930157185,0.03344608098268509,0.007016262970864773,-0.03592817485332489,-0.01303576212376356,0.008276466280221939,0.01307408418506384,-0.007777166087180376,0.01874537393450737,0.012593050487339497,0.024325404316186905,-0.02437417767941952,0.0508071631193161,-0.1069781333208084,-0.05601733550429344,0.07919231802225113,0.00286273704841733,-0.0819130465388298,-0.06534022092819214,-0.04123834520578384,-0.013097669929265976,0.022518543526530262,0.004014468286186457,0.04766584932804108,0.04625300318002701,0.05554885044693947,0.03502865880727768,-0.02076628990471363,-0.03520170971751213,-0.03580428287386894,-0.028989655897021297,-0.028998028486967087,0.011842465028166773,0.015427007339894772,0.0835094302892685,0.023758241906762123,-0.002444828394800425,0.0227345023304224,-0.0386001318693161,0.01413789764046669,-0.03455312177538872,0.01581161841750145,0.02085563912987709,0.017975470051169395,-0.0035688779316842556,0.054859552532434464,-0.02812391333281994,-0.017851516604423523,-0.06861165165901184,-0.0015417448012158277,0.03890732303261757,0.0013391567626968026,-0.017538465559482574,2.2463075310952266e-7,0.026951812207698825,-0.00774385966360569,-0.03282041475176811,0.09257732331752776,0.020928721874952316,0.020586244761943817,0.006380537059158087,0.02576143853366375,-0.003436882281675935,0.003888475708663464,-0.012979855760931969,-0.0427749902009964,-0.002237242879346013,-0.007391232065856457,-0.01089764479547739,0.0024610511027276516,-0.03627433627843857,-0.010003073140978811,-0.020465919747948647,0.002037202473729849,0.0734887421131134,0.018512526527047157,0.030127082020044327,-0.0009775710059329867,-0.003079118672758341,-0.05176328122615814,-0.05513224005699158,0.036406975239515305,0.05382297933101654,0.06471472978591919,-0.010014587081968784,0.016729656606912613,0.025992317125201225,0.0710868090391159,-0.0423821434378624,0.008003376424312592,0.04430699720978737,0.03231395035982132,-0.026960670948028564,0.0801289901137352,0.02726324275135994,-0.002857810351997614,0.03303774073719978,-0.026728618890047073,0.01869972050189972,0.03559081256389618,0.018691208213567737,-0.02943197451531887,-0.005556528456509113,0.019652429968118668,-0.03790770471096039,-0.022069323807954788,-0.010444818995893002,-0.030446447432041168,0.008722160011529922,0.0171771589666605,-0.038615882396698,-0.031914327293634415,0.00924017746001482,0.04288874939084053,-0.02167481742799282,-0.04598354548215866,0.010356539860367777,0.03795146197080612,0.12220049649477004,-0.04326684772968292,0.017966659739613533,1.9926973347477206e-34,0.04197061434388161,0.050953928381204605,-0.06480035930871964,-0.015960704535245895,0.012613693252205849,-0.052829936146736145,-0.02632390893995762,-0.0013441862538456917,0.0010861357441172004,-0.05901218950748443,-0.0356571190059185]}'},\n", + " {'$': '{\"prefix\":\"prefix_2\",\"user\":\"john\",\"age\":34,\"job\":\"software engineer\",\"region\":\"us-west\",\"job_description\":\"Designs, develops, and maintains software applications and systems.\",\"job_embedding\":[0.04465870559215546,0.010587756521999836,-0.05353280529379845,-0.010134829208254814,0.005005690734833479,0.030759306624531742,-0.014902854338288307,-0.04698082059621811,-0.034925706684589386,-0.039096299558877945,0.04844668507575989,0.01839977689087391,-0.0014138900442048907,0.05128078907728195,-0.03908208757638931,-0.005652933847159147,0.044875431805849075,0.005909909959882498,-0.02784912846982479,0.05582724139094353,-0.04550224915146828,0.04076167196035385,-0.03107522800564766,0.07929772138595581,0.001851016888394952,-0.045421943068504333,0.024686742573976517,-0.012819361872971058,-0.0027404886204749346,0.008564073592424393,0.05924975499510765,-0.02189061790704727,0.018245821818709373,-0.03385121375322342,1.5249262332872604e-6,0.014445340260863304,0.008472551591694355,-0.018257101997733116,0.019804224371910095,-0.05231376364827156,0.01821916177868843,-0.03638678789138794,0.008458063006401062,-0.010556346736848354,0.005708886776119471,-0.012801719829440115,0.017545489594340324,-0.003853817703202367,0.00018880338757298887,0.019757159054279327,0.0144624887034297,-0.06486458331346512,-0.05547238886356354,0.0007543584797531366,-0.03300764784216881,0.005791503936052322,0.0706072449684143,0.028669003397226337,-0.01565077155828476,0.012256475165486336,0.005658522713929415,-0.02144368179142475,-0.06874477118253708,-0.004140299744904041,0.07985666394233704,0.02707027830183506,0.047376226633787155,-0.05364399775862694,0.07513970136642456,-0.03838520497083664,0.05580262094736099,-0.028208106756210327,-0.008463665843009949,0.005147140473127365,-0.006130254827439785,0.03338335454463959,0.01734927110373974,-0.026589877903461456,0.0042011612094938755,-0.03583930805325508,0.028226161375641823,0.04009611904621124,-0.00635960279032588,0.01978931576013565,-0.03259668126702309,0.04199458658695221,-0.017851844429969788,-0.004294633865356445,0.019615309312939644,-0.01458727102726698,0.06155366823077202,-0.004903111606836319,-0.007876947522163391,0.05156824737787247,-0.0200202614068985,-0.015131845138967035,-0.0420171320438385,-0.06212737411260605,0.003615464083850384,0.008403409272432327,-0.028165841475129128,-0.04172233119606972,0.023405209183692932,0.008914102800190449,0.01742619462311268,0.009811930358409882,0.06951268017292023,0.05322611704468727,-0.01027042791247368,-0.029340257868170735,-0.01377735286951065,-0.024378562346100807,-0.05883018299937248,-0.07888967543840408,0.043934427201747894,0.020339537411928177,0.030926089733839035,0.06548662483692169,-0.017526190727949142,0.02179715223610401,0.023180047050118446,-0.011735012754797935,-0.04570522904396057,0.06130192056298256,-0.05169547349214554,-0.05671132355928421,0.013352531008422376,0.04106322303414345,0.03835366666316986,0.0952172726392746,-0.02331560105085373,0.04318320006132126,0.04439179226756096,-0.07245053350925446,-0.0212740208953619,0.02844415046274662,0.07966526597738266,0.029575632885098457,0.0020792530849575996,-0.0512903593480587,0.010419570840895176,-0.025810690596699715,-0.007542443461716175,0.01621614582836628,-0.00010836944420589134,-0.005424370989203453,0.006950909737497568,-0.044698648154735565,0.0027721188962459564,0.032937370240688324,-0.008376521058380604,0.02040264941751957,-0.05888582766056061,0.022673219442367554,0.021120436489582065,-0.016786502674221992,-0.05020292103290558,-0.04792346432805061,-0.010282994247972964,0.06183501332998276,0.02124611847102642,-0.030474167317152023,0.032991476356983185,-0.03589514642953873,-0.05750162526965141,-0.03402270749211311,0.02953892946243286,0.04329780489206314,0.010238228365778925,-0.015392176806926727,0.015344868414103985,-0.08808453381061554,0.0014120446285232902,0.026648560538887978,0.01757194660604,0.05683029443025589,0.09528370946645735,-0.06084143742918968,0.0491555854678154,0.0533013753592968,0.015687769278883934,-0.058529552072286606,0.06815719604492188,0.019825482740998268,0.03856364265084267,0.026698391884565353,-0.1271713525056839,0.0198629442602396,0.032641589641571045,-0.0013364723417907951,0.02446918748319149,0.01197370607405901,0.02447534166276455,0.03078152798116207,0.028993841260671616,0.026045816019177437,-0.02668362855911255,0.042730774730443954,0.016784878447651863,0.01373843103647232,-0.011448456905782224,-0.000605617300607264,-0.02513749711215496,0.053134240210056305,-0.002671192865818739,-0.08087806403636932,-0.0152437100186944,-0.030810166150331497,-0.05792485550045967,0.02000865526497364,0.030052082613110542,-0.05141400545835495,-0.05317755043506622,0.04202063009142876,-0.013559513725340366,-0.05682005733251572,0.02087032794952393,-0.026059286668896675,0.05425924435257912,0.0014789339620620012,0.019972937181591988,-0.006503177806735039,-0.028391238301992416,0.03559108078479767,-0.00927132647484541,-0.030650366097688675,0.016098273918032646,0.018095768988132477,-0.04999242350459099,0.02510037831962109,-0.004824165254831314,0.023220734670758247,0.00495633389800787,0.055009014904499054,0.01661697030067444,-0.02695009671151638,0.015451953746378422,0.02166012115776539,-0.06815879046916962,0.0367053747177124,-0.0005299412296153605,0.030327554792165756,0.020127788186073303,0.03413009271025658,0.01899602822959423,-0.010855140164494514,-0.05183544009923935,0.028192434459924694,0.006386462599039078,0.007999991998076439,0.016479570418596268,-0.028493184596300125,0.04614035412669182,0.0490453876554966,-0.034517351537942886,-0.05584757402539253,-0.048515040427446365,0.02208630181849003,-0.07525543868541718,-0.007137464825063944,0.03509140759706497,-0.02124635875225067,0.014397354796528816,0.019801365211606026,0.002038734033703804,-0.015954433009028435,-0.04233185946941376,0.020379304885864254,-0.044618163257837296,0.027857569977641106,0.026164568960666656,0.026104288175702095,-0.011441050097346306,-0.041615501046180725,0.011017434298992155,-0.016569258645176888,-0.014039809815585612,-0.06655829399824142,-0.022975444793701172,0.048915307968854904,0.0383693166077137,-0.003366320626810193,-0.009515364654362202,0.0004410519322846085,-0.026810435578227043,0.011147692799568176,-0.02208589948713779,-0.04045959934592247,0.004336528480052948,-0.0044569396413862705,-0.020270539447665215,0.012656289152801037,0.0203036367893219,-0.009235053323209286,-0.033484891057014465,0.06517349928617477,-0.026231884956359863,0.015807827934622765,-0.005210281349718571,0.046363454312086105,0.036493875086307526,0.022376226261258125,0.04786619916558266,0.027203161269426342,-0.005276663228869438,0.08965267241001129,0.03998945653438568,0.003212028881534934,0.013152167201042175,0.009163230657577516,0.016631200909614563,-0.00746575091034174,-0.007537082303315401,0.021438689902424812,0.02376984991133213,-0.006901635322719812,-0.050480298697948456,-0.008092441596090794,0.010061741806566715,0.027116617187857628,-0.006690903566777706,0.0009932616958394649,0.024158434942364693,0.02275818958878517,0.04967658221721649,-0.06021817401051521,-0.07022630423307419,-0.02696800045669079,0.03406010940670967,-0.03273770213127136,0.027883876115083694,0.01337795238941908,-0.025057410821318623,0.013362097553908823,0.04701513797044754,0.003923701122403145,-0.04618452861905098,-0.03003666549921036,0.015678154304623604,-0.007023845333606005,-0.020407039672136307,0.008067491464316845,-0.04248598590493202,0.015716487541794777,-0.028392242267727852,0.03516009822487831,-0.007695518899708986,-0.021124351769685745,-0.07776573300361633,-0.005828561261296272,-0.025031480938196186,0.05122635141015053,0.040730927139520645,0.002972182584926486,-0.055967263877391815,-0.03269585594534874,0.07373689860105515,0.03529178723692894,-0.08661627769470215,0.030852768570184708,0.020371168851852417,-0.02736722491681576,-0.0648396760225296,0.014588996767997742,0.016788939014077187,-0.010172214359045029,-0.02986638061702251,-0.08333751559257507,0.011108599603176115,-0.017572185024619102,-0.08196686953306198,-0.025323400273919106,-0.010699030943214892,0.013932229951024055,-0.011705647222697737,0.02149447239935398,0.028941592201590535,0.03692883625626564,-0.13413920998573303,0.04255669564008713,0.038019899278879166,-0.06765256077051163,-0.006805430166423321,-0.003157504601404071,0.02068285644054413,0.0120023125782609,0.04418373852968216,-0.0350869856774807,0.03865808993577957,0.005781974643468857,0.014032726176083088,-0.02135581150650978,-0.034278467297554016,-0.06327281892299652,-0.022360803559422493,-0.031069112941622738,-0.04077891260385513,-0.003460371401160955,0.013886417262256144,0.008947626687586308,0.033127691596746445,0.05420679971575737,-0.05928004905581474,0.0677759200334549,0.05554516613483429,-0.05297881364822388,0.05733130127191544,0.025021331384778023,0.0015319109661504626,-0.021326791495084763,0.0016730449860915542,0.029583754017949104,-0.005570429842919111,0.03734691068530083,0.03247775137424469,-0.07861404865980148,-0.027241727337241173,0.0010142368264496326,-0.023824498057365417,-0.09941552579402924,-0.0013529404532164335,-0.007340687792748213,0.07179094105958939,0.03368861973285675,0.0418473556637764,-0.05536535754799843,-0.010485627688467504,0.00845787487924099,0.07209079712629318,-0.02944222278892994,0.010411819443106651,0.005573534872382879,0.001070575905032456,-0.04579661041498184,0.001051754574291408,-0.0015043772291392088,0.021390117704868317,-0.02959471568465233,0.008632016368210316,-0.035281289368867874,-0.051078833639621735,0.03796431049704552,-0.03065161593258381,-0.00018123297195415944,0.029030172154307365,0.03871051222085953,-0.011236165650188925,-0.08649012446403503,-0.006070153322070837,0.015582511201500893,0.0876385048031807,0.026383766904473305,-0.1041778326034546,-0.003999432548880577,-0.00015469823847524822,-0.04784208536148071,-0.07154097408056259,-0.0031742581631988287,0.008527224883437157,-0.0099567249417305,-0.03959545120596886,0.0184535663574934,-0.037420134991407394,-0.030903376638889313,0.0032852927688509226,-0.06925247609615326,0.05274089798331261,-0.02097955346107483,-0.02662389539182186,-0.03413928672671318,-0.005745565518736839,-0.012825817801058292,-0.01180932391434908,-0.004063298460096121,0.007535572163760662,-0.052773475646972656,-0.0353076308965683,0.06955849379301071,-0.012281234376132488,0.003093794919550419,0.027969347313046455,-0.023082371801137924,0.03956673666834831,-0.04250053316354752,-0.03760653734207153,0.005985671654343605,-0.03234819322824478,-0.007203991990536451,0.004373366013169289,-0.03956164792180061,-0.0010423494968563318,-0.06416654586791992,0.026362525299191475,0.04471899569034576,0.015243026427924631,0.01574576087296009,-0.044725965708494186,0.023194169625639915,0.015917260199785233,0.0048509095795452595,-0.0027987430803477764,-0.01413262728601694,-0.003690459532663226,-0.07854505628347397,-0.012334320694208143,0.013173775747418404,0.01886870339512825,-0.003687628312036395,-0.032245997339487076,0.013316662050783634,-0.004381058271974325,-0.018151583150029182,-0.040154702961444855,-0.015029806643724442,0.015661481767892838,-0.05530605837702751,-0.02742258831858635,-0.04182353988289833,-0.0027738912031054497,0.015038341283798218,-0.004403000697493553,-0.0474509634077549,0.01090835127979517,0.025810757651925087,0.015593734569847584,-0.025652870535850525,-0.007837526500225067,0.004585846792906523,-0.019243549555540085,0.0900978296995163,0.0058959838934242725,0.036284394562244415,-0.024973612278699875,-0.02088586613535881,-0.015253443270921707,0.005590340588241816,0.0528322272002697,0.07389015704393387,-0.010602890513837336,0.03244037553668022,-0.03009670414030552,-0.028876788914203644,-0.04536840692162514,-0.013469278812408447,0.05582284554839134,-0.005577081348747015,-0.052328675985336304,0.03389532491564751,-0.0010674572549760342,-0.005366648081690073,-0.020758986473083496,-0.034027453511953354,0.018306003883481026,0.06809838116168976,0.005569029599428177,-4.3393351920085665e-33,-0.00973324105143547,-0.04467969387769699,0.03002370148897171,-0.01250439416617155,0.016666710376739502,0.02318446896970272,0.04204557463526726,-0.03259154409170151,-0.04651488736271858,-0.007368916179984808,0.02794821374118328,-0.003272237256169319,0.017125636339187622,-0.005343317054212093,0.0024764370173215866,0.05281930789351463,0.005955635569989681,0.031859930604696274,0.0038711426313966513,-0.005840168334543705,0.003616162575781346,-0.03967035934329033,0.06066974252462387,-0.012133347801864147,0.033040232956409454,-0.02197360433638096,-0.022924693301320076,-0.011500217020511627,0.001590083003975451,0.028223734349012375,-0.024313317611813545,0.03297041729092598,-0.0021596101578325033,0.017391007393598557,-0.01773904636502266,-0.03350696340203285,-0.053558144718408585,0.021118270233273503,-0.05480914190411568,0.0191555954515934,0.03171834349632263,-0.007546309381723404,0.014071046374738216,0.04323604702949524,-0.01361356396228075,-0.03759201616048813,0.02297029085457325,-0.013601225800812244,0.02264777570962906,0.018254593014717106,-0.06235458701848984,-0.0062710861675441265,0.007994781248271465,0.0033081835135817528,-0.0029781123157590628,-0.01680983230471611,0.02104908972978592,0.015528872609138489,-0.01188796665519476,-0.005531811621040106,0.0063726031221449375,-0.020053302869200703,0.0489845909178257,0.04094725847244263,0.005458582658320665,0.008285298943519592,0.00833235401660204,0.08911659568548203,0.03960005193948746,-0.003904785728082061,-0.03730185702443123,0.037068650126457214,-0.04585558548569679,-0.05195619538426399,-0.001891149440780282,-0.038914501667022705,-0.024431433528661728,-0.011677077040076256,-0.009039110504090786,0.02113197930157185,0.03344608098268509,0.007016262970864773,-0.03592817485332489,-0.01303576212376356,0.008276466280221939,0.01307408418506384,-0.007777166087180376,0.01874537393450737,0.012593050487339497,0.024325404316186905,-0.02437417767941952,0.0508071631193161,-0.1069781333208084,-0.05601733550429344,0.07919231802225113,0.00286273704841733,-0.0819130465388298,-0.06534022092819214,-0.04123834520578384,-0.013097669929265976,0.022518543526530262,0.004014468286186457,0.04766584932804108,0.04625300318002701,0.05554885044693947,0.03502865880727768,-0.02076628990471363,-0.03520170971751213,-0.03580428287386894,-0.028989655897021297,-0.028998028486967087,0.011842465028166773,0.015427007339894772,0.0835094302892685,0.023758241906762123,-0.002444828394800425,0.0227345023304224,-0.0386001318693161,0.01413789764046669,-0.03455312177538872,0.01581161841750145,0.02085563912987709,0.017975470051169395,-0.0035688779316842556,0.054859552532434464,-0.02812391333281994,-0.017851516604423523,-0.06861165165901184,-0.0015417448012158277,0.03890732303261757,0.0013391567626968026,-0.017538465559482574,2.2463075310952266e-7,0.026951812207698825,-0.00774385966360569,-0.03282041475176811,0.09257732331752776,0.020928721874952316,0.020586244761943817,0.006380537059158087,0.02576143853366375,-0.003436882281675935,0.003888475708663464,-0.012979855760931969,-0.0427749902009964,-0.002237242879346013,-0.007391232065856457,-0.01089764479547739,0.0024610511027276516,-0.03627433627843857,-0.010003073140978811,-0.020465919747948647,0.002037202473729849,0.0734887421131134,0.018512526527047157,0.030127082020044327,-0.0009775710059329867,-0.003079118672758341,-0.05176328122615814,-0.05513224005699158,0.036406975239515305,0.05382297933101654,0.06471472978591919,-0.010014587081968784,0.016729656606912613,0.025992317125201225,0.0710868090391159,-0.0423821434378624,0.008003376424312592,0.04430699720978737,0.03231395035982132,-0.026960670948028564,0.0801289901137352,0.02726324275135994,-0.002857810351997614,0.03303774073719978,-0.026728618890047073,0.01869972050189972,0.03559081256389618,0.018691208213567737,-0.02943197451531887,-0.005556528456509113,0.019652429968118668,-0.03790770471096039,-0.022069323807954788,-0.010444818995893002,-0.030446447432041168,0.008722160011529922,0.0171771589666605,-0.038615882396698,-0.031914327293634415,0.00924017746001482,0.04288874939084053,-0.02167481742799282,-0.04598354548215866,0.010356539860367777,0.03795146197080612,0.12220049649477004,-0.04326684772968292,0.017966659739613533,1.9926973347477206e-34,0.04197061434388161,0.050953928381204605,-0.06480035930871964,-0.015960704535245895,0.012613693252205849,-0.052829936146736145,-0.02632390893995762,-0.0013441862538456917,0.0010861357441172004,-0.05901218950748443,-0.0356571190059185]}'},\n", + " {'$': '{\"prefix\":\"prefix_2\",\"user\":\"john\",\"age\":34,\"job\":\"software engineer\",\"region\":\"us-west\",\"job_description\":\"Designs, develops, and maintains software applications and systems.\",\"job_embedding\":[0.04465870559215546,0.010587756521999836,-0.05353280529379845,-0.010134829208254814,0.005005690734833479,0.030759306624531742,-0.014902854338288307,-0.04698082059621811,-0.034925706684589386,-0.039096299558877945,0.04844668507575989,0.01839977689087391,-0.0014138900442048907,0.05128078907728195,-0.03908208757638931,-0.005652933847159147,0.044875431805849075,0.005909909959882498,-0.02784912846982479,0.05582724139094353,-0.04550224915146828,0.04076167196035385,-0.03107522800564766,0.07929772138595581,0.001851016888394952,-0.045421943068504333,0.024686742573976517,-0.012819361872971058,-0.0027404886204749346,0.008564073592424393,0.05924975499510765,-0.02189061790704727,0.018245821818709373,-0.03385121375322342,1.5249262332872604e-6,0.014445340260863304,0.008472551591694355,-0.018257101997733116,0.019804224371910095,-0.05231376364827156,0.01821916177868843,-0.03638678789138794,0.008458063006401062,-0.010556346736848354,0.005708886776119471,-0.012801719829440115,0.017545489594340324,-0.003853817703202367,0.00018880338757298887,0.019757159054279327,0.0144624887034297,-0.06486458331346512,-0.05547238886356354,0.0007543584797531366,-0.03300764784216881,0.005791503936052322,0.0706072449684143,0.028669003397226337,-0.01565077155828476,0.012256475165486336,0.005658522713929415,-0.02144368179142475,-0.06874477118253708,-0.004140299744904041,0.07985666394233704,0.02707027830183506,0.047376226633787155,-0.05364399775862694,0.07513970136642456,-0.03838520497083664,0.05580262094736099,-0.028208106756210327,-0.008463665843009949,0.005147140473127365,-0.006130254827439785,0.03338335454463959,0.01734927110373974,-0.026589877903461456,0.0042011612094938755,-0.03583930805325508,0.028226161375641823,0.04009611904621124,-0.00635960279032588,0.01978931576013565,-0.03259668126702309,0.04199458658695221,-0.017851844429969788,-0.004294633865356445,0.019615309312939644,-0.01458727102726698,0.06155366823077202,-0.004903111606836319,-0.007876947522163391,0.05156824737787247,-0.0200202614068985,-0.015131845138967035,-0.0420171320438385,-0.06212737411260605,0.003615464083850384,0.008403409272432327,-0.028165841475129128,-0.04172233119606972,0.023405209183692932,0.008914102800190449,0.01742619462311268,0.009811930358409882,0.06951268017292023,0.05322611704468727,-0.01027042791247368,-0.029340257868170735,-0.01377735286951065,-0.024378562346100807,-0.05883018299937248,-0.07888967543840408,0.043934427201747894,0.020339537411928177,0.030926089733839035,0.06548662483692169,-0.017526190727949142,0.02179715223610401,0.023180047050118446,-0.011735012754797935,-0.04570522904396057,0.06130192056298256,-0.05169547349214554,-0.05671132355928421,0.013352531008422376,0.04106322303414345,0.03835366666316986,0.0952172726392746,-0.02331560105085373,0.04318320006132126,0.04439179226756096,-0.07245053350925446,-0.0212740208953619,0.02844415046274662,0.07966526597738266,0.029575632885098457,0.0020792530849575996,-0.0512903593480587,0.010419570840895176,-0.025810690596699715,-0.007542443461716175,0.01621614582836628,-0.00010836944420589134,-0.005424370989203453,0.006950909737497568,-0.044698648154735565,0.0027721188962459564,0.032937370240688324,-0.008376521058380604,0.02040264941751957,-0.05888582766056061,0.022673219442367554,0.021120436489582065,-0.016786502674221992,-0.05020292103290558,-0.04792346432805061,-0.010282994247972964,0.06183501332998276,0.02124611847102642,-0.030474167317152023,0.032991476356983185,-0.03589514642953873,-0.05750162526965141,-0.03402270749211311,0.02953892946243286,0.04329780489206314,0.010238228365778925,-0.015392176806926727,0.015344868414103985,-0.08808453381061554,0.0014120446285232902,0.026648560538887978,0.01757194660604,0.05683029443025589,0.09528370946645735,-0.06084143742918968,0.0491555854678154,0.0533013753592968,0.015687769278883934,-0.058529552072286606,0.06815719604492188,0.019825482740998268,0.03856364265084267,0.026698391884565353,-0.1271713525056839,0.0198629442602396,0.032641589641571045,-0.0013364723417907951,0.02446918748319149,0.01197370607405901,0.02447534166276455,0.03078152798116207,0.028993841260671616,0.026045816019177437,-0.02668362855911255,0.042730774730443954,0.016784878447651863,0.01373843103647232,-0.011448456905782224,-0.000605617300607264,-0.02513749711215496,0.053134240210056305,-0.002671192865818739,-0.08087806403636932,-0.0152437100186944,-0.030810166150331497,-0.05792485550045967,0.02000865526497364,0.030052082613110542,-0.05141400545835495,-0.05317755043506622,0.04202063009142876,-0.013559513725340366,-0.05682005733251572,0.02087032794952393,-0.026059286668896675,0.05425924435257912,0.0014789339620620012,0.019972937181591988,-0.006503177806735039,-0.028391238301992416,0.03559108078479767,-0.00927132647484541,-0.030650366097688675,0.016098273918032646,0.018095768988132477,-0.04999242350459099,0.02510037831962109,-0.004824165254831314,0.023220734670758247,0.00495633389800787,0.055009014904499054,0.01661697030067444,-0.02695009671151638,0.015451953746378422,0.02166012115776539,-0.06815879046916962,0.0367053747177124,-0.0005299412296153605,0.030327554792165756,0.020127788186073303,0.03413009271025658,0.01899602822959423,-0.010855140164494514,-0.05183544009923935,0.028192434459924694,0.006386462599039078,0.007999991998076439,0.016479570418596268,-0.028493184596300125,0.04614035412669182,0.0490453876554966,-0.034517351537942886,-0.05584757402539253,-0.048515040427446365,0.02208630181849003,-0.07525543868541718,-0.007137464825063944,0.03509140759706497,-0.02124635875225067,0.014397354796528816,0.019801365211606026,0.002038734033703804,-0.015954433009028435,-0.04233185946941376,0.020379304885864254,-0.044618163257837296,0.027857569977641106,0.026164568960666656,0.026104288175702095,-0.011441050097346306,-0.041615501046180725,0.011017434298992155,-0.016569258645176888,-0.014039809815585612,-0.06655829399824142,-0.022975444793701172,0.048915307968854904,0.0383693166077137,-0.003366320626810193,-0.009515364654362202,0.0004410519322846085,-0.026810435578227043,0.011147692799568176,-0.02208589948713779,-0.04045959934592247,0.004336528480052948,-0.0044569396413862705,-0.020270539447665215,0.012656289152801037,0.0203036367893219,-0.009235053323209286,-0.033484891057014465,0.06517349928617477,-0.026231884956359863,0.015807827934622765,-0.005210281349718571,0.046363454312086105,0.036493875086307526,0.022376226261258125,0.04786619916558266,0.027203161269426342,-0.005276663228869438,0.08965267241001129,0.03998945653438568,0.003212028881534934,0.013152167201042175,0.009163230657577516,0.016631200909614563,-0.00746575091034174,-0.007537082303315401,0.021438689902424812,0.02376984991133213,-0.006901635322719812,-0.050480298697948456,-0.008092441596090794,0.010061741806566715,0.027116617187857628,-0.006690903566777706,0.0009932616958394649,0.024158434942364693,0.02275818958878517,0.04967658221721649,-0.06021817401051521,-0.07022630423307419,-0.02696800045669079,0.03406010940670967,-0.03273770213127136,0.027883876115083694,0.01337795238941908,-0.025057410821318623,0.013362097553908823,0.04701513797044754,0.003923701122403145,-0.04618452861905098,-0.03003666549921036,0.015678154304623604,-0.007023845333606005,-0.020407039672136307,0.008067491464316845,-0.04248598590493202,0.015716487541794777,-0.028392242267727852,0.03516009822487831,-0.007695518899708986,-0.021124351769685745,-0.07776573300361633,-0.005828561261296272,-0.025031480938196186,0.05122635141015053,0.040730927139520645,0.002972182584926486,-0.055967263877391815,-0.03269585594534874,0.07373689860105515,0.03529178723692894,-0.08661627769470215,0.030852768570184708,0.020371168851852417,-0.02736722491681576,-0.0648396760225296,0.014588996767997742,0.016788939014077187,-0.010172214359045029,-0.02986638061702251,-0.08333751559257507,0.011108599603176115,-0.017572185024619102,-0.08196686953306198,-0.025323400273919106,-0.010699030943214892,0.013932229951024055,-0.011705647222697737,0.02149447239935398,0.028941592201590535,0.03692883625626564,-0.13413920998573303,0.04255669564008713,0.038019899278879166,-0.06765256077051163,-0.006805430166423321,-0.003157504601404071,0.02068285644054413,0.0120023125782609,0.04418373852968216,-0.0350869856774807,0.03865808993577957,0.005781974643468857,0.014032726176083088,-0.02135581150650978,-0.034278467297554016,-0.06327281892299652,-0.022360803559422493,-0.031069112941622738,-0.04077891260385513,-0.003460371401160955,0.013886417262256144,0.008947626687586308,0.033127691596746445,0.05420679971575737,-0.05928004905581474,0.0677759200334549,0.05554516613483429,-0.05297881364822388,0.05733130127191544,0.025021331384778023,0.0015319109661504626,-0.021326791495084763,0.0016730449860915542,0.029583754017949104,-0.005570429842919111,0.03734691068530083,0.03247775137424469,-0.07861404865980148,-0.027241727337241173,0.0010142368264496326,-0.023824498057365417,-0.09941552579402924,-0.0013529404532164335,-0.007340687792748213,0.07179094105958939,0.03368861973285675,0.0418473556637764,-0.05536535754799843,-0.010485627688467504,0.00845787487924099,0.07209079712629318,-0.02944222278892994,0.010411819443106651,0.005573534872382879,0.001070575905032456,-0.04579661041498184,0.001051754574291408,-0.0015043772291392088,0.021390117704868317,-0.02959471568465233,0.008632016368210316,-0.035281289368867874,-0.051078833639621735,0.03796431049704552,-0.03065161593258381,-0.00018123297195415944,0.029030172154307365,0.03871051222085953,-0.011236165650188925,-0.08649012446403503,-0.006070153322070837,0.015582511201500893,0.0876385048031807,0.026383766904473305,-0.1041778326034546,-0.003999432548880577,-0.00015469823847524822,-0.04784208536148071,-0.07154097408056259,-0.0031742581631988287,0.008527224883437157,-0.0099567249417305,-0.03959545120596886,0.0184535663574934,-0.037420134991407394,-0.030903376638889313,0.0032852927688509226,-0.06925247609615326,0.05274089798331261,-0.02097955346107483,-0.02662389539182186,-0.03413928672671318,-0.005745565518736839,-0.012825817801058292,-0.01180932391434908,-0.004063298460096121,0.007535572163760662,-0.052773475646972656,-0.0353076308965683,0.06955849379301071,-0.012281234376132488,0.003093794919550419,0.027969347313046455,-0.023082371801137924,0.03956673666834831,-0.04250053316354752,-0.03760653734207153,0.005985671654343605,-0.03234819322824478,-0.007203991990536451,0.004373366013169289,-0.03956164792180061,-0.0010423494968563318,-0.06416654586791992,0.026362525299191475,0.04471899569034576,0.015243026427924631,0.01574576087296009,-0.044725965708494186,0.023194169625639915,0.015917260199785233,0.0048509095795452595,-0.0027987430803477764,-0.01413262728601694,-0.003690459532663226,-0.07854505628347397,-0.012334320694208143,0.013173775747418404,0.01886870339512825,-0.003687628312036395,-0.032245997339487076,0.013316662050783634,-0.004381058271974325,-0.018151583150029182,-0.040154702961444855,-0.015029806643724442,0.015661481767892838,-0.05530605837702751,-0.02742258831858635,-0.04182353988289833,-0.0027738912031054497,0.015038341283798218,-0.004403000697493553,-0.0474509634077549,0.01090835127979517,0.025810757651925087,0.015593734569847584,-0.025652870535850525,-0.007837526500225067,0.004585846792906523,-0.019243549555540085,0.0900978296995163,0.0058959838934242725,0.036284394562244415,-0.024973612278699875,-0.02088586613535881,-0.015253443270921707,0.005590340588241816,0.0528322272002697,0.07389015704393387,-0.010602890513837336,0.03244037553668022,-0.03009670414030552,-0.028876788914203644,-0.04536840692162514,-0.013469278812408447,0.05582284554839134,-0.005577081348747015,-0.052328675985336304,0.03389532491564751,-0.0010674572549760342,-0.005366648081690073,-0.020758986473083496,-0.034027453511953354,0.018306003883481026,0.06809838116168976,0.005569029599428177,-4.3393351920085665e-33,-0.00973324105143547,-0.04467969387769699,0.03002370148897171,-0.01250439416617155,0.016666710376739502,0.02318446896970272,0.04204557463526726,-0.03259154409170151,-0.04651488736271858,-0.007368916179984808,0.02794821374118328,-0.003272237256169319,0.017125636339187622,-0.005343317054212093,0.0024764370173215866,0.05281930789351463,0.005955635569989681,0.031859930604696274,0.0038711426313966513,-0.005840168334543705,0.003616162575781346,-0.03967035934329033,0.06066974252462387,-0.012133347801864147,0.033040232956409454,-0.02197360433638096,-0.022924693301320076,-0.011500217020511627,0.001590083003975451,0.028223734349012375,-0.024313317611813545,0.03297041729092598,-0.0021596101578325033,0.017391007393598557,-0.01773904636502266,-0.03350696340203285,-0.053558144718408585,0.021118270233273503,-0.05480914190411568,0.0191555954515934,0.03171834349632263,-0.007546309381723404,0.014071046374738216,0.04323604702949524,-0.01361356396228075,-0.03759201616048813,0.02297029085457325,-0.013601225800812244,0.02264777570962906,0.018254593014717106,-0.06235458701848984,-0.0062710861675441265,0.007994781248271465,0.0033081835135817528,-0.0029781123157590628,-0.01680983230471611,0.02104908972978592,0.015528872609138489,-0.01188796665519476,-0.005531811621040106,0.0063726031221449375,-0.020053302869200703,0.0489845909178257,0.04094725847244263,0.005458582658320665,0.008285298943519592,0.00833235401660204,0.08911659568548203,0.03960005193948746,-0.003904785728082061,-0.03730185702443123,0.037068650126457214,-0.04585558548569679,-0.05195619538426399,-0.001891149440780282,-0.038914501667022705,-0.024431433528661728,-0.011677077040076256,-0.009039110504090786,0.02113197930157185,0.03344608098268509,0.007016262970864773,-0.03592817485332489,-0.01303576212376356,0.008276466280221939,0.01307408418506384,-0.007777166087180376,0.01874537393450737,0.012593050487339497,0.024325404316186905,-0.02437417767941952,0.0508071631193161,-0.1069781333208084,-0.05601733550429344,0.07919231802225113,0.00286273704841733,-0.0819130465388298,-0.06534022092819214,-0.04123834520578384,-0.013097669929265976,0.022518543526530262,0.004014468286186457,0.04766584932804108,0.04625300318002701,0.05554885044693947,0.03502865880727768,-0.02076628990471363,-0.03520170971751213,-0.03580428287386894,-0.028989655897021297,-0.028998028486967087,0.011842465028166773,0.015427007339894772,0.0835094302892685,0.023758241906762123,-0.002444828394800425,0.0227345023304224,-0.0386001318693161,0.01413789764046669,-0.03455312177538872,0.01581161841750145,0.02085563912987709,0.017975470051169395,-0.0035688779316842556,0.054859552532434464,-0.02812391333281994,-0.017851516604423523,-0.06861165165901184,-0.0015417448012158277,0.03890732303261757,0.0013391567626968026,-0.017538465559482574,2.2463075310952266e-7,0.026951812207698825,-0.00774385966360569,-0.03282041475176811,0.09257732331752776,0.020928721874952316,0.020586244761943817,0.006380537059158087,0.02576143853366375,-0.003436882281675935,0.003888475708663464,-0.012979855760931969,-0.0427749902009964,-0.002237242879346013,-0.007391232065856457,-0.01089764479547739,0.0024610511027276516,-0.03627433627843857,-0.010003073140978811,-0.020465919747948647,0.002037202473729849,0.0734887421131134,0.018512526527047157,0.030127082020044327,-0.0009775710059329867,-0.003079118672758341,-0.05176328122615814,-0.05513224005699158,0.036406975239515305,0.05382297933101654,0.06471472978591919,-0.010014587081968784,0.016729656606912613,0.025992317125201225,0.0710868090391159,-0.0423821434378624,0.008003376424312592,0.04430699720978737,0.03231395035982132,-0.026960670948028564,0.0801289901137352,0.02726324275135994,-0.002857810351997614,0.03303774073719978,-0.026728618890047073,0.01869972050189972,0.03559081256389618,0.018691208213567737,-0.02943197451531887,-0.005556528456509113,0.019652429968118668,-0.03790770471096039,-0.022069323807954788,-0.010444818995893002,-0.030446447432041168,0.008722160011529922,0.0171771589666605,-0.038615882396698,-0.031914327293634415,0.00924017746001482,0.04288874939084053,-0.02167481742799282,-0.04598354548215866,0.010356539860367777,0.03795146197080612,0.12220049649477004,-0.04326684772968292,0.017966659739613533,1.9926973347477206e-34,0.04197061434388161,0.050953928381204605,-0.06480035930871964,-0.015960704535245895,0.012613693252205849,-0.052829936146736145,-0.02632390893995762,-0.0013441862538456917,0.0010861357441172004,-0.05901218950748443,-0.0356571190059185]}'},\n", + " {'$': '{\"user\":\"bill\",\"age\":54,\"job\":\"engineer\",\"region\":\"us-central\",\"job_description\":\"Applies scientific and mathematical principles to solve technical problems.\",\"job_embedding\":[0.03714476153254509,-0.00739729218184948,-0.03678752481937408,-0.0034561653155833483,-0.00719857681542635,0.0062834518030285835,-0.0106437336653471,-0.028371015563607216,-0.052128616720438,-0.02324206754565239,0.07201956957578659,0.015932096168398857,-0.0032016311306506395,0.05974847450852394,-0.04393239691853523,-0.02530195564031601,0.02610727585852146,0.01629062555730343,0.01072105672210455,0.037566475570201874,-0.056516777724027634,0.016249699518084526,-0.019266946241259575,0.06083810701966286,0.009834500961005688,-0.022169778123497963,0.02385234087705612,-0.03583553805947304,0.023450665175914764,-0.008082675747573376,0.0652461051940918,-0.0116201164200902,0.007840285077691078,-0.03315737843513489,1.5049121202537208e-6,0.004104946739971638,0.008771699853241444,-0.0017788441618904471,0.029319168999791145,-0.05405957251787186,0.011540569365024568,-0.02947952970862389,0.04047330096364021,-0.0068633342161774635,-0.007451807614415884,0.0352027453482151,0.001041763694956899,-0.007832045666873455,0.008846805430948734,0.017020605504512787,0.010431625880301,-0.08948108553886414,-0.0500706359744072,-0.016267146915197372,-0.012347332201898098,-0.015331183560192583,0.0739942416548729,0.01906953752040863,0.009058200754225254,0.027531813830137253,-0.006378641817718744,0.0194482896476984,-0.04803531616926193,-0.00650259992107749,0.09214931726455688,0.01599440909922123,0.06433700025081635,-0.015245575457811356,0.0735311359167099,-0.014071094803512096,0.07064857333898544,-0.025511065497994423,-0.005757654085755348,0.0034612221643328667,-0.002029021270573139,0.049675896763801575,-0.011487609706819056,-0.053843215107917786,-0.022419605404138565,-0.05678648874163627,0.03141246363520622,0.029399903491139412,-0.0012889389181509614,0.038158953189849854,-0.03692067414522171,0.07289976626634598,-0.029857736080884933,-0.02702624537050724,0.012924164533615112,-0.018186243250966072,0.06918890029191971,-0.002928528469055891,0.020308030769228935,0.03572939708828926,0.012629523873329164,-0.008250202052295208,-0.034641556441783905,-0.06928113102912903,-0.0004496081091929227,-0.01737191714346409,-0.02403111569583416,-0.030917296186089516,0.07512091845273972,0.02617439441382885,-0.0035722036845982075,-0.010050159879028795,0.041072893887758255,0.04792056232690811,-0.02685832418501377,-0.02744485065340996,-0.05456770956516266,-0.0420338474214077,-0.052694160491228104,-0.05680311471223831,0.07740502804517746,0.014134937897324562,0.01582852564752102,0.04530501365661621,-0.01779060624539852,-0.0034023604821413755,0.034191057085990906,-0.008953600190579891,-0.04675019904971123,0.03405787795782089,-0.05516207590699196,-0.0331571102142334,0.03463934734463692,0.03951836749911308,0.045646801590919495,0.08700145781040192,-0.021053073927760124,0.03031427040696144,0.04204704239964485,-0.06396111845970154,-0.021366620436310768,0.055632565170526505,0.06628534197807312,0.011044595390558245,0.023442840203642845,-0.06346933543682098,0.009798994287848473,-0.03794090449810028,-0.005832528229802847,0.018361855298280716,0.01350889541208744,0.0036758568603545423,0.006702398415654898,-0.06205897405743599,0.02370033785700798,0.01307986956089735,-0.024200456216931343,0.03150784224271774,-0.06264550238847733,0.008132128044962883,0.03215835615992546,-0.022436605766415596,-0.05212349072098732,-0.04270476475358009,-0.0023329537361860275,0.04074603691697121,-0.00034804418100975454,-0.02878635935485363,0.0326860211789608,-0.04691816493868828,-0.027600213885307312,-0.04232899472117424,0.026438439264893532,0.030102325603365895,-0.022761736065149307,0.04520098865032196,0.014592171646654606,-0.08947356790304184,-0.008131437934935093,0.03434465453028679,0.003175870981067419,0.06444312632083893,0.11125514656305312,-0.04722969979047775,0.04326571151614189,0.04153771698474884,0.02441215142607689,-0.04339403659105301,0.027884606271982193,0.016277100890874863,0.001354020438157022,0.029810333624482155,-0.07844391465187073,0.00012162351777078584,0.025651518255472183,-0.03053232654929161,0.022365981712937355,0.04597897455096245,0.01564561203122139,0.01933920942246914,0.026589788496494293,0.0070913853123784065,-0.021856026723980904,0.04001093655824661,0.0023146492894738913,0.030311431735754013,0.01527166087180376,-0.003434104612097144,-0.022234871983528137,0.03528813272714615,-0.012212280184030533,-0.06713730841875076,-0.03727640584111214,-0.03919104486703873,-0.042998187243938446,0.01903083361685276,0.03523055836558342,-0.02004680223762989,-0.0627378597855568,0.02348332665860653,-0.03302530571818352,-0.06233473122119904,0.015835851430892944,-0.04678987339138985,0.046145956963300705,0.009829234331846235,0.02449983172118664,-0.014533326029777529,-0.033581286668777466,0.007751382421702147,-0.021691903471946716,-0.030849551782011982,0.04839356616139412,0.0009275057818740606,-0.05922368168830872,0.029963523149490356,0.013024741783738136,0.044179514050483704,-0.010739225894212725,0.029228530824184418,-0.006955612916499376,-0.01801636628806591,-0.004379132762551308,0.05127241462469101,-0.043530017137527466,-0.03265959396958351,8.339640771737322e-6,0.028594039380550385,0.023887285962700844,0.030612466856837273,0.01340977381914854,-0.01515811774879694,-0.04258834198117256,0.011820703744888306,0.005850526969879866,0.007696135900914669,0.03642495721578598,-0.04136204347014427,0.033474039286375046,0.04665401950478554,-0.020755795761942863,-0.05175543576478958,-0.011801068671047688,0.01979146338999271,-0.1011274829506874,-0.010468972846865654,0.03152601420879364,-0.002929197158664465,0.008459026925265789,0.0282197929918766,-0.012887351214885712,0.0019427110673859715,-0.034540340304374695,0.06254729628562927,-0.046808332204818726,0.044418830424547195,0.01523094903677702,0.024555182084441185,-0.017924685031175613,-0.03422831743955612,0.010746014304459097,-0.015769533812999725,-0.013998780399560928,-0.042356595396995544,-0.03463457524776459,0.05812117084860802,0.04693581163883209,-0.029332244768738747,-0.014935720711946487,-0.0000972671914496459,-0.04042451083660126,0.008905543014407158,0.01697538048028946,0.003914837259799242,-0.005139733199030161,0.006699599325656891,-0.017167923972010612,0.006805749610066414,0.008924545720219612,-0.013424907810986042,-0.0247688926756382,0.05148867517709732,-0.025626132264733315,-0.004478545859456062,0.006930702831596136,0.013430983759462832,0.012988094240427015,0.029558787122368813,0.03083486109972,0.013261702843010426,-0.033168692141771317,0.11005906015634535,0.05187990516424179,-0.011008723638951778,0.0018848247127607465,0.006288467440754175,0.013362142257392406,-0.010404403321444988,-0.002019078703597188,0.07607491314411163,-0.006623558234423399,-0.024070706218481064,-0.0660141184926033,0.019154295325279236,0.011698362417519093,0.02689356356859207,0.035133641213178635,0.01775795966386795,0.028195640072226524,0.03432346507906914,0.05861923843622208,-0.08589104562997818,-0.06982661783695221,-0.029040830209851265,0.01945418305695057,-0.04105202853679657,0.02007036842405796,0.04102747142314911,-0.02390601672232151,0.02356366068124771,0.07057029753923416,0.010388636030256748,-0.04105761647224426,-0.051813941448926926,0.02457430586218834,0.011568929068744184,-0.01378251425921917,-0.010561546310782433,-0.032571353018283844,0.00011439097579568624,0.0014271580148488283,0.06461840122938156,-0.027562851086258888,-0.030330656096339222,-0.07353313267230988,-0.011860213242471218,0.00888910237699747,0.021222809329628944,0.021000543609261513,0.011381999589502811,-0.056113775819540024,-0.0012506572529673576,0.0751735046505928,0.05051180720329285,-0.052414242178201675,0.02466917410492897,0.02122385986149311,-0.0203102994710207,-0.08079605549573898,0.005896106828004122,-0.013596602715551851,-0.016475245356559753,-0.042370639741420746,-0.06524480134248734,-0.00002611751369840931,-0.023253286257386208,-0.120810367166996,-0.02430086024105549,-0.01267726719379425,0.01090111956000328,-0.028435619547963142,0.03351210057735443,0.01764466054737568,0.004608475603163242,-0.11966228485107422,0.02141882106661797,0.03127092495560646,-0.07656203955411911,0.026353156194090843,0.013433510437607763,0.024257900193333622,0.00933617539703846,0.016023335978388786,-0.05248995125293732,0.03406073898077011,0.020285822451114655,0.00862362515181303,0.008917372673749924,-0.0690029114484787,-0.07311654835939407,-0.02589438296854496,-0.023764587938785553,-0.04801294207572937,-0.004290859214961529,0.01660720445215702,0.00982246734201908,0.04253216460347176,0.04932695627212525,-0.09203673899173737,0.03849088400602341,0.05610467121005058,-0.08239351958036423,0.011575081385672092,0.02106906846165657,-0.024943551048636436,-0.04031698405742645,-0.018404139205813408,0.06739187240600586,-0.017091965302824974,0.00913706049323082,0.016931088641285896,-0.05996422469615936,-0.03801806643605232,-0.01521415263414383,0.0034720792900770903,-0.10751175880432128,-0.008238452486693859,-0.004784609191119671,0.08470816910266876,0.02589845284819603,0.04698886349797249,-0.06866174936294556,-0.011215539649128914,0.020399078726768497,0.06725049018859863,-0.0415940135717392,0.0068704974837601185,-0.016965921968221664,-0.002564793219789862,-0.031255632638931274,-0.022184720262885097,-0.024415910243988037,-0.0016498793847858906,-0.031855545938014984,0.022277912124991417,-0.03133244439959526,-0.02305314876139164,0.038785092532634735,-0.003782161045819521,0.01159192342311144,0.03064240701496601,0.04735832288861275,-0.006815187633037567,-0.08163338154554367,0.010241907089948654,0.022931961342692375,0.08842751383781433,-0.00968953501433134,-0.09301009774208067,0.02974468283355236,-0.002882078057155013,-0.04259596019983291,-0.08483448624610901,0.0027559383306652308,-0.01285142544656992,0.013232327066361904,-0.035183895379304886,0.026714997366070747,-0.010742233134806156,-0.028570298105478287,-0.0066695851273834705,-0.032421622425317764,0.044481292366981506,-0.02830183133482933,-0.026018967851996425,-0.027817178517580032,0.01799335516989231,-0.010149057023227217,-0.014487781561911106,0.01809072680771351,0.018718933686614037,-0.05527259409427643,-0.028456931933760643,0.014944292604923248,-0.04119318723678589,-0.0036447637248784304,0.05321866646409035,-0.002055675722658634,0.02000885270535946,-0.06110333278775215,-0.03416769579052925,-0.0033835622016340494,-0.019265498965978622,0.00003057794674532488,0.013020576909184456,-0.030504781752824783,-0.00651447894051671,-0.055143579840660095,0.021179838106036183,0.012172173708677292,0.026151705533266068,-0.0038829820696264505,-0.0338832251727581,0.05517534911632538,0.004608093295246363,-0.004879816435277462,0.031797025352716446,-0.01672358252108097,0.016970640048384666,-0.07502707839012146,-0.015029761008918284,0.025808729231357574,0.02299804612994194,-0.000589223112910986,-0.01322252955287695,0.02367742918431759,-0.015922769904136658,-0.007459308486431837,-0.0528251975774765,-0.029248273000121117,-0.022481903433799744,-0.034382447600364685,-0.05227747559547424,-0.018499115481972694,-0.04573157057166099,0.020755542442202568,-0.01611945964396,-0.02249385230243206,0.00485266512259841,0.004456853028386831,0.049704112112522125,0.01603599265217781,0.0044006770476698875,0.019944433122873303,-0.03407353162765503,0.0628838911652565,0.022810539230704308,0.041649095714092255,-0.03474824130535126,-0.009630289860069752,-0.024129632860422134,0.01407462265342474,0.08002449572086334,0.08051567524671555,0.006202539429068565,0.035148341208696365,-0.04215238615870476,-0.01701504923403263,-0.011841303668916224,-0.012014812789857388,0.03865235671401024,-0.016479332000017166,-0.029955057427287105,0.021419379860162735,-0.017217006534337997,-0.004550143610686064,-0.004089713096618652,0.008453886024653912,0.028843024745583538,0.05277089402079582,0.022193972021341324,-4.784573881191663e-33,0.0018227333202958107,-0.04885735735297203,0.03202769532799721,0.05321668088436127,0.007685084361582995,0.021824494004249573,0.055888451635837555,-0.031199580058455467,-0.045234594494104385,-0.0007119972142390907,0.00549817131832242,-0.008715281262993813,0.00644097663462162,-0.00858980044722557,0.016113296151161194,0.0378011092543602,-0.004252283833920956,0.020844686776399612,-0.0009098300943151116,-0.024664482101798058,0.00850024912506342,-0.01667683757841587,0.08643721044063568,-0.017272043973207474,0.03856298327445984,-0.0024644562508910894,-0.021116791293025017,-0.0025913112331181765,0.001981882844120264,0.01750882901251316,-0.04024527594447136,0.02778775431215763,-0.011554842814803123,-0.010239205323159696,-0.03448633849620819,0.0004324669134803116,-0.04290112853050232,0.019592344760894775,-0.048738256096839905,0.03785990551114082,0.008226130157709122,-0.0324641652405262,0.0028055687434971333,0.03877320885658264,-0.03486292436718941,-0.04881686344742775,0.017001822590827942,-0.027564343065023426,0.03407733514904976,0.02994762733578682,-0.05727822706103325,0.004448353312909603,0.011789443902671335,-0.0022656763903796673,0.010585201904177666,-0.01704205945134163,0.005667113233357668,0.001843595178797841,-0.03346922621130943,-0.0104334969073534,0.036996990442276,-0.02001471072435379,0.037737488746643066,0.006436967756599188,0.0003901989839505404,0.021604998037219048,0.031449850648641586,0.05990062281489372,0.046629976481199265,-0.020015176385641095,-0.02967577986419201,0.03384219482541084,-0.017502842471003532,-0.013836282305419443,0.014355508610606194,-0.03705321624875069,-0.04597638919949531,-0.002131591085344553,-0.004005508963018656,0.027276160195469856,0.04003189131617546,0.0042450265027582645,-0.01169614214450121,-0.006779146380722523,-0.0021899561397731304,0.03148511052131653,-0.006856205873191357,0.024255922064185143,-0.003992144018411636,0.011215374805033209,-0.04009361565113067,0.06855133920907974,-0.07205209881067276,-0.0365125872194767,0.08633728325366974,0.003412778489291668,-0.0456111766397953,-0.04388313367962837,-0.03112998977303505,-0.011303132399916649,0.0021985098719596863,-0.006745655555278063,0.020911239087581635,0.026718011125922203,0.05388464406132698,0.02258460596203804,-0.0519089438021183,-0.017298445105552673,-0.03208953142166138,-0.029945630580186844,-0.009361812844872476,-0.014641579240560532,0.02630343846976757,0.07037889957427979,0.011681154370307922,0.00967866275459528,0.012222493067383766,-0.04879191890358925,0.02241196483373642,-0.015588481910526752,0.026590630412101746,0.050404783338308334,0.055015698075294495,0.021289803087711338,0.03977291285991669,-0.043699778616428375,0.0008924695430323482,-0.08056236058473587,-0.0026137398090213537,0.035361431539058685,0.011045957915484904,-0.01674063503742218,2.2601911098263375e-7,0.01690332032740116,0.0023277923464775085,-0.039466723799705505,0.07288116961717606,0.006613120436668396,0.00982262659817934,-0.0052389525808393955,0.022709008306264877,-0.005889535415917635,0.016438914462924004,0.02171887829899788,-0.0585668720304966,-0.03254368528723717,-0.023351985961198807,-0.047077108174562454,-0.016803201287984848,-0.03491942957043648,-0.017735624685883522,0.010713264346122742,-0.015618734061717989,0.0560283400118351,0.02372867800295353,0.040302641689777374,-0.0038569283206015825,-0.023130228742957115,-0.04228193685412407,-0.018794188275933266,0.03092249296605587,0.05173787847161293,0.065462127327919,0.011748508550226688,0.0063350992277264595,0.025835271924734116,0.07680980861186981,-0.03813736140727997,0.025843750685453415,0.06936757266521454,0.051666755229234695,-0.0011307119857519865,0.08434838801622391,0.0047728088684380054,0.0025580639485269785,0.04044389724731445,-0.0340828038752079,-5.191628247303015e-7,0.008670873008668423,-0.0007192999473772943,-0.04794353246688843,-0.029808130115270615,0.016312437132000923,-0.009208285249769688,-0.011225633323192596,-0.011009822599589825,-0.029704969376325607,0.0004324363835621625,0.025289973244071007,-0.025384116917848587,-0.01764563098549843,0.025983620434999462,0.040095824748277664,-0.038067903369665146,-0.03436708450317383,0.009237819351255894,0.018956948071718216,0.12840363383293152,-0.04317876696586609,0.010139450430870056,1.9797942179487824e-34,0.03702948614954949,0.0456843264400959,-0.05728635564446449,0.012524413876235483,-0.0028274597134441137,-0.027879653498530388,-0.03213181346654892,0.006487383507192135,0.024529408663511276,-0.07162538915872574,-0.030150746926665303]}'},\n", + " {'$': '{\"user\":\"stacy\",\"age\":61,\"job\":\"project manager\",\"region\":\"us-west\",\"job_description\":\"Plans, organizes, and oversees projects from inception to completion.\",\"job_embedding\":[0.02899340912699699,0.022221779450774193,-0.022562529891729355,0.01775331050157547,0.011595080606639383,0.03318494185805321,-0.03774332627654075,-0.06087971478700638,-0.09546949714422226,0.015694674104452133,0.04026399925351143,0.0013510221615433693,-0.01312387641519308,0.05164538696408272,-0.02006269618868828,-0.04799939692020416,0.019323324784636497,-0.017591534182429314,-0.023619284853339195,0.06035340949892998,-0.038550786674022675,0.03013814613223076,-0.02146303467452526,0.07938672602176666,-0.0276190098375082,-0.023233754560351372,-0.010895739309489729,0.012868753634393215,-0.009521513245999811,-0.0376834012567997,0.05846494063735008,-0.017441993579268456,0.03026852384209633,-0.05034729093313217,1.6073147435236024e-6,0.011279323138296604,0.03687521815299988,-0.040978990495204926,0.028986379504203796,-0.054673053324222565,0.0012970375828444958,-0.0775204747915268,0.04780450835824013,-0.01408819667994976,0.019979899749159813,-0.005178736988455057,0.019342752173542976,0.04053046926856041,-0.030579671263694763,-0.007991042919456959,0.018901342526078224,-0.05974676087498665,-0.05810188502073288,-0.04178774356842041,0.006047445349395275,-0.024575646966695786,0.05659046396613121,-0.007456447463482618,-0.05602142959833145,0.01545725017786026,-0.0014964324655011296,-0.024125713855028152,-0.06319190561771393,-0.009629439562559128,0.05754568427801132,0.006331864278763533,0.06884288787841797,-0.07598119229078293,0.07088304311037064,-0.031324345618486404,0.08138009160757065,-0.02642546221613884,-0.019187081605196,0.013823003508150578,0.008002779446542263,0.0153226675465703,-0.017653340473771095,-0.012652626261115074,-0.0015036625554785132,-0.029615318402647972,0.02882424183189869,0.07693707942962646,0.029697628691792488,0.020671481266617775,-0.03164990618824959,0.05523265898227692,-0.01897629164159298,-0.008170593529939651,0.010558268055319786,-0.0032612106297165155,0.06387568265199661,0.00295635056681931,-0.00881296768784523,0.028685005381703377,-0.0014933819184079766,-0.015905242413282394,-0.03481225669384003,-0.04432761296629906,0.018268099054694176,0.030272837728261948,-0.023790646344423298,-0.03161105513572693,0.0054801166988909245,0.01775270886719227,0.04614558070898056,0.03945459425449371,0.04126491770148277,0.029500095173716545,-0.006738665513694286,-0.03880549594759941,-0.01820591278374195,0.0036847281735390425,-0.046038590371608734,-0.07931314408779144,0.047049157321453094,0.03120683506131172,0.020641984418034554,0.060359347611665726,-0.0096670500934124,0.02874581329524517,0.06885656714439392,0.005463944748044014,-0.0462612546980381,0.04286589473485947,-0.06774214655160904,-0.04534158483147621,0.011337755247950554,0.02642303891479969,0.038140371441841125,0.11272209882736206,-0.032396260648965836,0.03552377596497536,0.030773259699344635,-0.05863887071609497,-0.0281611792743206,0.045962899923324585,0.06000392884016037,-0.0014095907099545002,0.017855415120720863,-0.01899571530520916,-0.012234757654368876,-0.03347072750329971,-0.013007028959691525,0.022082088515162468,0.006693881005048752,0.004739072173833847,-0.009209335781633854,0.0132640628144145,0.013720308430492878,0.028113162145018578,-0.0054841842502355576,0.009455876424908638,-0.04116712138056755,0.041715554893016815,0.00375041039660573,-0.0001142399778473191,-0.092621311545372,-0.012365888804197311,-0.016185758635401726,0.062022022902965546,0.016358884051442146,0.005348714534193277,0.010256880894303322,-0.03126624971628189,-0.044333819299936295,-0.005557643249630928,0.04524016752839088,0.027474820613861084,0.007414877414703369,-0.027571694925427437,0.003585220081731677,-0.0806250348687172,-0.014823799021542072,0.050377171486616135,0.03660518676042557,0.03884173929691315,0.08296351879835129,-0.07784482091665268,0.05126720666885376,0.03270399942994118,0.007864560931921005,-0.06910388171672821,0.03841015323996544,-0.002013141056522727,0.04184100776910782,0.0275180134922266,-0.12216690927743912,0.006080565042793751,0.051564812660217285,0.0005583192105405033,0.045174308121204376,0.010165628977119924,-0.008067436516284943,0.015561893582344055,0.013746199198067188,0.03418424725532532,-0.04747231677174568,0.03927505016326904,-0.007226229645311832,0.024768490344285965,0.01117383036762476,0.030652273446321487,-0.05128197744488716,0.023976121097803116,-0.011049844324588776,-0.07602787017822266,-0.04511508718132973,-0.042434126138687134,-0.04981108754873276,0.04256719723343849,0.03273313492536545,-0.04211166873574257,-0.029810501262545586,0.044769808650016785,-0.04263051599264145,-0.05121579021215439,0.02490888349711895,-0.03545086085796356,0.03444785997271538,-0.00008794992754701525,-0.001775247510522604,-0.04350770264863968,-0.017021387815475464,0.034028153866529465,0.016784099861979485,-0.05064787715673447,0.02993095852434635,0.0007091201259754598,-0.024462949484586716,0.017619416117668152,0.01144897285848856,0.02140048146247864,0.02330016903579235,0.03545387461781502,0.03325653076171875,-0.028879139572381973,-0.009972716681659222,0.05185086652636528,-0.05497191846370697,0.04533546417951584,-0.015172622166574,0.023470783606171608,0.01561465673148632,0.012190023437142372,0.02080681174993515,0.007853937335312366,-0.04236825555562973,0.00967771839350462,0.008191576227545738,-0.007494522258639336,0.020564980804920197,-0.012621174566447737,0.04388607665896416,0.018242638558149334,-0.05339835956692696,0.01554061658680439,-0.0369630865752697,0.018019309267401695,-0.06558918207883835,0.0001234902738360688,0.06388071179389954,-0.02152351103723049,-0.0033629180397838354,0.03536687046289444,0.0006709382287226617,-0.005155568011105061,-0.05015704780817032,0.025362877175211903,-0.021528076380491257,0.013195478357374668,0.02763737179338932,0.03600696101784706,-0.007783942855894566,-0.041976407170295715,-0.0005980943096801639,0.0012860780116170645,-0.019358649849891663,-0.03537020832300186,-0.021252864971756935,0.05130600184202194,0.03604046627879143,-0.0054163504391908646,-0.014503681100904942,0.017356928437948227,-0.008085779845714569,0.005986441858112812,-0.03478115424513817,-0.06712129712104797,0.032763902097940445,-0.03020450659096241,-0.001935127074830234,-0.019945502281188965,0.012240277603268623,-0.01206696406006813,-0.02010153979063034,0.05605268478393555,-0.008741901256144047,-0.030393673107028008,-0.005795078817754984,0.05135184153914451,0.02401907928287983,-0.0006247475394047797,0.06716427206993103,0.007028155494481325,-0.011821243911981584,0.06940178573131561,0.00459287129342556,0.022265024483203888,-0.0053640552796423435,0.025676442310214043,0.020665928721427917,0.015102864243090153,-0.007055232301354408,0.018226325511932373,0.015597259625792503,-0.006409869529306889,-0.04441540688276291,-0.010553057305514812,0.015812475234270096,0.06207884103059769,-0.013640359044075012,0.000822719419375062,-0.009353091940283775,0.017377926036715508,0.0295860655605793,-0.02030998468399048,-0.07388705015182495,-0.017246613278985023,0.04608146846294403,-0.013279210776090622,0.012983485125005243,-0.02029039151966572,-0.04116249084472656,0.01865767501294613,0.05679427087306976,-0.01816736720502377,-0.04575057700276375,-0.05985769256949425,0.007200473919510841,-0.012534871697425842,-0.03207677602767944,0.00402563763782382,-0.059203941375017166,0.031285129487514496,-0.04103521257638931,0.05736790969967842,0.006668417248874903,-0.0031413002870976925,-0.07973791658878326,0.00527765229344368,-0.022923851385712624,0.04057155922055245,0.025743452832102776,0.019432926550507545,-0.05428042635321617,-0.052015386521816254,0.03951722010970116,0.04812880977988243,-0.08504178375005722,0.025910962373018265,0.014877649955451488,-0.04965604096651077,-0.06282525509595871,-0.031002219766378403,0.008803718723356724,-0.0030834185890853405,-0.054148945957422256,-0.042983319610357285,0.020352378487586975,-0.007267802953720093,-0.0964089408516884,-0.03326726704835892,-0.005986662115901709,-0.0020002215169370174,-0.03546881675720215,-0.016371745616197586,0.02580946870148182,0.019323263317346573,-0.0980563387274742,0.03529144451022148,0.0009229640127159656,-0.049863334745168686,-0.006684139836579561,0.009105962701141834,0.021623384207487103,0.021785207092761993,0.040878936648368835,-0.04103006422519684,0.06416049599647522,-0.005437395069748163,0.013484850525856018,-0.02670535258948803,-0.030505187809467316,-0.038110051304101944,-0.019587727263569832,-0.040954913944005966,-0.05480985715985298,0.013947195373475552,0.020745523273944855,-0.009305136278271677,0.03824371099472046,0.07438687235116959,-0.048929713666439056,0.03242180123925209,0.042388468980789185,-0.07068920880556107,0.05907871946692467,0.013633069582283497,0.007640390656888485,0.023061521351337433,-0.04483378678560257,0.04912187159061432,-0.02704537473618984,0.047115061432123184,0.03917886316776275,-0.06537540256977081,-0.023625213652849197,-0.015600201673805714,-0.032025665044784546,-0.08146513998508453,0.0006068347138352692,-0.010904096066951752,0.06126230210065842,0.015069513581693172,0.02711573988199234,-0.0368720218539238,0.004392314702272415,0.0436587780714035,0.0535908006131649,-0.014576268382370472,0.014891527593135834,0.0021130815148353577,0.001315235043875873,-0.000696890871040523,-0.029158812016248703,0.011453872546553612,0.017220554873347282,-0.02174423635005951,0.055330440402030945,-0.021879486739635468,-0.0158648993819952,0.06896545737981796,-0.029748696833848953,0.013656912371516228,0.020143793895840645,0.023150911554694176,-0.03449635207653046,-0.08347748965024948,-0.038106366991996765,0.04878976941108704,0.03560120612382889,-0.008534654043614864,-0.12233560532331468,0.018796350806951523,0.00027647402021102607,0.007821233943104744,-0.016531458124518394,-0.0612298808991909,0.020110314711928368,-0.012997529469430448,-0.03882286325097084,0.02360977604985237,-0.034570299088954926,-0.005298761185258627,0.00427410239353776,-0.004706206731498241,0.029401957988739017,-0.0017745428485795856,-0.019665274769067764,-0.03141898661851883,0.06517744809389114,-0.008254026994109154,0.0028763399459421635,-0.022861722856760025,0.0284231249243021,-0.09018309414386748,-0.01176224648952484,0.025817502290010452,0.0011024848790839314,-0.006964212749153376,0.0064245485700666904,0.02097048796713352,0.011497818864881992,-0.0690450370311737,-0.03388099744915962,0.002166322199627757,-0.0603577122092247,-0.0025102843064814806,-0.008472194895148277,-0.008947010152041912,0.0044541931711137295,-0.06980546563863754,0.02993505820631981,0.03187485784292221,0.01506261806935072,-0.00011850192822748797,-0.015610304661095142,-0.017142388969659805,0.03832258656620979,-0.049084894359111786,-0.0015871316427364943,0.0004924146924167871,0.0018951698439195752,-0.06625988334417343,-0.008354702033102512,-0.010153645649552344,0.030238261446356773,-0.013597944751381874,-0.02586783468723297,-0.011644991114735603,-0.0011330911656841636,-0.030920976772904396,-0.01748080551624298,-0.005832865368574858,0.040499042719602585,-0.04593607783317566,-0.027553223073482513,-0.04763009399175644,0.017689773812890053,-0.007589668966829777,0.0026824709493666887,-0.04387638717889786,-0.016238385811448097,0.006077826488763094,0.01659487746655941,0.003587604034692049,-0.00783101748675108,0.007484964095056057,-0.005921813193708658,0.07263810932636261,0.03667611628770828,0.02403925359249115,-0.04061540588736534,-0.0045168413780629635,-0.019925609230995175,0.01094149611890316,0.061878036707639694,0.11308151483535768,0.00401971023529768,0.028771914541721344,-0.028805287554860115,-0.0066599855199456215,-0.010868588462471962,-0.02092607505619526,0.03430075943470001,-0.02841309644281864,-0.06681492924690247,-0.0016281991265714169,-0.016458885744214058,-0.02584969252347946,-0.01766178570687771,-0.03351273015141487,0.041557226330041885,0.060653191059827805,-0.006632203236222267,-4.577480429251577e-33,-0.003511193208396435,-0.0784594863653183,0.046610232442617416,0.004169749561697245,0.012499948963522913,0.04108119755983353,0.061884015798568726,-0.03442469239234924,-0.01959601230919361,0.006686790846288204,0.03088124841451645,-0.00050581363029778,0.009372697211802006,0.021210940554738045,0.02361816167831421,0.0516207180917263,0.02241726778447628,-0.008887053467333317,0.018575534224510193,0.003119019325822592,0.021224765107035637,-0.03410238400101662,0.05394582077860832,0.01271009724587202,0.03942097723484039,-0.012118754908442496,-0.027356578037142754,0.014077930711209774,0.030438797548413277,0.014959615655243397,-0.02607175335288048,-0.00819804985076189,-0.004419455770403147,0.029766356572508812,-0.01709524169564247,-0.005113645922392607,-0.04376103729009628,0.03421781584620476,-0.04806535318493843,-0.0027068578638136387,-0.02004993706941605,-0.004862269386649132,0.0070143043994903564,0.040554292500019073,-0.013729936443269253,-0.035344742238521576,-0.015810266137123108,-0.03132788464426994,0.010773558169603348,0.04990176483988762,-0.0645173192024231,-0.0003064009652007371,0.009702498093247414,-0.004010362550616264,0.03657590225338936,-0.034525640308856964,0.013817064464092256,0.03857770934700966,-0.027067728340625763,-0.01855241134762764,-0.03950805217027664,-0.02582256682217121,0.055197060108184814,0.0748237669467926,-0.001956295920535922,0.02929081581532955,0.024861492216587067,0.1101442277431488,0.02294003963470459,0.03296205401420593,-0.04798106104135513,0.06175709515810013,-0.02985440194606781,-0.03943919017910957,-0.013791701756417751,-0.010075697675347328,-0.005336214788258076,-0.023220904171466827,0.034165095537900925,0.025616582483053207,0.03364214301109314,-0.005315003450959921,-0.0001777073193807155,-0.007894779555499554,0.00213141948916018,-0.03005058504641056,-0.0059772455133497715,0.05054880678653717,0.01813163422048092,0.015980517491698265,-0.030451416969299316,0.08544710278511047,-0.10132600367069244,-0.03238682448863983,0.1033436730504036,0.029700104147195816,-0.07877960801124573,-0.054472871124744415,-0.019467292353510857,-0.024977032095193863,0.03922523185610771,0.009095796383917332,0.041835617274045944,0.0472467765212059,0.043953560292720795,0.03204527869820595,-0.038687948137521744,-0.03909479081630707,-0.021118242293596268,-0.0381355844438076,-0.022257357835769653,0.01963324286043644,0.02841278724372387,0.03441145271062851,0.01299949176609516,-0.041159506887197495,0.008455917239189148,-0.04823213070631027,0.026359373703598976,-0.03670600429177284,0.04097040742635727,0.021798525005578995,0.01054143998771906,-0.013165436685085297,0.03595249354839325,-0.037208229303359985,-0.02308909222483635,-0.06502506881952286,0.031897012144327164,0.04026839882135391,-0.005874866619706154,-0.025206975638866425,2.3663429260523117e-7,0.023632880300283432,0.032257888466119766,-0.014513743110001087,0.06451402604579926,0.028107134625315663,0.02249029651284218,-0.01599108800292015,0.016707494854927063,-0.035814620554447174,-0.025366351008415226,0.03054198808968067,-0.04490065947175026,-0.013542436063289642,0.0203291866928339,-0.015586487017571926,-0.023771440610289577,-0.04805871099233627,-0.022885285317897797,-0.044002242386341095,-0.016578102484345436,0.041788384318351746,0.013753644190728664,0.02764694206416607,-0.004267602693289518,0.004687544424086809,-0.05506843328475952,-0.035745855420827866,0.03286490589380264,0.03702550381422043,0.03900880739092827,0.004297059960663319,-0.01005226094275713,0.009467637166380882,0.0458277128636837,-0.022131582722067833,0.01544195506721735,0.053432464599609375,0.03308882936835289,-0.008957459591329098,0.06029060110449791,0.012929418124258518,-0.012491658329963684,0.029766973108053207,-0.012415618635714054,-0.0060104867443442345,0.02782285585999489,0.00877326913177967,-0.01737518422305584,0.0011388895800337195,0.0354909710586071,-0.006772330496460199,-0.014612244442105292,-0.004996708128601313,-0.019461655989289284,0.0029080286622047424,0.023209845647215843,-0.04891340434551239,-0.00630920147523284,0.036900755017995834,0.00746958376839757,-0.01776387169957161,-0.019362464547157288,-0.005619883071631193,0.028677061200141907,0.08796064555644989,-0.03479388356208801,0.02049041911959648,1.7836961610870706e-34,0.027522211894392967,0.030901066958904263,-0.06743663549423218,-0.06642237305641174,0.007514960132539272,-0.05602094531059265,-0.05617784336209297,0.017062336206436157,-0.0060576810501515865,-0.06948957592248917,-0.02218511514365673]}'},\n", + " {'$': '{\"user\":\"mary\",\"age\":24,\"job\":\"doctor\",\"region\":\"us-central\",\"job_description\":\"Diagnoses and treats illnesses, injuries, and other medical conditions in the healthcare field.\",\"job_embedding\":[0.04454879462718963,0.01708805374801159,-0.004231484141200781,-0.023331958800554276,-0.004114257637411356,0.04582072794437408,-0.014009997248649595,-0.002039531944319606,-0.011973315849900246,-0.025282446295022964,0.051192134618759155,-0.01052328385412693,0.007922729477286339,0.01877511292695999,-0.034461963921785355,-0.0320771150290966,0.024941056966781616,0.021213117986917496,-0.0037147151306271553,0.046645816415548325,-0.07126107066869736,0.02736794203519821,-0.02877422980964184,0.07668906450271606,0.011690506711602213,-0.03777015954256058,0.03492218628525734,-0.0276932492852211,0.016995232552289963,-0.02370324172079563,0.04932255297899246,-0.04759824648499489,0.007767138071358204,-0.04853592813014984,1.4349043340189382e-6,0.022479061037302017,0.011136106215417383,0.009119989350438118,0.006830011960119009,-0.09861305356025696,-0.007858938537538052,-0.06678599119186401,0.03449978679418564,-0.00228167325258255,0.003032505977898836,-0.02371986024081707,0.012271668761968613,0.0003364636213518679,-0.021755250170826912,0.02545011229813099,0.005683752708137035,-0.04911157488822937,-0.06845685839653015,-0.018949005752801895,-0.005360751878470182,0.009578952565789224,0.06687026470899582,-0.0035978632513433695,-0.00024639046750962734,-0.025723516941070557,-0.007864813320338726,-0.013673651032149792,-0.07835397869348526,0.0011555858654901383,0.0535251758992672,0.026353850960731503,0.064972884953022,-0.02111617475748062,0.07157499343156815,-0.02616978995501995,0.03770754858851433,-0.01900457963347435,-0.001560426433570683,0.01771584525704384,-0.004259844776242971,0.02846275083720684,0.00864716712385416,-0.035943470895290375,-0.02413034997880459,-0.03621584177017212,0.03620774671435356,0.08082558959722519,0.03596339002251625,0.04656396806240082,-0.03187175467610359,0.01857466995716095,-0.01143748126924038,-0.012450099922716618,0.030144277960062027,-0.020370569080114365,0.0729600340127945,-0.0010903574293479323,0.011908376589417458,0.052168603986501694,-0.02338007278740406,-0.0053266133181750774,-0.04380548745393753,-0.06202943995594978,0.03958531096577644,0.007586859166622162,-0.0561363510787487,-0.044201064854860306,0.02303418703377247,0.027441615238785744,0.02044270746409893,-0.026095302775502205,0.02256368286907673,0.03689037263393402,-0.01766049489378929,-0.024111388251185417,-0.03620058298110962,-0.02400734461843967,-0.045380931347608566,-0.04568415880203247,0.07655256241559982,0.024233654141426086,-0.0034445002675056458,0.009736689738929272,-0.03573869541287422,0.019135640934109688,0.054537151008844376,0.0042672595009207726,-0.06086820363998413,0.05436275526881218,-0.036123961210250854,-0.025657735764980316,0.028698230162262917,0.024990512058138847,0.035168301314115524,0.09065953642129898,-0.0014619540888816118,0.043459389358758926,0.0579066202044487,-0.06697030365467072,-0.02349095977842808,0.041562583297491074,0.0636909231543541,0.007863709703087807,0.04288958758115768,-0.05693717300891876,-0.01225106418132782,-0.028272535651922222,0.01975945010781288,0.029778528958559036,0.025035079568624496,-0.00521258357912302,0.03959973156452179,-0.04591942206025123,0.027935663238167763,0.018903931602835655,-0.05500571429729462,0.02378286048769951,-0.0876990482211113,0.0025745911989361048,0.03144952282309532,-0.016654644161462784,-0.034540124237537384,-0.07020001113414764,-0.013680118136107922,0.014383431524038317,-0.006941581144928932,-0.019812192767858505,0.021119965240359303,-0.06938156485557556,-0.027774689719080925,-0.039240919053554535,0.037650879472494125,0.044759105890989304,-0.009759747423231602,0.047071672976017,0.0046951971016824245,-0.04476232826709747,-0.02534623257815838,0.011900904588401318,0.006286389660090208,0.082516148686409,0.08721663802862167,-0.0886007770895958,0.03757705911993981,0.003516430733725429,0.002847470575943589,-0.04662508890032768,0.033441636711359024,-0.03363692760467529,0.05559864640235901,0.01845979318022728,-0.09541022032499312,-0.03362889587879181,0.03452971577644348,-0.029589587822556496,0.00173288700170815,0.020156847313046455,0.006540464237332344,0.022147342562675476,0.008532282896339893,0.0055303857661783695,-0.036800213158130646,0.05384651571512222,0.02478555031120777,0.012056545354425907,0.0030047856271266937,-0.04675056412816048,-0.045408688485622406,0.06781681627035141,0.008645018562674522,-0.05245037376880646,-0.04495980218052864,-0.05148840323090553,-0.051179636269807816,0.008254407905042171,0.005847238935530186,-0.03805410489439964,-0.039817534387111664,0.025147825479507446,-0.005727674346417189,-0.018419116735458377,0.05211011320352554,-0.046494148671627045,0.019879376515746117,0.025583740323781967,0.006601599045097828,-0.036587707698345184,-0.028219331055879593,0.02101061679422855,-0.0019962110091000795,-0.014952527359127998,0.028041396290063855,0.00857739057391882,-0.033402130007743835,0.02591954544186592,0.025621561333537105,0.037654709070920944,-0.018004007637500763,0.03544408828020096,0.021492844447493553,-0.00305945985019207,-0.008734425529837608,0.04076306521892547,-0.057831209152936935,0.01086723618209362,0.031098252162337303,0.016671450808644295,0.051472995430231094,0.01705809496343136,-0.013820058666169643,-0.009514417499303818,-0.04166558384895325,0.015360153280198574,0.006977487355470657,0.05641806498169899,0.016833841800689697,-0.023014593869447708,0.028362737968564034,0.017753930762410164,-0.04215700924396515,-0.05559220910072327,-0.016122838482260704,-0.0012042437447234988,-0.06628667563199997,0.008465351536870003,0.03027988038957119,-0.013648975640535356,0.02675759606063366,0.008728794753551483,-0.022565793246030807,0.014110655523836613,-0.032644934952259064,0.007087731268256903,-0.03860873356461525,0.03420965000987053,0.0023368769325315952,0.0292284544557333,0.001866739592514932,-0.0209802333265543,0.005405825562775135,-0.017271829769015312,-0.017037995159626007,-0.013564104214310646,0.01290924847126007,0.04597386717796326,0.04843093454837799,0.004206886049360037,-0.01136064436286688,0.01587117835879326,-0.01768266223371029,-0.02701226808130741,-0.028409920632839203,-0.013419064693152905,0.04180018976330757,-0.038482505828142166,-0.01593782752752304,0.006915387697517872,0.0001980535889742896,-0.03897705674171448,-0.03325193375349045,0.04142800718545914,-0.05619407817721367,0.01642300747334957,-0.01461863238364458,0.009437954984605312,0.059236250817775726,0.017152493819594383,0.05772939696907997,0.028031358495354652,-0.016169492155313492,0.05247999727725983,0.0305689238011837,-0.024128831923007965,0.003912066575139761,0.028506793081760406,0.02835055999457836,-0.008687603287398815,-0.0013867092784494162,0.012066920287907124,-0.020278438925743103,0.009707252494990826,-0.09203914552927016,0.005283826030790806,-0.009980288334190844,0.028586482629179955,0.02765020541846752,0.006529873702675104,0.007977577857673168,0.03355500102043152,0.06742902845144272,-0.05960248038172722,-0.06476552784442902,-0.012493600137531756,0.0032982085831463337,-0.02083723060786724,0.023411467671394348,0.008107160218060017,-0.021757733076810837,0.06214763969182968,0.04795828461647034,-0.003971832804381847,-0.07196298241615295,-0.020796969532966617,0.03200780972838402,0.005982197355479002,-0.018063349649310112,-0.004748756531625986,-0.047417379915714264,0.022266093641519547,-0.01666136458516121,0.040921371430158615,-0.03067716211080551,0.02376369945704937,-0.08084520697593689,0.004314294084906578,-0.05981266871094704,0.06861262768507004,0.04792516678571701,0.01269147451967001,-0.05850010737776756,-0.005192546173930168,0.04480767622590065,0.038581788539886475,-0.0933382734656334,0.01952129229903221,0.011136033572256563,-0.007436478976160288,-0.058613747358322144,0.014493698254227638,0.026108646765351295,-0.012605573050677776,-0.046943116933107376,-0.07293735444545746,0.011588223278522491,-0.017570968717336655,-0.08396150171756744,-0.03471928834915161,-0.01770923286676407,0.01252435613423586,-0.04128732159733772,-0.0071595339104533195,0.04673443362116814,0.020049843937158585,-0.14951065182685852,0.004564756993204355,0.030230563133955,-0.06743364036083221,-0.003531516995280981,0.020546749234199524,0.026945967227220535,0.010649639181792736,0.025063056498765945,-0.025068240240216255,0.06143159046769142,0.0115469666197896,0.004478050395846367,0.00868087075650692,-0.06838269531726837,-0.0370064340531826,-0.03411612659692764,-0.02233545109629631,-0.03186684101819992,-0.019568901509046555,0.005743507761508226,-0.00136433111038059,0.02290811948478222,0.05572489649057388,-0.06743817776441574,0.04157017543911934,0.050052009522914886,-0.06630434840917587,0.03794371336698532,0.03200172260403633,-0.013537704944610596,-0.03797779232263565,-0.06074855104088783,0.05308151990175247,-0.025430120527744293,0.03771941736340523,0.06371979415416718,-0.06982607394456863,-0.020584816113114357,-0.020050154998898503,-0.014204544015228748,-0.0897083580493927,0.00347348814830184,-0.022570496425032616,0.06362242996692657,0.013083127327263355,0.0013562330277636647,-0.047062285244464874,0.008241566829383373,0.03946557641029358,0.0891670361161232,-0.039371002465486526,0.031416889280080795,-0.011516201309859753,0.03058178909122944,0.0047472016885876656,-0.01566123217344284,0.00908429455012083,0.03415559604763985,-0.0588974691927433,0.001306998310610652,-0.03355700522661209,-0.03959766402840614,0.04483641684055328,-0.04543393850326538,0.014302295632660387,0.03526866436004639,0.043165694922208786,-0.020603759214282036,-0.08115498721599579,0.028260257095098495,0.03968700021505356,0.050793372094631195,-0.007960767485201359,-0.11488877981901167,0.042206164449453354,0.03524675965309143,-0.0195783581584692,-0.047870274633169174,0.0003281059325672686,0.03702429682016373,0.00243264134041965,-0.026458052918314937,0.01531948707997799,-0.018861733376979828,-0.014750908128917216,0.01999596692621708,-0.06191974878311157,0.054460134357213974,-0.0364711731672287,-0.03869025409221649,-0.031297143548727036,0.007825011387467384,-0.005396517924964428,0.008439796976745129,0.004476859234273434,0.026949390769004825,-0.03943302482366562,-0.024582551792263985,0.0337645448744297,0.00523300701752305,-0.009956594556570051,0.02050791308283806,0.017380613833665848,-0.00110126833897084,-0.027087420225143433,-0.02775687724351883,0.005832228343933821,-0.04882275313138962,-0.008672981522977352,-0.002882502507418394,-0.03844589740037918,0.011864435859024525,-0.05100938305258751,0.018446095287799835,0.03218577802181244,-0.0010861540213227272,-0.014580215327441692,-0.06040755659341812,0.02749667316675186,0.0023029393050819635,-0.04366978257894516,0.022118067368865013,-0.007149583660066128,0.0004462611977942288,-0.0636456236243248,0.0012036282569169998,0.007305020000785589,0.037062518298625946,0.016711514443159103,-0.017389776185154915,0.007321099285036325,0.009313591755926607,-0.013477408327162266,-0.002821192843839526,-0.002448559971526265,0.004678861703723669,-0.037384212017059326,-0.04569138586521149,-0.04702009633183479,-0.03738853335380554,0.01922869309782982,-0.011991046369075775,-0.028683876618742943,-0.0005284024518914521,0.010614288039505482,0.043833132833242416,0.02552366442978382,0.0009966266807168722,0.0052346657030284405,-0.027730410918593407,0.08081462234258652,0.03024850413203239,0.055050820112228394,0.004532320890575647,-0.004522927105426788,-0.026175834238529205,0.006993886083364487,0.07347681373357773,0.09887508302927016,-0.022610455751419067,0.05010379105806351,-0.03365675359964371,-0.028441188856959343,-0.0196791123598814,-0.051836881786584854,0.02040472626686096,0.0014423556858673692,-0.04143974557518959,-0.01614542491734028,-0.00036064122105017304,-0.023156872019171715,-0.01793116331100464,0.026601064950227737,0.018239691853523254,0.06859753280878067,0.03927658498287201,-4.3293001436723905e-33,-0.00937176588922739,-0.06005459278821945,0.04930717870593071,-0.029729144647717476,0.02810727246105671,0.054349880665540695,0.06899542361497879,-0.049275100231170654,-0.028633316978812218,0.011439881287515163,0.024168865755200383,-0.004416124429553747,0.0072797928005456924,0.034292228519916534,0.002505109878256917,0.026111984625458717,0.011239423416554928,0.007792243734002113,0.010135376825928688,-0.016613606363534927,-0.005239991005510092,-0.03391147404909134,0.0672977864742279,-0.011866250075399876,0.022902311757206917,-0.0006298055523075163,-0.025695178657770157,-0.006183761637657881,0.0018253986490890384,0.03021620213985443,-0.04918106645345688,0.02646392583847046,0.001545001519843936,0.03912416100502014,-0.0014360271161422131,-0.016097456216812134,-0.04562824219465256,0.02567988820374012,-0.03335215151309967,0.016547275707125664,-0.028072291985154152,-0.026451263576745987,0.003649936057627201,0.041535936295986176,-0.01700364425778389,-0.05403626337647438,-0.007749590557068586,-0.052018243819475174,0.00455446494743228,0.017315803095698357,-0.0017531972844153645,0.02009999379515648,0.03580885007977486,0.0006782436394132674,-0.008257115259766579,-0.04291710630059242,0.008050923235714436,0.05998260900378227,-0.0356389656662941,-0.04565625637769699,0.031242610886693,-0.008270803838968277,0.011750987730920317,0.02747081220149994,-0.006715917494148016,0.022117506712675095,0.00471158605068922,0.057432983070611954,0.02279104664921761,0.018556898459792137,-0.06457803398370743,0.03776590898633003,-0.04457753896713257,-0.01387899462133646,0.02013351581990719,-0.006140981800854206,-0.015928074717521667,-0.007470014039427042,-0.029576757922768593,0.032111600041389465,0.039856817573308945,0.0048090494237840176,-0.016545847058296204,0.0014002126408740878,-0.016583608463406563,0.027240075170993805,-0.0005170221556909382,0.025062229484319687,0.007804907858371735,0.03163363039493561,-0.01801763102412224,0.06475523859262466,-0.09472357481718063,-0.02637656219303608,0.07913284748792648,0.0016746008768677711,-0.02450830116868019,-0.058214958757162094,-0.04165773466229439,-0.03513309359550476,0.03464778512716293,0.0003738695231731981,0.02248997986316681,0.017366599291563034,0.045089125633239746,0.04312356188893318,-0.021599123254418373,-0.05800948292016983,-0.03804511204361915,-0.03341640532016754,0.012617959640920162,0.005476292688399553,0.023607000708580017,0.09166043251752852,0.045160699635744095,-0.006774914916604757,0.024082185700535778,-0.0009512025862932204,0.02672547660768032,-0.023601368069648743,0.011977110989391804,0.011999554000794888,0.04033053666353226,-0.028841841965913773,0.025519777089357376,-0.04989268630743027,0.003182006534188986,-0.051979295909404755,0.013372777961194515,0.011129219084978104,0.022007452324032784,-0.03248388320207596,2.2780967867674917e-7,0.04911353066563606,0.001817197655327618,-0.04603026434779167,0.04461534321308136,0.039225850254297256,-0.0012145651271566749,0.012773577123880386,0.005553277209401131,-0.0345175601541996,0.010712025687098505,0.026349520310759544,-0.06310829520225525,-0.017192021012306213,0.013291798532009125,-0.03354121744632721,-0.008240072056651115,-0.054865512996912,-0.01664501242339611,0.02179642952978611,-0.051758453249931335,0.016016138717532158,0.02331101894378662,0.010784159414470196,-0.000773429055698216,-0.00507966848090291,-0.015658460557460785,-0.019843386486172676,0.0065702698193490505,0.01627242937684059,0.032211560755968094,0.016490332782268524,0.01774699240922928,0.025805043056607246,0.06322609633207321,-0.04117734730243683,0.020115818828344345,0.04711821302771568,0.0707571879029274,-0.04904532805085182,0.07813892513513565,0.0031890454702079296,-0.004704857245087624,0.026120902970433235,-0.03160902485251427,0.025039037689566612,0.01103560347110033,-0.00012663811503443867,-0.057674720883369446,-0.017726751044392586,0.03174556419253349,-0.0023709682282060385,-0.03609519824385643,-0.03730930760502815,-0.0006843652809038758,0.0023975803051143885,0.044766638427972794,-0.025093669071793556,-0.0346340611577034,0.023021088913083076,0.027509840205311775,-0.0025589156430214643,-0.02984767034649849,0.03159533441066742,0.000321689760312438,0.1025768592953682,-0.07700145244598389,-0.0047820876352488995,1.7652040507863275e-34,0.03937849774956703,0.0727999210357666,-0.04722773656249046,-0.02304566651582718,-0.003864222439005971,-0.06564616411924362,-0.015392042696475984,0.017299309372901917,0.032463978976011276,-0.05189667269587517,-0.034156475216150284]}'},\n", + " {'$': '{\"user\":\"joe\",\"age\":27,\"job\":\"dentist\",\"region\":\"us-east\",\"job_description\":\"Provides oral healthcare including diagnosing and treating teeth and gum issues.\",\"job_embedding\":[0.06978058815002441,0.0032775099389255047,-0.01793205365538597,-0.017792467027902603,-0.026435529813170433,0.055108483880758286,-0.03491858392953873,-0.007366097532212734,-0.03371300920844078,-0.025472283363342285,0.06854928284883499,-0.025957906618714333,0.02603302337229252,-0.004844693932682276,-0.030316924676299095,-0.02325795218348503,0.02549103833734989,0.04054653272032738,0.00003240325168007985,0.032348357141017914,-0.046981509774923325,0.043021854013204575,-0.04997080937027931,0.08438212424516678,-0.02867760695517063,-0.02014574594795704,0.029280567541718483,-0.030570631846785545,0.0018484080210328104,-0.037661224603652954,0.06161481887102127,-0.03312867879867554,-0.0013442665804177525,-0.0539594329893589,1.5035723208711715e-6,0.013137518428266048,-0.002371381502598524,0.0034002342727035284,-0.02176443114876747,-0.06395739316940308,-0.0033671315759420395,-0.06431390345096588,0.033030569553375244,-0.0009462439338676633,-0.0011723014758899808,-0.03666854277253151,0.017894010990858078,-0.052884772419929504,-0.018388479948043823,0.02099521830677986,-0.002903064247220754,-0.07516885548830032,-0.07091116905212402,-0.008929109200835228,-0.016914580017328262,-0.014465300366282465,0.07961380481719971,-0.010423838160932064,-0.029587365686893463,-0.03265484422445297,0.0069853560999035835,0.0019629190210253,-0.0748521238565445,0.018896091729402546,0.09289661794900894,0.016011064872145653,0.017277926206588745,-0.02342916652560234,0.0627848282456398,-0.024567490443587303,0.01536605041474104,-0.012022115290164948,-0.0015159101458266375,0.0060980855487287045,-0.02284293808043003,0.0269687045365572,0.0076052118092775345,-0.01658550091087818,-0.02779854461550713,-0.03866305202245712,0.016034523025155067,0.0718107596039772,0.03609597310423851,0.027858171612024307,-0.035064443945884705,0.03884049877524376,-0.03168535232543945,0.0032565484289079905,0.04803568124771118,-0.013826037757098677,0.043612945824861526,-0.024819688871502876,-0.00699649890884757,0.05919669568538666,-0.008172732777893543,0.003334763692691922,-0.031422436237335205,-0.02374424040317535,0.06507197767496109,-0.011645655147731304,-0.030785081908106804,-0.03889394551515579,-0.007117799483239651,0.014288037084043026,0.011273597367107868,-0.046248018741607666,0.05235636234283447,0.023589013144373897,-0.010259144008159636,-0.015880657359957695,-0.015145579352974892,-0.009795788675546646,-0.04535899683833122,-0.03758159652352333,0.038423046469688416,0.04484947398304939,0.013707016594707966,0.021879112347960472,-0.015571010299026966,0.0302145816385746,0.014277022331953049,0.000565006339456886,-0.053799934685230255,0.02836720459163189,-0.052002668380737305,-0.0324641577899456,0.020876197144389153,0.0334271639585495,0.011445102281868458,0.06299900263547897,-0.0002055288932751864,0.05555151775479317,0.05637581646442413,-0.07975611835718155,-0.015277215279638767,0.0436604879796505,0.04385911300778389,-0.02734258398413658,0.017094861716032028,-0.05207023769617081,-0.019303545355796817,-0.04230298846960068,0.0005861679674126208,0.015268271788954737,0.014028914272785189,0.009575387462973596,0.00855241995304823,-0.005594789050519466,0.029041459783911705,0.013241914100944996,-0.02908351644873619,0.036680687218904495,-0.03838048502802849,0.007148832082748413,0.023592496290802956,-0.02653029188513756,-0.00019958175835199657,-0.06830207258462906,-0.02107084169983864,0.036893848329782486,-0.02038976363837719,-0.023324759677052494,-0.003619050607085228,-0.053395651280879974,-0.024918975308537483,-0.03797946125268936,0.053680170327425,0.05442969128489494,0.01825265772640705,0.04180240258574486,-0.004864271264523268,-0.018222112208604813,-0.006727227475494146,0.0022195701021701097,0.01821045018732548,0.055898517370224,0.0662330612540245,-0.07938185334205627,0.04296897351741791,0.024161064997315407,-0.00013335217954590917,-0.048086442053318024,0.027719896286726,-0.04847507178783417,0.030322281643748283,0.0388566181063652,-0.09316853433847427,0.007912129163742065,0.035782698541879654,-0.023349298164248463,0.02661279588937759,0.023826567456126213,0.0035488198045641184,0.02088967338204384,-0.006507543846964836,-0.02063186652958393,-0.07703351229429245,0.04738263413310051,0.024138079956173897,0.01782582513988018,0.012046621181070805,-0.061330098658800125,0.0007167381700128317,0.023895660415291783,0.015794899314641953,-0.05563268065452576,-0.03704546391963959,-0.04509355872869491,-0.06645524501800537,-0.00680200383067131,-0.010373716242611408,-0.03135502338409424,-0.043525684624910355,0.012056541629135609,-0.01974966563284397,-0.02971959672868252,0.05243377760052681,-0.014369103126227856,0.02065061777830124,0.0315336212515831,0.001059657195582986,0.01696888916194439,0.008106276392936707,0.016704123467206955,0.029559047892689705,-0.02130501344799995,0.01672900840640068,0.0353844091296196,-0.04648108407855034,0.04165293276309967,0.04007629677653313,0.023707237094640732,-0.012647812254726888,0.029832663014531136,0.0393332839012146,-0.011869984678924084,-0.019607488065958023,0.010540486313402653,-0.06029664725065231,-0.008135981857776642,0.04162510484457016,0.01634404808282852,0.06662257015705109,0.02114835754036903,-0.027084724977612495,-0.021750997751951218,-0.012995392084121704,0.009169548749923706,0.0026204760652035475,0.03142300993204117,0.02721653133630753,-0.03679727762937546,0.05470502749085426,0.020433535799384117,-0.05206212028861046,-0.053100746124982834,-0.027409495785832405,0.011788392439484596,-0.07397875934839249,-0.006479810923337936,0.01789257675409317,-0.012474867515265942,-0.004662312567234039,-0.007001837249845266,-0.0031285174190998077,0.002329838927835226,-0.034421417862176895,0.0026087029837071896,-0.057985760271549225,0.02896103821694851,0.0004301408189348877,0.02774876542389393,0.025391295552253723,-0.015864331275224686,0.009241809137165546,-0.012623063288629057,-0.04092230275273323,-0.013024582527577875,-0.00526798702776432,0.06698545813560486,0.050647519528865814,-0.014934596605598928,-0.03590558096766472,0.011002399027347565,0.006855981424450874,-0.007621100172400475,-0.031082680448889732,-0.07082506269216537,0.01860516518354416,-0.051842547953128815,0.012088493444025517,-0.00445124926045537,0.0040427036583423615,-0.04850349575281143,-0.03657433763146401,0.05201044678688049,-0.02727638930082321,0.03336863964796066,-0.014509881846606731,-0.004633271135389805,0.04496696963906288,0.024569081142544743,0.026524528861045837,0.018465803936123848,-0.027647338807582855,0.0918636992573738,0.0362401008605957,-0.020621944218873978,0.01978044956922531,0.00819350779056549,0.03945881128311157,-0.0039494894444942474,-0.00008673051343066618,0.007237206213176251,0.0014201265294104817,-0.0016513016307726502,-0.07758412510156631,0.005595002789050341,-0.010801844298839567,0.03267984092235565,0.013268915005028248,-0.014348365366458893,-0.011538832448422909,0.020516594871878624,0.052368711680173874,-0.03139011561870575,-0.062418386340141296,-0.022649576887488365,-0.006875102873891592,-0.01487431675195694,0.021782439202070236,0.0658823549747467,-0.06219155713915825,0.053350672125816345,0.022561604157090187,0.003827490145340562,-0.04752550274133682,0.016296695917844772,0.02340787835419178,0.013533891178667544,-0.024009408429265022,-0.018335379660129547,-0.05543561652302742,-0.0224392581731081,-0.006483136676251888,0.03340933099389076,0.027050305157899857,0.007145827636122704,-0.08440634608268738,0.02831159345805645,-0.052222274243831635,0.040609974414110184,0.031142935156822205,0.02818308956921101,-0.06132791191339493,0.002541144378483295,0.0362992100417614,0.04908709600567818,-0.08112861961126328,0.040832627564668655,-0.0002514196967240423,-0.028794292360544205,-0.06167372688651085,0.04483211040496826,0.03319424018263817,0.004883799701929092,-0.050135474652051926,-0.06003911048173904,-0.007766907569020987,-0.03811369091272354,-0.08818540722131729,-0.009809792041778564,-0.02435007505118847,0.012840996496379375,-0.02084300667047501,0.014348967000842094,0.023794956505298615,0.04827476292848587,-0.12927421927452087,0.0008381541119888425,0.027856186032295227,-0.05019653961062431,0.006808887235820293,0.02866162359714508,0.02882019802927971,0.021780431270599365,0.04605565220117569,-0.05225829780101776,0.04048369079828262,0.017789483070373535,0.01819595508277416,-0.00860660057514906,-0.04674392193555832,-0.03162055462598801,-0.038825295865535736,-0.02295546792447567,-0.024332279339432716,-0.0004520292859524488,0.020703431218862537,0.0008113141520880163,0.043211761862039566,0.03749376162886619,-0.062350500375032425,0.05058012902736664,0.04430023580789566,-0.08097722381353378,0.03771800920367241,0.03900083899497986,0.01984253525733948,-0.0514189638197422,-0.033448219299316406,0.023460714146494865,-0.02148556150496006,0.024611586704850197,0.1062210574746132,-0.05435395613312721,-0.01568400114774704,-0.02020838111639023,-0.011897774413228037,-0.09336191415786745,0.015986310318112373,-0.04895765334367752,0.058774761855602264,0.05015558376908302,0.04324779659509659,-0.04103345796465874,0.024527961388230324,0.014944610185921192,0.08984224498271942,-0.03649899736046791,0.04017992690205574,-0.005238563288003206,0.02995419129729271,-0.011689539067447186,-0.028105806559324265,0.006240951828658581,0.04885794594883919,-0.003833472030237317,0.021462414413690567,-0.05165105685591698,-0.024353107437491417,0.0471370667219162,-0.02622106671333313,0.018273834139108654,0.03355269134044647,0.01892486959695816,-0.01599150337278843,-0.03247210010886192,-0.0006570457480847836,0.019093582406640053,0.06681074947118759,-0.031195562332868576,-0.09037510305643082,0.027489908039569855,0.014781095087528229,-0.028891943395137787,-0.06284088641405106,-0.00968022271990776,0.02854789048433304,-0.003976686857640743,-0.025577548891305923,0.02108534425497055,0.002162024611607194,-0.011620740406215193,0.026744140312075615,-0.05782923474907875,0.05187131464481354,-0.04883440583944321,-0.03633280098438263,-0.02787165157496929,0.0368678979575634,0.013298398815095425,0.004451545886695385,0.007000659126788378,-0.008358645252883434,-0.05631339922547341,-0.030338238924741745,0.06932955235242844,-0.018871508538722992,-0.00938335247337818,0.02867851033806801,-0.0096843671053648,0.008342440240085125,-0.020360896363854408,-0.03325284644961357,0.013690708205103874,-0.011870089918375015,0.0017806807300075889,-0.0057275258004665375,-0.02282174676656723,0.00406113313511014,-0.04655028507113457,0.0035712518729269505,0.024702565744519234,-0.007429690565913916,0.017270542681217194,-0.05612722784280777,0.023551687598228455,0.006819717586040497,-0.029376192018389705,0.008728999644517899,-0.0041395737789571285,-0.000881498446688056,-0.09243818372488022,-0.02255627140402794,0.008310550823807716,0.041382014751434326,0.0037350922357290983,-0.031287241727113724,0.002307212445884943,-0.005713948979973793,-0.017966793850064278,0.004017147701233625,0.010410948656499386,0.007629900239408016,-0.06206938624382019,-0.02825513295829296,-0.09011373668909071,-0.02764735370874405,0.04184872657060623,-0.0033936104737222195,-0.017090309411287308,0.009591979905962944,0.013027817942202091,0.002261742949485779,0.043357547372579575,0.0015695905312895777,-0.0015145711367949843,-0.03744783625006676,0.08231498301029205,0.018415192142128944,0.04560703784227371,-0.014650614932179453,-0.005610054824501276,-0.031535666435956955,0.008005180396139622,0.07482064515352249,0.12008444964885712,-0.040768563747406006,0.02690100483596325,-0.027875542640686035,-0.029643002897500992,-0.02076958306133747,-0.0548362098634243,0.05840284377336502,0.0150145897641778,-0.03983061388134957,-0.020739911124110225,0.004617768805474043,-0.0753607302904129,-0.028882568702101707,0.01740245334804058,0.029614431783556935,0.08873812854290009,0.021463939920067787,-4.8767623929968845e-33,-0.00920068472623825,-0.035016462206840515,0.03878338262438774,0.020006274804472923,0.01973527856171131,0.032020531594753265,0.06964517384767532,-0.04047306627035141,-0.0394928976893425,-0.006297404412180185,0.02423872798681259,-0.0116649828851223,0.013905096799135208,0.04959045723080635,-0.01411069929599762,0.04448512941598892,0.002487449673935771,-0.014005579985678196,0.007505796849727631,-0.0244253259152174,-0.013607105240225792,-0.01941067911684513,0.08988024294376373,0.012494763359427452,0.02081616222858429,0.004946175962686539,-0.019007503986358643,-0.005237066186964512,-0.005262476857751608,0.04274533689022064,-0.06084485352039337,0.011341079138219357,-0.013165243901312351,0.031634267419576645,-0.003365553682669997,-0.0034032207913696766,-0.049920883029699326,0.02757001854479313,-0.04221336171030998,0.03758709132671356,-0.014555413275957108,-0.020234333351254463,-0.011155915446579456,0.044938601553440094,-0.00031745736487209797,-0.03915875777602196,-0.007016999647021294,-0.06306912004947662,0.02008366398513317,0.058697089552879333,-0.02328968606889248,0.02188180387020111,0.04688335210084915,-0.008024290204048157,0.003115201136097312,0.002137962030246854,0.021027332171797752,0.041809141635894775,-0.00819473061710596,-0.05082077905535698,0.01134545262902975,-0.024848297238349915,0.014682409353554249,0.019181694835424423,-0.010700605809688568,0.025882884860038757,0.01456042192876339,0.04684709012508392,0.04129987955093384,0.0518621988594532,-0.06455059349536896,0.05396562814712525,-0.0667375773191452,-0.020536484196782112,-0.001382493181154132,-0.0058394805528223515,-0.030711814761161804,0.01252173911780119,-0.03430437669157982,0.039655763655900955,0.06166289374232292,0.005118637811392546,-0.0013204558053985238,0.004815662745386362,-0.006541174836456776,0.009665205143392086,0.004565021954476833,0.02854014933109283,0.01318041980266571,0.022019919008016583,-0.0248380359262228,0.06710644066333771,-0.08909425884485245,-0.02992003783583641,0.06897690892219543,0.010172206908464432,-0.021596983075141907,-0.060195837169885635,-0.04861391708254814,-0.0416574664413929,0.019091179594397545,0.016767006367444992,0.0035515001509338617,0.04601703956723213,0.03376253694295883,0.034416187554597855,-0.03400035575032234,-0.046449434012174606,-0.053845569491386414,-0.0326739139854908,-0.015230623073875904,0.03744005784392357,0.02288178913295269,0.1128084361553192,0.04121195524930954,-0.015490145422518252,0.001931392587721348,0.008428264409303665,0.02240961603820324,-0.033004600554704666,0.01809048280119896,-0.011668103747069836,0.026786746457219124,-0.009917913004755974,0.020747441798448563,-0.0202396996319294,-0.006033965386450291,-0.07556130737066269,-0.011229133233428,0.04489750415086746,0.0232241228222847,-0.030354607850313187,2.289941960498254e-7,0.013871141709387302,0.016078997403383255,-0.03312775120139122,0.07074674963951111,0.03682475537061691,-0.000687651801854372,0.008035209029912949,-0.008710208348929882,-0.05506788566708565,-0.019359508529305455,0.014564326964318752,-0.0918194130063057,-0.007503886241465807,0.006072203628718853,-0.05126678943634033,0.03257528319954872,-0.04258055239915848,-0.030300745740532875,0.0043328688479959965,-0.059207648038864136,0.027155930176377296,0.026219286024570465,0.007080291397869587,-0.021365201100707058,0.0031676313374191523,-0.02256934531033039,-0.017509184777736664,0.018352001905441284,0.003614463144913316,0.02136574126780033,0.00705422880128026,-0.0230204313993454,0.02628489211201668,0.07847212255001068,-0.022464843466877937,0.028954317793250084,0.03169868141412735,0.059197694063186646,-0.02287701703608036,0.05039152503013611,0.01833495870232582,0.013621173799037932,0.03225411847233772,-0.04469358175992966,0.024514218792319294,0.032903824001550674,0.0013346493942663074,-0.04830582067370415,-0.013378791511058807,0.04707891121506691,-0.007123452611267567,-0.004303290508687496,-0.028324315324425697,-0.00847945548593998,-0.015558765269815922,0.0062516070902347565,-0.020816590636968613,-0.03009452112019062,0.02972617745399475,0.04230564832687378,-0.020893067121505737,-0.029188252985477448,0.023879399523139,0.024794109165668488,0.0638563483953476,-0.05173766613006592,-0.005064454395323992,1.7531633840906831e-34,0.030617838725447655,0.049961406737565994,-0.027718594297766685,-0.018702015280723572,0.012593978084623814,-0.05086865648627281,-0.014507502317428589,0.01836198940873146,0.03203745186328888,-0.06770750135183334,-0.0323941633105278]}'}]" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Additional query support\n", + "\n", + "### Conditional operators" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@age:[(17 +inf] @region:{us\\-west}\" RETURN 4 user region job age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'john',\n", + " 'region': 'us-west',\n", + " 'job': 'software engineer',\n", + " 'age': '34'},\n", + " {'user': 'stacy', 'region': 'us-west', 'job': 'project manager', 'age': '61'}]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT user, region, job, age\n", + " FROM user_simple\n", + " WHERE age > 17 and region = 'us-west'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"((@region:{us\\-west})|(@region:{us\\-central}))\" RETURN 4 user region job age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'john',\n", + " 'region': 'us-west',\n", + " 'job': 'software engineer',\n", + " 'age': '34'},\n", + " {'user': 'bill', 'region': 'us-central', 'job': 'engineer', 'age': '54'},\n", + " {'user': 'stacy', 'region': 'us-west', 'job': 'project manager', 'age': '61'},\n", + " {'user': 'mary', 'region': 'us-central', 'job': 'doctor', 'age': '24'}]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT user, region, job, age\n", + " FROM user_simple\n", + " WHERE region = 'us-west' or region = 'us-central'\n", + " \"\"\"\n", + "\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job:{software engineer|engineer|pancake tester}\" RETURN 4 user region job age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'john',\n", + " 'region': 'us-west',\n", + " 'job': 'software engineer',\n", + " 'age': '34'},\n", + " {'user': 'bill', 'region': 'us-central', 'job': 'engineer', 'age': '54'}]" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# job is a tag field therefore this syntax works\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, age\n", + " FROM user_simple\n", + " WHERE job IN ('software engineer', 'engineer', 'pancake tester')\n", + " \"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Text based searches\n", + "\n", + "See [the docs](https://redis.io/docs/latest/develop/ai/search-and-query/query/full-text/) for available text queries in Redis.\n", + "\n", + "For more on exact matching see [here](https://redis.io/docs/latest/develop/ai/search-and-query/query/exact-match/)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job_description:sci*\" RETURN 5 user region job job_description age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'bill',\n", + " 'region': 'us-central',\n", + " 'job': 'engineer',\n", + " 'job_description': 'Applies scientific and mathematical principles to solve technical problems.',\n", + " 'age': '54'}]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Prefix\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, job_description, age\n", + " FROM user_simple\n", + " WHERE job_description = 'sci*'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job_description:*care\" RETURN 5 user region job job_description age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'mary',\n", + " 'region': 'us-central',\n", + " 'job': 'doctor',\n", + " 'job_description': 'Diagnoses and treats illnesses, injuries, and other medical conditions in the healthcare field.',\n", + " 'age': '24'},\n", + " {'user': 'joe',\n", + " 'region': 'us-east',\n", + " 'job': 'dentist',\n", + " 'job_description': 'Provides oral healthcare including diagnosing and treating teeth and gum issues.',\n", + " 'age': '27'}]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Suffix\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, job_description, age\n", + " FROM user_simple\n", + " WHERE job_description = '*care'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job_description:%diagnose%\" RETURN 5 user region job job_description age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'mary',\n", + " 'region': 'us-central',\n", + " 'job': 'doctor',\n", + " 'job_description': 'Diagnoses and treats illnesses, injuries, and other medical conditions in the healthcare field.',\n", + " 'age': '24'}]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Fuzzy\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, job_description, age\n", + " FROM user_simple\n", + " WHERE job_description = '%diagnose%'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job_description:\"healthcare including\"\" RETURN 5 user region job job_description age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'joe',\n", + " 'region': 'us-east',\n", + " 'job': 'dentist',\n", + " 'job_description': 'Provides oral healthcare including diagnosing and treating teeth and gum issues.',\n", + " 'age': '27'}]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Phrase no stop words\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, job_description, age\n", + " FROM user_simple\n", + " WHERE job_description = 'healthcare including'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@job_description:\"diagnosing treating\"\" RETURN 5 user region job job_description age\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/robert.shelton/Documents/redis-vl-python/.venv/lib/python3.11/site-packages/sql_redis/translator.py:136: UserWarning: Stopwords ['and'] were removed from phrase search 'diagnosing and treating'. By default, Redis does not index stopwords. To include stopwords in your index, create it with STOPWORDS 0.\n", + " return self._query_builder.build_text_condition(\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'joe',\n", + " 'region': 'us-east',\n", + " 'job': 'dentist',\n", + " 'job_description': 'Provides oral healthcare including diagnosing and treating teeth and gum issues.',\n", + " 'age': '27'}]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Phrase with stop words currently limitation of core Redis\n", + "sql_str = \"\"\"\n", + " SELECT user, region, job, job_description, age\n", + " FROM user_simple\n", + " WHERE job_description = 'diagnosing and treating'\n", + "\"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"@age:[40 60]\" RETURN 4 user region job age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'user': 'bill', 'region': 'us-central', 'job': 'engineer', 'age': '54'}]" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT user, region, job, age\n", + " FROM user_simple\n", + " WHERE age BETWEEN 40 and 60\n", + " \"\"\"\n", + "\n", + "# could maybe be nice to set a connection string at the class level\n", + "# this would deviate from our other query like classes though so thinking on it\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Aggregations\n", + "\n", + "See docs for redis supported reducer functions: [https://redis.io/docs/latest/develop/ai/search-and-query/advanced-concepts/aggregations/#supported-groupby-reducers](docs)." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.AGGREGATE user_simple \"*\" LOAD 2 age region GROUPBY 1 @region REDUCE COUNT 0 AS count_age REDUCE COUNT_DISTINCT 1 @age AS count_distinct_age REDUCE MIN 1 @age AS min_age REDUCE MAX 1 @age AS max_age REDUCE AVG 1 @age AS avg_age REDUCE STDDEV 1 @age AS std_age REDUCE FIRST_VALUE 1 @age AS fist_value_age REDUCE TOLIST 1 @age AS to_list_age REDUCE QUANTILE 2 @age 0.99 AS quantile_age\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'region': 'us-west',\n", + " 'count_age': '2',\n", + " 'count_distinct_age': '2',\n", + " 'min_age': '34',\n", + " 'max_age': '61',\n", + " 'avg_age': '47.5',\n", + " 'std_age': '19.091883092',\n", + " 'fist_value_age': '34',\n", + " 'to_list_age': [b'34', b'61'],\n", + " 'quantile_age': '61'},\n", + " {'region': 'us-central',\n", + " 'count_age': '2',\n", + " 'count_distinct_age': '2',\n", + " 'min_age': '24',\n", + " 'max_age': '54',\n", + " 'avg_age': '39',\n", + " 'std_age': '21.2132034356',\n", + " 'fist_value_age': '54',\n", + " 'to_list_age': [b'24', b'54'],\n", + " 'quantile_age': '54'},\n", + " {'region': 'us-east',\n", + " 'count_age': '1',\n", + " 'count_distinct_age': '1',\n", + " 'min_age': '27',\n", + " 'max_age': '27',\n", + " 'avg_age': '27',\n", + " 'std_age': '0',\n", + " 'fist_value_age': '27',\n", + " 'to_list_age': [b'27'],\n", + " 'quantile_age': '27'}]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT\n", + " user,\n", + " COUNT(age) as count_age,\n", + " COUNT_DISTINCT(age) as count_distinct_age,\n", + " MIN(age) as min_age,\n", + " MAX(age) as max_age,\n", + " AVG(age) as avg_age,\n", + " STDEV(age) as std_age,\n", + " FIRST_VALUE(age) as fist_value_age,\n", + " ARRAY_AGG(age) as to_list_age,\n", + " QUANTILE(age, 0.99) as quantile_age\n", + " FROM user_simple\n", + " GROUP BY region\n", + " \"\"\"\n", + "\n", + "sql_query = SQLQuery(sql_str)\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Vector search" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"*=>[KNN 10 @job_embedding $vector AS vector_distance]\" PARAMS 2 vector $vector DIALECT 2 RETURN 4 user job job_description vector_distance SORTBY vector_distance ASC\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'vector_distance': '0.823510587215',\n", + " 'user': 'bill',\n", + " 'job': 'engineer',\n", + " 'job_description': 'Applies scientific and mathematical principles to solve technical problems.'},\n", + " {'vector_distance': '0.965160429478',\n", + " 'user': 'john',\n", + " 'job': 'software engineer',\n", + " 'job_description': 'Designs, develops, and maintains software applications and systems.'},\n", + " {'vector_distance': '1.00401365757',\n", + " 'user': 'mary',\n", + " 'job': 'doctor',\n", + " 'job_description': 'Diagnoses and treats illnesses, injuries, and other medical conditions in the healthcare field.'},\n", + " {'vector_distance': '1.0062687397',\n", + " 'user': 'stacy',\n", + " 'job': 'project manager',\n", + " 'job_description': 'Plans, organizes, and oversees projects from inception to completion.'},\n", + " {'vector_distance': '1.01110625267',\n", + " 'user': 'joe',\n", + " 'job': 'dentist',\n", + " 'job_description': 'Provides oral healthcare including diagnosing and treating teeth and gum issues.'}]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT user, job, job_description, cosine_distance(job_embedding, :vec) AS vector_distance\n", + " FROM user_simple\n", + " ORDER BY vector_distance ASC\n", + " \"\"\"\n", + "\n", + "vec = hf.embed(\"looking for someone to use base principles to solve problems\", as_buffer=True)\n", + "sql_query = SQLQuery(sql_str, params={\"vec\": vec})\n", + "\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Resulting redis query: FT.SEARCH user_simple \"(@region:{us\\-central})=>[KNN 10 @job_embedding $vector AS vector_distance]\" PARAMS 2 vector $vector DIALECT 2 RETURN 3 user region vector_distance SORTBY vector_distance ASC\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'vector_distance': '0.823510587215', 'user': 'bill', 'region': 'us-central'},\n", + " {'vector_distance': '1.00401365757', 'user': 'mary', 'region': 'us-central'}]" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sql_str = \"\"\"\n", + " SELECT user, region, cosine_distance(job_embedding, :vec) AS vector_distance\n", + " FROM user_simple\n", + " WHERE region = 'us-central'\n", + " ORDER BY vector_distance ASC\n", + " \"\"\"\n", + "\n", + "vec = hf.embed(\"looking for someone to use base principles to solve problems\", as_buffer=True)\n", + "sql_query = SQLQuery(sql_str, params={\"vec\": vec})\n", + "\n", + "redis_query = sql_query.redis_query_string(redis_url=\"redis://localhost:6379\")\n", + "print(\"Resulting redis query: \", redis_query)\n", + "results = index.query(sql_query)\n", + "\n", + "results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below we will clean up after our work. First, you can flush all data from Redis associated with the index by\n", + "using the `.clear()` method. This will leave the secondary index in place for future insertions or updates.\n", + "\n", + "But if you want to clean up everything, including the index, just use `.delete()`\n", + "which will by default remove the index AND the underlying data." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "5" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Clear all data from Redis associated with the index\n", + "index.clear()" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# But the index is still in place\n", + "index.exists()" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "# Remove / delete the index in its entirety\n", + "index.delete()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "redisvl", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 602983b5..f89fe51e 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -23,4 +23,5 @@ User guides provide helpful resources for using RedisVL and its different compon 09_svs_vamana 10_embeddings_cache 11_advanced_queries +12_sql_to_redis_queries ``` \ No newline at end of file diff --git a/examples/multi_prefix_example.py b/examples/multi_prefix_example.py new file mode 100644 index 00000000..aff74580 --- /dev/null +++ b/examples/multi_prefix_example.py @@ -0,0 +1,140 @@ +""" +Example: Multi-prefix index workaround + +This script demonstrates how to: +1. Manually create a multi-prefix index using execute_command +2. Connect to it using SearchIndex.from_existing() +3. Load data with different prefixes using the keys parameter +4. Query and verify results come from both prefixes +""" + +import redis +from redisvl.index import SearchIndex +from redisvl.query import VectorQuery +from redisvl.redis.utils import array_to_buffer + +# Connect to Redis +client = redis.Redis(host="localhost", port=6379, decode_responses=True) + +INDEX_NAME = "user_simple" + +# Clean up any existing index +try: + client.ft(INDEX_NAME).dropindex(delete_documents=True) +except Exception: + pass + +# 1. Manually create the multi-prefix index +print("Creating multi-prefix index...") +client.execute_command( + "FT.CREATE", INDEX_NAME, + "ON", "HASH", + "PREFIX", "2", "prefix_1:", "prefix_2:", + "SCHEMA", + "user", "TAG", + "credit_score", "TAG", + "job", "TEXT", + "age", "NUMERIC", + "user_embedding", "VECTOR", "FLAT", "6", + "TYPE", "FLOAT32", + "DIM", "3", + "DISTANCE_METRIC", "COSINE" +) +print("Index created with prefixes: prefix_1:, prefix_2:") + +# 2. Connect using from_existing +index = SearchIndex.from_existing(INDEX_NAME, redis_client=client) +print(f"Connected to index. Prefixes: {index.schema.index.prefix}") + +# 3. Prepare test data +data_prefix_1 = [ + { + "user": "john", + "credit_score": "high", + "job": "engineer", + "age": 30, + "user_embedding": array_to_buffer([0.1, 0.2, 0.3], dtype="float32"), + }, + { + "user": "jane", + "credit_score": "medium", + "job": "doctor", + "age": 35, + "user_embedding": array_to_buffer([0.2, 0.3, 0.4], dtype="float32"), + }, +] + +data_prefix_2 = [ + { + "user": "bob", + "credit_score": "low", + "job": "teacher", + "age": 40, + "user_embedding": array_to_buffer([0.3, 0.4, 0.5], dtype="float32"), + }, + { + "user": "alice", + "credit_score": "high", + "job": "lawyer", + "age": 45, + "user_embedding": array_to_buffer([0.4, 0.5, 0.6], dtype="float32"), + }, +] + +# 4. Load data with explicit keys for each prefix +print("\nLoading data with prefix_1...") +keys_p1 = ["prefix_1:doc1", "prefix_1:doc2"] +index.load(data_prefix_1, keys=keys_p1) +print(f"Loaded keys: {keys_p1}") + +print("\nLoading data with prefix_2...") +keys_p2 = ["prefix_2:doc1", "prefix_2:doc2"] +index.load(data_prefix_2, keys=keys_p2) +print(f"Loaded keys: {keys_p2}") + +# 5. Query and verify we get results from both prefixes +print("\n" + "="*50) +print("Running vector search...") +print("="*50) + +query = VectorQuery( + vector=[0.2, 0.3, 0.4], + vector_field_name="user_embedding", + return_fields=["user", "credit_score", "job", "age"], + num_results=10, +) + +results = index.query(query) + +print(f"\nFound {len(results)} results:\n") + +prefix_1_count = 0 +prefix_2_count = 0 + +for r in results: + key = r.get("id", "") + if key.startswith("prefix_1:"): + prefix_1_count += 1 + elif key.startswith("prefix_2:"): + prefix_2_count += 1 + print(f" Key: {key}") + print(f" User: {r.get('user')}, Job: {r.get('job')}, Age: {r.get('age')}") + print() + +# 6. Verify both prefixes are represented +print("="*50) +print("VERIFICATION") +print("="*50) +print(f"Results from prefix_1: {prefix_1_count}") +print(f"Results from prefix_2: {prefix_2_count}") + +if prefix_1_count > 0 and prefix_2_count > 0: + print("\n✅ SUCCESS: Query returned results from BOTH prefixes!") +else: + print("\n❌ FAILED: Expected results from both prefixes") + +# Cleanup +print("\nCleaning up...") +client.ft(INDEX_NAME).dropindex(delete_documents=True) +print("Done!") + diff --git a/pyproject.toml b/pyproject.toml index 9286b066..65758811 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "redisvl" -version = "0.13.2" +version = "0.14.0" description = "Python client library and CLI for using Redis as a vector database" authors = [{ name = "Redis Inc.", email = "applied.ai@redis.com" }] requires-python = ">=3.9.2,<3.14" @@ -51,6 +51,9 @@ bedrock = [ pillow = [ "pillow>=11.3.0", ] +sql-redis = [ + "sql-redis>=0.1.2", +] [project.urls] Homepage = "https://github.com/redis/redis-vl-python" @@ -64,6 +67,9 @@ rvl = "redisvl.cli.runner:main" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +allow-direct-references = true + [dependency-groups] dev = [ "black>=25.1.0,<26", diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 4bc66f67..46277c65 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -31,6 +31,7 @@ from redisvl.query.hybrid import HybridQuery from redisvl.query.query import VectorQuery +from redisvl.query.sql import SQLQuery from redisvl.redis.utils import ( _keys_share_hash_tag, async_cluster_create_index, @@ -644,8 +645,11 @@ def create(self, overwrite: bool = False, drop: bool = False) -> None: self.delete(drop=drop) try: + # Handle prefix as either a single string or a list of strings + prefix = self.schema.index.prefix + prefix_list = prefix if isinstance(prefix, list) else [prefix] definition = IndexDefinition( - prefix=[self.schema.index.prefix], index_type=self._storage.type + prefix=prefix_list, index_type=self._storage.type ) # Extract stopwords from schema stopwords = self.schema.index.stopwords @@ -917,6 +921,49 @@ def _aggregate(self, aggregation_query: AggregationQuery) -> List[Dict[str, Any] storage_type=self.schema.index.storage_type, ) + def _sql_query(self, sql_query: SQLQuery) -> List[Dict[str, Any]]: + """Execute a SQL query and return results. + + Args: + sql_query: The SQLQuery object containing the SQL statement. + + Returns: + List of dictionaries containing the query results. + + Raises: + ImportError: If sql-redis package is not installed. + """ + try: + from sql_redis.executor import Executor + from sql_redis.schema import SchemaRegistry + except ImportError: + raise ImportError( + "sql-redis is required for SQL query support. " + "Install it with: pip install redisvl[sql-redis]" + ) + + registry = SchemaRegistry(self._redis_client) + registry.load_all() # Loads index schemas from Redis + + executor = Executor(self._redis_client, registry) + + # Execute the query with any params + result = executor.execute(sql_query.sql, params=sql_query.params) + + # Decode bytes to strings in the results (Redis may return bytes) + decoded_rows = [] + for row in result.rows: + decoded_row = {} + for key, value in row.items(): + # Decode key if bytes + str_key = key.decode("utf-8") if isinstance(key, bytes) else key + # Decode value if bytes + str_value = value.decode("utf-8") if isinstance(value, bytes) else value + decoded_row[str_key] = str_value + decoded_rows.append(decoded_row) + + return decoded_rows + def aggregate(self, *args, **kwargs) -> "AggregateResult": """Perform an aggregation operation against the index. @@ -1118,7 +1165,7 @@ def _query(self, query: BaseQuery) -> List[Dict[str, Any]]: return process_results(results, query=query, schema=self.schema) def query( - self, query: Union[BaseQuery, AggregationQuery, HybridQuery] + self, query: Union[BaseQuery, AggregationQuery, HybridQuery, SQLQuery] ) -> List[Dict[str, Any]]: """Execute a query on the index. @@ -1146,6 +1193,8 @@ def query( """ if isinstance(query, AggregationQuery): return self._aggregate(query) + elif isinstance(query, SQLQuery): + return self._sql_query(query) elif isinstance(query, HybridQuery): return self._hybrid_search(query) else: @@ -1528,8 +1577,11 @@ async def create(self, overwrite: bool = False, drop: bool = False) -> None: await self.delete(drop) try: + # Handle prefix as either a single string or a list of strings + prefix = self.schema.index.prefix + prefix_list = prefix if isinstance(prefix, list) else [prefix] definition = IndexDefinition( - prefix=[self.schema.index.prefix], index_type=self._storage.type + prefix=prefix_list, index_type=self._storage.type ) # Extract stopwords from schema stopwords = self.schema.index.stopwords diff --git a/redisvl/query/__init__.py b/redisvl/query/__init__.py index aa84633e..3f78c755 100644 --- a/redisvl/query/__init__.py +++ b/redisvl/query/__init__.py @@ -15,6 +15,7 @@ VectorQuery, VectorRangeQuery, ) +from redisvl.query.sql import SQLQuery __all__ = [ "BaseQuery", @@ -29,4 +30,5 @@ "AggregateHybridQuery", "MultiVectorQuery", "Vector", + "SQLQuery", ] diff --git a/redisvl/query/sql.py b/redisvl/query/sql.py new file mode 100644 index 00000000..06dd2369 --- /dev/null +++ b/redisvl/query/sql.py @@ -0,0 +1,159 @@ +"""SQL Query class for executing SQL-like queries against Redis.""" + +import re +from typing import Any, Dict, Optional + + +class SQLQuery: + """A query class that translates SQL-like syntax into Redis queries. + + This class allows users to write SQL SELECT statements that are + automatically translated into Redis FT.SEARCH or FT.AGGREGATE commands. + + .. code-block:: python + + from redisvl.query import SQLQuery + from redisvl.index import SearchIndex + + index = SearchIndex.from_existing("products", redis_url="redis://localhost:6379") + + sql_query = SQLQuery(''' + SELECT title, price, category + FROM products + WHERE category = 'electronics' AND price < 100 + ''') + + results = index.query(sql_query) + + Note: + Requires the optional `sql-redis` package. Install with: + ``pip install redisvl[sql]`` + """ + + def __init__(self, sql: str, params: Optional[Dict[str, Any]] = None): + """Initialize a SQLQuery. + + Args: + sql: The SQL SELECT statement to execute. + params: Optional dictionary of parameters for parameterized queries. + Useful for passing vector data for similarity searches. + """ + self.sql = sql + self.params = params or {} + + def _substitute_params(self, sql: str, params: Dict[str, Any]) -> str: + """Substitute parameter placeholders in SQL with actual values. + + Uses token-based approach: splits SQL on :param patterns, then rebuilds + with substituted values. This prevents partial matching (e.g., :id + won't match inside :product_id) and is faster than regex at scale. + + Args: + sql: The SQL string with :param placeholders. + params: Dictionary mapping parameter names to values. + + Returns: + SQL string with parameters substituted. + + Note: + - String values are wrapped in single quotes with proper escaping + - Numeric values are converted to strings + - Bytes values (e.g., vectors) are NOT substituted here + """ + if not params: + return sql + + # Split SQL on :param patterns, keeping the delimiters + # Pattern matches : followed by valid identifier (letter/underscore, then alphanumeric/underscore) + tokens = re.split(r"(:[a-zA-Z_][a-zA-Z0-9_]*)", sql) + + result = [] + for token in tokens: + if token.startswith(":"): + key = token[1:] # Remove leading : + if key in params: + value = params[key] + if isinstance(value, (int, float)): + result.append(str(value)) + elif isinstance(value, str): + # Escape single quotes using SQL standard: ' -> '' + escaped = value.replace("'", "''") + result.append(f"'{escaped}'") + else: + # Keep placeholder for bytes (vectors handled by Executor) + result.append(token) + else: + # Keep unmatched placeholders as-is + result.append(token) + else: + result.append(token) + + return "".join(result) + + def redis_query_string( + self, + redis_client: Optional[Any] = None, + redis_url: str = "redis://localhost:6379", + ) -> str: + """Translate the SQL query to a Redis command string. + + This method uses the sql-redis translator to convert the SQL statement + into the equivalent Redis FT.SEARCH or FT.AGGREGATE command. + + Args: + redis_client: A Redis client connection used to load index schemas. + If not provided, a connection will be created using redis_url. + redis_url: The Redis URL to connect to if redis_client is not provided. + Defaults to "redis://localhost:6379". + + Returns: + The Redis command string (e.g., 'FT.SEARCH products "@category:{electronics}"'). + + Raises: + ImportError: If sql-redis package is not installed. + + Example: + .. code-block:: python + + from redisvl.query import SQLQuery + + sql_query = SQLQuery("SELECT * FROM products WHERE category = 'electronics'") + + # Using redis_url + redis_cmd = sql_query.redis_query_string(redis_url="redis://localhost:6379") + + # Or using an existing client + from redis import Redis + client = Redis() + redis_cmd = sql_query.redis_query_string(redis_client=client) + + print(redis_cmd) + # Output: FT.SEARCH products "@category:{electronics}" + """ + try: + from sql_redis.schema import SchemaRegistry + from sql_redis.translator import Translator + except ImportError: + raise ImportError( + "sql-redis is required for SQL query support. " + "Install it with: pip install redisvl[sql]" + ) + + # Get or create Redis client + if redis_client is None: + from redis import Redis + + redis_client = Redis.from_url(redis_url) + + # Load schemas from Redis + registry = SchemaRegistry(redis_client) + registry.load_all() + + # Translate SQL to Redis command + translator = Translator(registry) + + # Substitute non-bytes params in SQL before translation + sql = self._substitute_params(self.sql, self.params) + + translated = translator.translate(sql) + return translated.to_command_string() diff --git a/tests/integration/test_multi_prefix.py b/tests/integration/test_multi_prefix.py new file mode 100644 index 00000000..228cd635 --- /dev/null +++ b/tests/integration/test_multi_prefix.py @@ -0,0 +1,385 @@ +"""Integration tests for multi-prefix index support. + +Tests that queries return results from all configured prefixes when using +multi-prefix indexes. +""" + +import uuid + +import pytest + +from redisvl.index import SearchIndex +from redisvl.query import ( + CountQuery, + FilterQuery, + TextQuery, + VectorQuery, + VectorRangeQuery, +) +from redisvl.query.filter import Num, Tag +from redisvl.redis.utils import array_to_buffer + + +@pytest.fixture +def multi_prefix_index(redis_url, worker_id): + """Create a multi-prefix index with data loaded under different prefixes.""" + unique_id = str(uuid.uuid4())[:8] + index_name = f"multi_prefix_test_{worker_id}_{unique_id}" + + # Connect to get raw client for manual index creation + from redisvl.redis.connection import RedisConnectionFactory + + client = RedisConnectionFactory.get_redis_connection(redis_url=redis_url) + + # Clean up any existing index + try: + client.ft(index_name).dropindex(delete_documents=True) + except Exception: + pass + + # Create index with multiple prefixes using raw command + client.execute_command( + "FT.CREATE", + index_name, + "ON", + "HASH", + "PREFIX", + "2", + f"prefix_a_{unique_id}:", + f"prefix_b_{unique_id}:", + "SCHEMA", + "user", + "TAG", + "credit_score", + "TAG", + "job", + "TEXT", + "age", + "NUMERIC", + "user_embedding", + "VECTOR", + "FLAT", + "6", + "TYPE", + "FLOAT32", + "DIM", + "3", + "DISTANCE_METRIC", + "COSINE", + ) + + # Connect using from_existing + index = SearchIndex.from_existing(index_name, redis_url=redis_url) + + # Verify multi-prefix was preserved + assert isinstance(index.schema.index.prefix, list) + assert len(index.schema.index.prefix) == 2 + + # Prepare test data for prefix_a (2 docs) + data_prefix_a = [ + { + "user": "john", + "credit_score": "high", + "job": "engineer at tech company", + "age": 30, + "user_embedding": array_to_buffer([0.1, 0.2, 0.3], dtype="float32"), + }, + { + "user": "jane", + "credit_score": "medium", + "job": "doctor at hospital", + "age": 35, + "user_embedding": array_to_buffer([0.2, 0.3, 0.4], dtype="float32"), + }, + ] + + # Prepare test data for prefix_b (2 docs) + data_prefix_b = [ + { + "user": "bob", + "credit_score": "low", + "job": "teacher at school", + "age": 40, + "user_embedding": array_to_buffer([0.3, 0.4, 0.5], dtype="float32"), + }, + { + "user": "alice", + "credit_score": "high", + "job": "lawyer at firm", + "age": 45, + "user_embedding": array_to_buffer([0.4, 0.5, 0.6], dtype="float32"), + }, + ] + + # Load data with explicit keys for each prefix + keys_a = [ + f"prefix_a_{unique_id}:doc1", + f"prefix_a_{unique_id}:doc2", + ] + keys_b = [ + f"prefix_b_{unique_id}:doc1", + f"prefix_b_{unique_id}:doc2", + ] + + index.load(data_prefix_a, keys=keys_a) + index.load(data_prefix_b, keys=keys_b) + + yield index, unique_id + + # Cleanup + try: + index.delete(drop=True) + except Exception: + pass + + +def _count_prefixes(results, unique_id): + """Count how many results come from each prefix.""" + prefix_a_count = sum( + 1 for r in results if r.get("id", "").startswith(f"prefix_a_{unique_id}:") + ) + prefix_b_count = sum( + 1 for r in results if r.get("id", "").startswith(f"prefix_b_{unique_id}:") + ) + return prefix_a_count, prefix_b_count + + +class TestMultiPrefixVectorQuery: + """Test VectorQuery with multi-prefix indexes.""" + + def test_vector_query_returns_both_prefixes(self, multi_prefix_index): + """VectorQuery should return results from all prefixes.""" + index, unique_id = multi_prefix_index + + query = VectorQuery( + vector=[0.25, 0.35, 0.45], + vector_field_name="user_embedding", + return_fields=["user", "credit_score", "job", "age"], + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 4, f"Expected 4 results, got {len(results)}" + + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count == 2, f"Expected 2 from prefix_a, got {prefix_a_count}" + assert prefix_b_count == 2, f"Expected 2 from prefix_b, got {prefix_b_count}" + + def test_vector_query_with_filter_both_prefixes(self, multi_prefix_index): + """VectorQuery with filter should return results from all prefixes.""" + index, unique_id = multi_prefix_index + + # Filter for credit_score == "high" (john from prefix_a, alice from prefix_b) + query = VectorQuery( + vector=[0.25, 0.35, 0.45], + vector_field_name="user_embedding", + return_fields=["user", "credit_score"], + filter_expression=Tag("credit_score") == "high", + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 2, f"Expected 2 results, got {len(results)}" + + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count == 1, f"Expected 1 from prefix_a, got {prefix_a_count}" + assert prefix_b_count == 1, f"Expected 1 from prefix_b, got {prefix_b_count}" + + +class TestMultiPrefixVectorRangeQuery: + """Test VectorRangeQuery with multi-prefix indexes.""" + + def test_range_query_returns_both_prefixes(self, multi_prefix_index): + """VectorRangeQuery should return results from all prefixes within range.""" + index, unique_id = multi_prefix_index + + query = VectorRangeQuery( + vector=[0.25, 0.35, 0.45], + vector_field_name="user_embedding", + return_fields=["user", "credit_score"], + distance_threshold=0.5, # Wide threshold to get all docs + num_results=10, + ) + + results = index.query(query) + + # Should get results from both prefixes + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count > 0, "Expected results from prefix_a" + assert prefix_b_count > 0, "Expected results from prefix_b" + + +class TestMultiPrefixFilterQuery: + """Test FilterQuery with multi-prefix indexes.""" + + def test_filter_query_returns_both_prefixes(self, multi_prefix_index): + """FilterQuery should return results from all prefixes.""" + index, unique_id = multi_prefix_index + + query = FilterQuery( + return_fields=["user", "credit_score", "age"], + filter_expression=Num("age") >= 30, + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 4, f"Expected 4 results, got {len(results)}" + + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count == 2, f"Expected 2 from prefix_a, got {prefix_a_count}" + assert prefix_b_count == 2, f"Expected 2 from prefix_b, got {prefix_b_count}" + + def test_filter_query_tag_both_prefixes(self, multi_prefix_index): + """FilterQuery with tag filter should return results from all prefixes.""" + index, unique_id = multi_prefix_index + + # Filter for credit_score == "high" (john from prefix_a, alice from prefix_b) + query = FilterQuery( + return_fields=["user", "credit_score"], + filter_expression=Tag("credit_score") == "high", + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 2, f"Expected 2 results, got {len(results)}" + + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count == 1, f"Expected 1 from prefix_a, got {prefix_a_count}" + assert prefix_b_count == 1, f"Expected 1 from prefix_b, got {prefix_b_count}" + + +class TestMultiPrefixCountQuery: + """Test CountQuery with multi-prefix indexes.""" + + def test_count_query_counts_all_prefixes(self, multi_prefix_index): + """CountQuery should count documents from all prefixes.""" + index, unique_id = multi_prefix_index + + query = CountQuery(filter_expression=Tag("credit_score") == "high") + count = index.query(query) + + assert count == 2, f"Expected count of 2, got {count}" + + def test_count_query_all_docs(self, multi_prefix_index): + """CountQuery with wildcard should count all documents from all prefixes.""" + index, unique_id = multi_prefix_index + + query = CountQuery(filter_expression="*") + count = index.query(query) + + assert count == 4, f"Expected count of 4, got {count}" + + +class TestMultiPrefixTextQuery: + """Test TextQuery with multi-prefix indexes.""" + + def test_text_query_returns_both_prefixes(self, multi_prefix_index): + """TextQuery should return results from all prefixes.""" + index, unique_id = multi_prefix_index + + # Search for terms that appear in jobs from both prefixes + # prefix_a: "engineer", "doctor" | prefix_b: "teacher", "lawyer" + # Use wildcard to match partial terms + query = TextQuery( + text="engineer|doctor|teacher|lawyer", + text_field_name="job", + return_fields=["user", "job"], + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 4, f"Expected 4 results, got {len(results)}" + + prefix_a_count, prefix_b_count = _count_prefixes(results, unique_id) + assert prefix_a_count == 2, f"Expected 2 from prefix_a, got {prefix_a_count}" + assert prefix_b_count == 2, f"Expected 2 from prefix_b, got {prefix_b_count}" + + def test_text_query_specific_term(self, multi_prefix_index): + """TextQuery for specific term should return matching docs from any prefix.""" + index, unique_id = multi_prefix_index + + # Search for "engineer" (only john from prefix_a) + query = TextQuery( + text="engineer", + text_field_name="job", + return_fields=["user", "job"], + num_results=10, + ) + + results = index.query(query) + + assert len(results) == 1, f"Expected 1 result, got {len(results)}" + assert results[0]["user"] == "john" + + +class TestMultiPrefixIndexCreation: + """Test creating multi-prefix indexes via redisvl (not raw commands).""" + + def test_create_index_with_prefix_list(self, redis_url, worker_id): + """Test that SearchIndex.create() works with a list of prefixes.""" + unique_id = str(uuid.uuid4())[:8] + index_name = f"create_multi_prefix_{worker_id}_{unique_id}" + + schema = { + "index": { + "name": index_name, + "prefix": [f"pfx_a_{unique_id}", f"pfx_b_{unique_id}"], + "storage_type": "hash", + }, + "fields": [ + {"name": "user", "type": "tag"}, + {"name": "age", "type": "numeric"}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 3, + "distance_metric": "cosine", + "algorithm": "flat", + "datatype": "float32", + }, + }, + ], + } + + index = SearchIndex.from_dict(schema, redis_url=redis_url) + + try: + # This should work with our fix + index.create(overwrite=True) + + # Verify index was created + assert index.exists() + + # Load data with explicit keys to different prefixes + data = [ + { + "user": "test_user", + "age": 25, + "embedding": array_to_buffer([0.1, 0.2, 0.3], dtype="float32"), + } + ] + + # Load to first prefix + keys_a = [f"pfx_a_{unique_id}:doc1"] + index.load(data, keys=keys_a) + + # Load to second prefix + keys_b = [f"pfx_b_{unique_id}:doc1"] + index.load(data, keys=keys_b) + + # Query should find both + query = CountQuery(filter_expression="*") + count = index.query(query) + assert count == 2, f"Expected 2 docs, got {count}" + + finally: + try: + index.delete(drop=True) + except Exception: + pass diff --git a/tests/integration/test_redis_cluster_support.py b/tests/integration/test_redis_cluster_support.py index 80b82420..0d18dea3 100644 --- a/tests/integration/test_redis_cluster_support.py +++ b/tests/integration/test_redis_cluster_support.py @@ -89,6 +89,7 @@ def test_search_index_cluster_info(redis_cluster_url): finally: index.delete(drop=True) + @pytest.mark.requires_cluster @pytest.mark.asyncio async def test_async_search_index_cluster_info(redis_cluster_url): @@ -110,6 +111,7 @@ async def test_async_search_index_cluster_info(redis_cluster_url): await index.delete(drop=True) await client.aclose() + @pytest.mark.requires_cluster @pytest.mark.asyncio async def test_async_search_index_client(redis_cluster_url): diff --git a/tests/integration/test_search_index.py b/tests/integration/test_search_index.py index ae64a229..ebfedbe7 100644 --- a/tests/integration/test_search_index.py +++ b/tests/integration/test_search_index.py @@ -304,6 +304,7 @@ def test_search_index_delete(index): assert not index.exists() assert index.name not in convert_bytes(index.client.execute_command("FT._LIST")) + @pytest.mark.parametrize("num_docs", [0, 1, 5, 10, 2042]) def test_search_index_clear(index, num_docs): index.create(overwrite=True, drop=True) diff --git a/tests/integration/test_sql_redis_hash.py b/tests/integration/test_sql_redis_hash.py new file mode 100644 index 00000000..33560c35 --- /dev/null +++ b/tests/integration/test_sql_redis_hash.py @@ -0,0 +1,1134 @@ +"""Integration tests for SQLQuery class. + +These tests verify that SQLQuery can translate SQL-like syntax +into proper Redis queries and return expected results. +""" + +import uuid + +import pytest + +from redisvl.index import SearchIndex +from redisvl.query import SQLQuery + + +@pytest.fixture +def sql_index(redis_url, worker_id): + """Create a products index for SQL query testing.""" + unique_id = str(uuid.uuid4())[:8] + index_name = f"sql_products_{worker_id}_{unique_id}" + + index = SearchIndex.from_dict( + { + "index": { + "name": index_name, + "prefix": f"product_{worker_id}_{unique_id}", + "storage_type": "hash", + }, + "fields": [ + {"name": "title", "type": "text", "attrs": {"sortable": True}}, + {"name": "name", "type": "text", "attrs": {"sortable": True}}, + {"name": "price", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "stock", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "rating", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "category", "type": "tag", "attrs": {"sortable": True}}, + {"name": "tags", "type": "tag"}, + ], + }, + redis_url=redis_url, + ) + + index.create(overwrite=True) + + # Load test data + products = [ + { + "title": "Gaming laptop Pro", + "name": "Gaming Laptop", + "price": 899, + "stock": 10, + "rating": 4.5, + "category": "electronics", + "tags": "sale,featured", + }, + { + "title": "Budget laptop Basic", + "name": "Budget Laptop", + "price": 499, + "stock": 25, + "rating": 3.8, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Premium laptop Ultra", + "name": "Premium Laptop", + "price": 1299, + "stock": 5, + "rating": 4.9, + "category": "electronics", + "tags": "featured", + }, + { + "title": "Python Programming", + "name": "Python Book", + "price": 45, + "stock": 100, + "rating": 4.7, + "category": "books", + "tags": "bestseller", + }, + { + "title": "Redis in Action", + "name": "Redis Book", + "price": 55, + "stock": 50, + "rating": 4.6, + "category": "books", + "tags": "featured", + }, + { + "title": "Data Science Guide", + "name": "DS Book", + "price": 65, + "stock": 30, + "rating": 4.4, + "category": "books", + "tags": "sale", + }, + { + "title": "Wireless Mouse", + "name": "Mouse", + "price": 29, + "stock": 200, + "rating": 4.2, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Mechanical Keyboard", + "name": "Keyboard", + "price": 149, + "stock": 75, + "rating": 4.6, + "category": "electronics", + "tags": "featured", + }, + { + "title": "USB Hub", + "name": "Hub", + "price": 25, + "stock": 150, + "rating": 3.9, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Monitor Stand", + "name": "Stand", + "price": 89, + "stock": 40, + "rating": 4.1, + "category": "accessories", + "tags": "sale,featured", + }, + { + "title": "Desk Lamp", + "name": "Lamp", + "price": 35, + "stock": 80, + "rating": 4.0, + "category": "accessories", + "tags": "sale", + }, + { + "title": "Notebook Set", + "name": "Notebooks", + "price": 15, + "stock": 300, + "rating": 4.3, + "category": "stationery", + "tags": "bestseller", + }, + { + "title": "Laptop and Keyboard Bundle", + "name": "Bundle Pack", + "price": 999, + "stock": 15, + "rating": 4.7, + "category": "electronics", + "tags": "featured,sale", + }, + ] + + index.load(products) + + yield index + + # Cleanup + index.delete(drop=True) + + +class TestSQLQueryBasic: + """Tests for basic SQL SELECT queries.""" + + def test_import_sql_query(self): + """Verify SQLQuery can be imported from redisvl.query.""" + from redisvl.query import SQLQuery + + assert SQLQuery is not None + + def test_select_all_fields(self, sql_index): + """Test SELECT * returns all fields.""" + sql_query = SQLQuery(f"SELECT * FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) > 0 + # Verify results contain expected fields + assert "title" in results[0] + assert "price" in results[0] + + def test_select_specific_fields(self, sql_index): + """Test SELECT with specific field list.""" + sql_query = SQLQuery(f"SELECT title, price FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) > 0 + # Results should contain requested fields + assert "title" in results[0] + assert "price" in results[0] + + def test_redis_query_string_with_client(self, sql_index): + """Test redis_query_string() with redis_client returns the Redis command string.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + + # Get the Redis command string using redis_client + redis_cmd = sql_query.redis_query_string(redis_client=sql_index._redis_client) + + # Verify it's a valid FT.SEARCH command + assert redis_cmd.startswith("FT.SEARCH") + assert sql_index.name in redis_cmd + assert "electronics" in redis_cmd + + def test_redis_query_string_with_url(self, sql_index, redis_url): + """Test redis_query_string() with redis_url returns the Redis command string.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + + # Get the Redis command string using redis_url + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify it's a valid FT.SEARCH command + assert redis_cmd.startswith("FT.SEARCH") + assert sql_index.name in redis_cmd + assert "electronics" in redis_cmd + + def test_redis_query_string_aggregate(self, sql_index): + """Test redis_query_string() returns FT.AGGREGATE for aggregation queries.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count + FROM {sql_index.name} + GROUP BY category + """ + ) + + redis_cmd = sql_query.redis_query_string(redis_client=sql_index._redis_client) + + # Verify it's a valid FT.AGGREGATE command + assert redis_cmd.startswith("FT.AGGREGATE") + assert sql_index.name in redis_cmd + assert "GROUPBY" in redis_cmd + + +class TestSQLQueryWhere: + """Tests for SQL WHERE clause filtering.""" + + def test_where_tag_equals(self, sql_index): + """Test WHERE with tag field equality.""" + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] == "electronics" + + def test_where_numeric_comparison(self, sql_index): + """Test WHERE with numeric field comparison.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price < 50 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) < 50 + + def test_where_combined_and(self, sql_index): + """Test WHERE with AND combining multiple conditions.""" + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'electronics' AND price < 100 + """ + ) + results = sql_index.query(sql_query) + + for result in results: + assert result["category"] == "electronics" + assert float(result["price"]) < 100 + + def test_where_numeric_range(self, sql_index): + """Test WHERE with numeric range (BETWEEN equivalent).""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price >= 25 AND price <= 50 + """ + ) + results = sql_index.query(sql_query) + + for result in results: + price = float(result["price"]) + assert 25 <= price <= 50 + + +class TestSQLQueryTagOperators: + """Tests for SQL tag field operators.""" + + def test_tag_not_equals(self, sql_index): + """Test tag != operator.""" + sql_query = SQLQuery( + f""" + SELECT title, category + FROM {sql_index.name} + WHERE category != 'electronics' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] != "electronics" + + def test_tag_in(self, sql_index): + """Test tag IN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, category + FROM {sql_index.name} + WHERE category IN ('books', 'accessories') + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] in ("books", "accessories") + + +class TestSQLQueryNumericOperators: + """Tests for SQL numeric field operators.""" + + def test_numeric_greater_than(self, sql_index): + """Test numeric > operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price > 100 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) > 100 + + def test_numeric_equals(self, sql_index): + """Test numeric = operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price = 45 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert float(result["price"]) == 45 + + def test_numeric_not_equals(self, sql_index): + """Test numeric != operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price != 45 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) != 45 + + @pytest.mark.xfail(reason="Numeric IN operator not yet supported in sql-redis") + def test_numeric_in(self, sql_index): + """Test numeric IN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price IN (45, 55, 65) + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert float(result["price"]) in (45, 55, 65) + + def test_numeric_between(self, sql_index): + """Test numeric BETWEEN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price BETWEEN 40 AND 60 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + price = float(result["price"]) + assert 40 <= price <= 60 + + +class TestSQLQueryTextOperators: + """Tests for SQL text field operators.""" + + def test_text_equals(self, sql_index): + """Test text = operator (full-text search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert "laptop" in result["title"].lower() + + def test_text_not_equals(self, sql_index): + """Test text != operator (negated full-text search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title != 'laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + # Results should not contain 'laptop' as a primary match + assert "laptop" not in result["title"].lower() + + def test_text_prefix(self, sql_index): + """Test text prefix search with wildcard (term*).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'lap*' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match titles starting with "lap" (e.g., "laptop") + assert "lap" in result["title"].lower() + + def test_text_suffix(self, sql_index): + """Test text suffix search with wildcard (*term).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE name = '*book' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match names ending with "book" (e.g., "Python Book") + assert "book" in result["name"].lower() + + def test_text_fuzzy(self, sql_index): + """Test text fuzzy search with Levenshtein distance (%term%).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = '%laptap%' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should fuzzy match "laptop" even with typo "laptap" + assert "laptop" in result["title"].lower() + + def test_text_phrase(self, sql_index): + """Test text phrase search (multi-word exact phrase).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'gaming laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match exact phrase "gaming laptop" + title_lower = result["title"].lower() + assert "gaming" in title_lower and "laptop" in title_lower + + def test_text_phrase_with_stopword(self, sql_index): + """Test text phrase search containing stop words. + + Redis does not index stop words (like 'and', 'the', 'is') by default. + The sql-redis library works around this by automatically stripping + stop words from phrase searches and emitting a warning. + See: https://redis.io/docs/latest/develop/ai/search-and-query/advanced-concepts/stopwords/ + """ + import warnings + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'laptop and keyboard' + """ + ) + results = sql_index.query(sql_query) + + # Should find the "Laptop and Keyboard Bundle" product + assert len(results) >= 1 + # Verify at least one result contains both "laptop" and "keyboard" + found_match = False + for result in results: + title_lower = result["title"].lower() + if "laptop" in title_lower and "keyboard" in title_lower: + found_match = True + break + assert found_match, "Expected to find a result with 'laptop' and 'keyboard'" + + # Verify a warning was emitted about stopword removal + stopword_warnings = [ + warning + for warning in w + if "Stopwords" in str(warning.message) + and "and" in str(warning.message).lower() + ] + assert ( + len(stopword_warnings) >= 1 + ), "Expected a warning about stopword removal" + + @pytest.mark.xfail(reason="Text IN operator not yet supported in sql-redis") + def test_text_in(self, sql_index): + """Test text IN operator (multiple term search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title IN ('Python', 'Redis') + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + title_lower = result["title"].lower() + assert "python" in title_lower or "redis" in title_lower + + +class TestSQLQueryOrderBy: + """Tests for SQL ORDER BY clause.""" + + def test_order_by_asc(self, sql_index): + """Test ORDER BY ascending.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + ORDER BY price ASC + """ + ) + results = sql_index.query(sql_query) + + prices = [float(r["price"]) for r in results] + assert prices == sorted(prices) + + def test_order_by_desc(self, sql_index): + """Test ORDER BY descending.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + ORDER BY price DESC + """ + ) + results = sql_index.query(sql_query) + + prices = [float(r["price"]) for r in results] + assert prices == sorted(prices, reverse=True) + + +class TestSQLQueryLimit: + """Tests for SQL LIMIT and OFFSET clauses.""" + + def test_limit(self, sql_index): + """Test LIMIT clause.""" + sql_query = SQLQuery(f"SELECT title FROM {sql_index.name} LIMIT 3") + results = sql_index.query(sql_query) + + assert len(results) == 3 + + def test_limit_with_offset(self, sql_index): + """Test LIMIT with OFFSET for pagination.""" + # First page + sql_query1 = SQLQuery( + f"SELECT title FROM {sql_index.name} ORDER BY price ASC LIMIT 3 OFFSET 0" + ) + results1 = sql_index.query(sql_query1) + + # Second page + sql_query2 = SQLQuery( + f"SELECT title FROM {sql_index.name} ORDER BY price ASC LIMIT 3 OFFSET 3" + ) + results2 = sql_index.query(sql_query2) + + assert len(results1) == 3 + assert len(results2) == 3 + # Pages should have different results + titles1 = {r["title"] for r in results1} + titles2 = {r["title"] for r in results2} + assert titles1.isdisjoint(titles2) + + +class TestSQLQueryAggregation: + """Tests for SQL aggregation (GROUP BY, COUNT, AVG, etc.).""" + + def test_count_all(self, sql_index): + """Test COUNT(*) aggregation.""" + sql_query = SQLQuery(f"SELECT COUNT(*) as total FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert int(results[0]["total"]) == 13 # 13 products in test data + + def test_group_by_with_count(self, sql_index): + """Test GROUP BY with COUNT.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + # Should have groups for electronics, books, accessories, stationery + categories = {r["category"] for r in results} + assert "electronics" in categories + assert "books" in categories + + def test_group_by_with_avg(self, sql_index): + """Test GROUP BY with AVG.""" + sql_query = SQLQuery( + f""" + SELECT category, AVG(price) as avg_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + # All results should have category and avg_price + for result in results: + assert "category" in result + assert "avg_price" in result + assert float(result["avg_price"]) > 0 + + def test_group_by_with_filter(self, sql_index): + """Test GROUP BY with WHERE filter.""" + sql_query = SQLQuery( + f""" + SELECT category, AVG(price) as avg_price + FROM {sql_index.name} + WHERE stock > 50 + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "avg_price" in result + + def test_group_by_with_sum(self, sql_index): + """Test GROUP BY with SUM reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, SUM(price) as total_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "total_price" in result + assert float(result["total_price"]) > 0 + + def test_group_by_with_min(self, sql_index): + """Test GROUP BY with MIN reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, MIN(price) as min_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "min_price" in result + assert float(result["min_price"]) > 0 + + def test_group_by_with_max(self, sql_index): + """Test GROUP BY with MAX reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, MAX(price) as max_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "max_price" in result + assert float(result["max_price"]) > 0 + + def test_global_sum(self, sql_index): + """Test global SUM aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT SUM(price) as total + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "total" in results[0] + assert float(results[0]["total"]) > 0 + + def test_global_min(self, sql_index): + """Test global MIN aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT MIN(price) as min_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "min_price" in results[0] + assert float(results[0]["min_price"]) > 0 + + def test_global_max(self, sql_index): + """Test global MAX aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT MAX(price) as max_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "max_price" in results[0] + assert float(results[0]["max_price"]) > 0 + + def test_multiple_reducers(self, sql_index): + """Test multiple reducers in a single query.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count, SUM(price) as total, AVG(price) as avg_price, MIN(price) as min_price, MAX(price) as max_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "count" in result + assert "total" in result + assert "avg_price" in result + assert "min_price" in result + assert "max_price" in result + + def test_count_distinct(self, sql_index): + """Test COUNT_DISTINCT reducer using Redis-specific syntax.""" + sql_query = SQLQuery( + f""" + SELECT COUNT_DISTINCT(category) as unique_categories + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "unique_categories" in results[0] + # Should have 4 unique categories: electronics, books, accessories, stationery + assert int(results[0]["unique_categories"]) == 4 + + def test_stddev(self, sql_index): + """Test STDDEV reducer.""" + sql_query = SQLQuery( + f""" + SELECT STDDEV(price) as price_stddev + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "price_stddev" in results[0] + # Verify it's a valid numeric value + stddev_value = float(results[0]["price_stddev"]) + assert stddev_value >= 0 # Standard deviation is always non-negative + + def test_quantile(self, sql_index): + """Test QUANTILE reducer.""" + sql_query = SQLQuery( + f""" + SELECT QUANTILE(price, 0.5) as median_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "median_price" in results[0] + # Verify it's a valid numeric value + median_value = float(results[0]["median_price"]) + assert median_value >= 0 + + def test_tolist(self, sql_index): + """Test TOLIST reducer via ARRAY_AGG SQL function.""" + sql_query = SQLQuery( + f""" + SELECT category, ARRAY_AGG(title) as titles + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "titles" in result + # TOLIST returns a comma-separated string or list of values + assert result["titles"] is not None + + def test_first_value(self, sql_index): + """Test FIRST_VALUE reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, FIRST_VALUE(title) as first_title + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "first_title" in result + # Verify it's a non-empty string + assert isinstance(result["first_title"], str) + assert len(result["first_title"]) > 0 + + +class TestSQLQueryIntegration: + """End-to-end integration tests matching proposal examples.""" + + def test_proposal_example_basic(self, sql_index): + """Test the basic example from the MLP proposal.""" + # Example from proposal doc (adapted for our test data) + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'books' + """ + ) + + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] == "books" + assert "title" in result + assert "price" in result + + +@pytest.fixture +def vector_index(redis_url, worker_id): + """Create a books index with vector embeddings for SQL query testing.""" + import numpy as np + + unique_id = str(uuid.uuid4())[:8] + index_name = f"sql_books_{worker_id}_{unique_id}" + + index = SearchIndex.from_dict( + { + "index": { + "name": index_name, + "prefix": f"book_{worker_id}_{unique_id}", + "storage_type": "hash", + }, + "fields": [ + {"name": "title", "type": "text", "attrs": {"sortable": True}}, + {"name": "genre", "type": "tag", "attrs": {"sortable": True}}, + {"name": "price", "type": "numeric", "attrs": {"sortable": True}}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 4, + "distance_metric": "cosine", + "algorithm": "flat", + "datatype": "float32", + }, + }, + ], + }, + redis_url=redis_url, + ) + + index.create(overwrite=True) + + # Create test books with embeddings + books = [ + { + "title": "Dune", + "genre": "Science Fiction", + "price": 15, + "embedding": np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes(), + }, + { + "title": "Foundation", + "genre": "Science Fiction", + "price": 18, + "embedding": np.array([0.15, 0.25, 0.35, 0.45], dtype=np.float32).tobytes(), + }, + { + "title": "Neuromancer", + "genre": "Science Fiction", + "price": 12, + "embedding": np.array([0.2, 0.3, 0.4, 0.5], dtype=np.float32).tobytes(), + }, + { + "title": "The Hobbit", + "genre": "Fantasy", + "price": 14, + "embedding": np.array([0.9, 0.8, 0.7, 0.6], dtype=np.float32).tobytes(), + }, + { + "title": "1984", + "genre": "Dystopian", + "price": 25, + "embedding": np.array([0.5, 0.5, 0.5, 0.5], dtype=np.float32).tobytes(), + }, + ] + + index.load(books) + + yield index + + # Cleanup + index.delete(drop=True) + + +class TestSQLQueryVectorSearch: + """Tests for SQL vector similarity search using cosine_distance() and vector_distance().""" + + def test_vector_distance_function(self, vector_index): + """Test vector search with vector_distance() function.""" + import numpy as np + + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, vector_distance(embedding, :vec) AS score + FROM {vector_index.name} + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + results = vector_index.query(sql_query) + + assert len(results) > 0 + assert len(results) <= 3 + for result in results: + assert "title" in result + assert "score" in result + # Score should be a valid non-negative distance value + score = float(result["score"]) + assert score >= 0 + + def test_vector_cosine_similarity(self, vector_index): + """Test vector search with cosine_distance() function - pgvector style.""" + import numpy as np + + # Query vector similar to Science Fiction books + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT + title, + genre, + price, + cosine_distance(embedding, :query_vector) AS vector_distance + FROM {vector_index.name} + WHERE genre = 'Science Fiction' + AND price <= 20 + ORDER BY cosine_distance(embedding, :query_vector) + LIMIT 3 + """, + params={"query_vector": query_vector}, + ) + + results = vector_index.query(sql_query) + + # Should return Science Fiction books under $20 + assert len(results) > 0 + assert len(results) <= 3 + for result in results: + assert result["genre"] == "Science Fiction" + assert float(result["price"]) <= 20 + # Verify vector_distance is returned (like VectorQuery with return_score=True) + assert "vector_distance" in result + # Distance should be a valid non-negative value + distance = float(result["vector_distance"]) + assert distance >= 0 + + def test_vector_redis_query_string(self, vector_index, redis_url): + """Test redis_query_string() returns correct KNN query for vector search.""" + import numpy as np + + # Query vector + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, cosine_distance(embedding, :vec) AS vector_distance + FROM {vector_index.name} + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + # Get the Redis command string + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify it's a valid FT.SEARCH with KNN syntax + assert redis_cmd.startswith("FT.SEARCH") + assert vector_index.name in redis_cmd + assert "KNN 3" in redis_cmd + assert "@embedding" in redis_cmd + assert "$vector" in redis_cmd + assert "vector_distance" in redis_cmd + + def test_vector_search_with_prefilter_redis_query_string( + self, vector_index, redis_url + ): + """Test redis_query_string() returns correct prefiltered KNN query.""" + import numpy as np + + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, genre, cosine_distance(embedding, :vec) AS vector_distance + FROM {vector_index.name} + WHERE genre = 'Science Fiction' + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify prefilter syntax: (filter)=>[KNN ...] + assert redis_cmd.startswith("FT.SEARCH") + assert "Science Fiction" in redis_cmd or "Science\\ Fiction" in redis_cmd + assert "=>[KNN" in redis_cmd + assert "KNN 3" in redis_cmd diff --git a/tests/integration/test_sql_redis_json.py b/tests/integration/test_sql_redis_json.py new file mode 100644 index 00000000..76191af0 --- /dev/null +++ b/tests/integration/test_sql_redis_json.py @@ -0,0 +1,1143 @@ +"""Integration tests for SQLQuery class. + +These tests verify that SQLQuery can translate SQL-like syntax +into proper Redis queries and return expected results. +""" + +import uuid + +import pytest + +from redisvl.index import SearchIndex +from redisvl.query import SQLQuery + + +@pytest.fixture +def sql_index(redis_url, worker_id): + """Create a products index for SQL query testing.""" + unique_id = str(uuid.uuid4())[:8] + index_name = f"sql_products_{worker_id}_{unique_id}" + + index = SearchIndex.from_dict( + { + "index": { + "name": index_name, + "prefix": f"product_{worker_id}_{unique_id}", + "storage_type": "json", + }, + "fields": [ + {"name": "title", "type": "text", "attrs": {"sortable": True}}, + {"name": "name", "type": "text", "attrs": {"sortable": True}}, + {"name": "price", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "stock", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "rating", "type": "numeric", "attrs": {"sortable": True}}, + {"name": "category", "type": "tag", "attrs": {"sortable": True}}, + {"name": "tags", "type": "tag"}, + ], + }, + redis_url=redis_url, + ) + + index.create(overwrite=True) + + # Load test data + products = [ + { + "title": "Gaming laptop Pro", + "name": "Gaming Laptop", + "price": 899, + "stock": 10, + "rating": 4.5, + "category": "electronics", + "tags": "sale,featured", + }, + { + "title": "Budget laptop Basic", + "name": "Budget Laptop", + "price": 499, + "stock": 25, + "rating": 3.8, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Premium laptop Ultra", + "name": "Premium Laptop", + "price": 1299, + "stock": 5, + "rating": 4.9, + "category": "electronics", + "tags": "featured", + }, + { + "title": "Python Programming", + "name": "Python Book", + "price": 45, + "stock": 100, + "rating": 4.7, + "category": "books", + "tags": "bestseller", + }, + { + "title": "Redis in Action", + "name": "Redis Book", + "price": 55, + "stock": 50, + "rating": 4.6, + "category": "books", + "tags": "featured", + }, + { + "title": "Data Science Guide", + "name": "DS Book", + "price": 65, + "stock": 30, + "rating": 4.4, + "category": "books", + "tags": "sale", + }, + { + "title": "Wireless Mouse", + "name": "Mouse", + "price": 29, + "stock": 200, + "rating": 4.2, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Mechanical Keyboard", + "name": "Keyboard", + "price": 149, + "stock": 75, + "rating": 4.6, + "category": "electronics", + "tags": "featured", + }, + { + "title": "USB Hub", + "name": "Hub", + "price": 25, + "stock": 150, + "rating": 3.9, + "category": "electronics", + "tags": "sale", + }, + { + "title": "Monitor Stand", + "name": "Stand", + "price": 89, + "stock": 40, + "rating": 4.1, + "category": "accessories", + "tags": "sale,featured", + }, + { + "title": "Desk Lamp", + "name": "Lamp", + "price": 35, + "stock": 80, + "rating": 4.0, + "category": "accessories", + "tags": "sale", + }, + { + "title": "Notebook Set", + "name": "Notebooks", + "price": 15, + "stock": 300, + "rating": 4.3, + "category": "stationery", + "tags": "bestseller", + }, + { + "title": "Laptop and Keyboard Bundle", + "name": "Bundle Pack", + "price": 999, + "stock": 15, + "rating": 4.7, + "category": "electronics", + "tags": "featured,sale", + }, + ] + + index.load(products) + + yield index + + # Cleanup + index.delete(drop=True) + + +class TestSQLQueryBasic: + """Tests for basic SQL SELECT queries.""" + + def test_import_sql_query(self): + """Verify SQLQuery can be imported from redisvl.query.""" + from redisvl.query import SQLQuery + + assert SQLQuery is not None + + def test_select_all_fields(self, sql_index): + """Test SELECT * returns all fields.""" + sql_query = SQLQuery(f"SELECT * FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) > 0 + # For JSON storage, results may contain '$' key with JSON string or parsed fields + first_result = results[0] + if "$" in first_result: + # JSON storage returns data under '$' key + import json + + data = json.loads(first_result["$"]) + assert "title" in data + assert "price" in data + else: + assert "title" in first_result + assert "price" in first_result + + def test_select_specific_fields(self, sql_index): + """Test SELECT with specific field list.""" + sql_query = SQLQuery(f"SELECT title, price FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) > 0 + # Results should contain requested fields + assert "title" in results[0] + assert "price" in results[0] + + def test_redis_query_string_with_client(self, sql_index): + """Test redis_query_string() with redis_client returns the Redis command string.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + + # Get the Redis command string using redis_client + redis_cmd = sql_query.redis_query_string(redis_client=sql_index._redis_client) + + # Verify it's a valid FT.SEARCH command + assert redis_cmd.startswith("FT.SEARCH") + assert sql_index.name in redis_cmd + assert "electronics" in redis_cmd + + def test_redis_query_string_with_url(self, sql_index, redis_url): + """Test redis_query_string() with redis_url returns the Redis command string.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + + # Get the Redis command string using redis_url + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify it's a valid FT.SEARCH command + assert redis_cmd.startswith("FT.SEARCH") + assert sql_index.name in redis_cmd + assert "electronics" in redis_cmd + + def test_redis_query_string_aggregate(self, sql_index): + """Test redis_query_string() returns FT.AGGREGATE for aggregation queries.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count + FROM {sql_index.name} + GROUP BY category + """ + ) + + redis_cmd = sql_query.redis_query_string(redis_client=sql_index._redis_client) + + # Verify it's a valid FT.AGGREGATE command + assert redis_cmd.startswith("FT.AGGREGATE") + assert sql_index.name in redis_cmd + assert "GROUPBY" in redis_cmd + + +class TestSQLQueryWhere: + """Tests for SQL WHERE clause filtering.""" + + def test_where_tag_equals(self, sql_index): + """Test WHERE with tag field equality.""" + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'electronics' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] == "electronics" + + def test_where_numeric_comparison(self, sql_index): + """Test WHERE with numeric field comparison.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price < 50 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) < 50 + + def test_where_combined_and(self, sql_index): + """Test WHERE with AND combining multiple conditions.""" + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'electronics' AND price < 100 + """ + ) + results = sql_index.query(sql_query) + + for result in results: + assert result["category"] == "electronics" + assert float(result["price"]) < 100 + + def test_where_numeric_range(self, sql_index): + """Test WHERE with numeric range (BETWEEN equivalent).""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price >= 25 AND price <= 50 + """ + ) + results = sql_index.query(sql_query) + + for result in results: + price = float(result["price"]) + assert 25 <= price <= 50 + + +class TestSQLQueryTagOperators: + """Tests for SQL tag field operators.""" + + def test_tag_not_equals(self, sql_index): + """Test tag != operator.""" + sql_query = SQLQuery( + f""" + SELECT title, category + FROM {sql_index.name} + WHERE category != 'electronics' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] != "electronics" + + def test_tag_in(self, sql_index): + """Test tag IN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, category + FROM {sql_index.name} + WHERE category IN ('books', 'accessories') + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] in ("books", "accessories") + + +class TestSQLQueryNumericOperators: + """Tests for SQL numeric field operators.""" + + def test_numeric_greater_than(self, sql_index): + """Test numeric > operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price > 100 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) > 100 + + def test_numeric_equals(self, sql_index): + """Test numeric = operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price = 45 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert float(result["price"]) == 45 + + def test_numeric_not_equals(self, sql_index): + """Test numeric != operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price != 45 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert float(result["price"]) != 45 + + @pytest.mark.xfail(reason="Numeric IN operator not yet supported in sql-redis") + def test_numeric_in(self, sql_index): + """Test numeric IN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price IN (45, 55, 65) + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert float(result["price"]) in (45, 55, 65) + + def test_numeric_between(self, sql_index): + """Test numeric BETWEEN operator.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + WHERE price BETWEEN 40 AND 60 + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + price = float(result["price"]) + assert 40 <= price <= 60 + + +class TestSQLQueryTextOperators: + """Tests for SQL text field operators.""" + + def test_text_equals(self, sql_index): + """Test text = operator (full-text search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + assert "laptop" in result["title"].lower() + + def test_text_not_equals(self, sql_index): + """Test text != operator (negated full-text search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title != 'laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + # Results should not contain 'laptop' as a primary match + assert "laptop" not in result["title"].lower() + + def test_text_prefix(self, sql_index): + """Test text prefix search with wildcard (term*).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'lap*' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match titles starting with "lap" (e.g., "laptop") + assert "lap" in result["title"].lower() + + def test_text_suffix(self, sql_index): + """Test text suffix search with wildcard (*term).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE name = '*book' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match names ending with "book" (e.g., "Python Book") + assert "book" in result["name"].lower() + + def test_text_fuzzy(self, sql_index): + """Test text fuzzy search with Levenshtein distance (%term%).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = '%laptap%' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should fuzzy match "laptop" even with typo "laptap" + assert "laptop" in result["title"].lower() + + def test_text_phrase(self, sql_index): + """Test text phrase search (multi-word exact phrase).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'gaming laptop' + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + # Should match exact phrase "gaming laptop" + title_lower = result["title"].lower() + assert "gaming" in title_lower and "laptop" in title_lower + + def test_text_phrase_with_stopword(self, sql_index): + """Test text phrase search containing stop words. + + Redis does not index stop words (like 'and', 'the', 'is') by default. + The sql-redis library works around this by automatically stripping + stop words from phrase searches and emitting a warning. + See: https://redis.io/docs/latest/develop/ai/search-and-query/advanced-concepts/stopwords/ + """ + import warnings + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title = 'laptop and keyboard' + """ + ) + results = sql_index.query(sql_query) + + # Should find the "Laptop and Keyboard Bundle" product + assert len(results) >= 1 + # Verify at least one result contains both "laptop" and "keyboard" + found_match = False + for result in results: + title_lower = result["title"].lower() + if "laptop" in title_lower and "keyboard" in title_lower: + found_match = True + break + assert found_match, "Expected to find a result with 'laptop' and 'keyboard'" + + # Verify a warning was emitted about stopword removal + stopword_warnings = [ + warning + for warning in w + if "Stopwords" in str(warning.message) + and "and" in str(warning.message).lower() + ] + assert ( + len(stopword_warnings) >= 1 + ), "Expected a warning about stopword removal" + + @pytest.mark.xfail(reason="Text IN operator not yet supported in sql-redis") + def test_text_in(self, sql_index): + """Test text IN operator (multiple term search).""" + sql_query = SQLQuery( + f""" + SELECT title, name + FROM {sql_index.name} + WHERE title IN ('Python', 'Redis') + """ + ) + results = sql_index.query(sql_query) + + assert len(results) >= 1 + for result in results: + title_lower = result["title"].lower() + assert "python" in title_lower or "redis" in title_lower + + +class TestSQLQueryOrderBy: + """Tests for SQL ORDER BY clause.""" + + def test_order_by_asc(self, sql_index): + """Test ORDER BY ascending.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + ORDER BY price ASC + """ + ) + results = sql_index.query(sql_query) + + prices = [float(r["price"]) for r in results] + assert prices == sorted(prices) + + def test_order_by_desc(self, sql_index): + """Test ORDER BY descending.""" + sql_query = SQLQuery( + f""" + SELECT title, price + FROM {sql_index.name} + ORDER BY price DESC + """ + ) + results = sql_index.query(sql_query) + + prices = [float(r["price"]) for r in results] + assert prices == sorted(prices, reverse=True) + + +class TestSQLQueryLimit: + """Tests for SQL LIMIT and OFFSET clauses.""" + + def test_limit(self, sql_index): + """Test LIMIT clause.""" + sql_query = SQLQuery(f"SELECT title FROM {sql_index.name} LIMIT 3") + results = sql_index.query(sql_query) + + assert len(results) == 3 + + def test_limit_with_offset(self, sql_index): + """Test LIMIT with OFFSET for pagination.""" + # First page + sql_query1 = SQLQuery( + f"SELECT title FROM {sql_index.name} ORDER BY price ASC LIMIT 3 OFFSET 0" + ) + results1 = sql_index.query(sql_query1) + + # Second page + sql_query2 = SQLQuery( + f"SELECT title FROM {sql_index.name} ORDER BY price ASC LIMIT 3 OFFSET 3" + ) + results2 = sql_index.query(sql_query2) + + assert len(results1) == 3 + assert len(results2) == 3 + # Pages should have different results + titles1 = {r["title"] for r in results1} + titles2 = {r["title"] for r in results2} + assert titles1.isdisjoint(titles2) + + +class TestSQLQueryAggregation: + """Tests for SQL aggregation (GROUP BY, COUNT, AVG, etc.).""" + + def test_count_all(self, sql_index): + """Test COUNT(*) aggregation.""" + sql_query = SQLQuery(f"SELECT COUNT(*) as total FROM {sql_index.name}") + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert int(results[0]["total"]) == 13 # 13 products in test data + + def test_group_by_with_count(self, sql_index): + """Test GROUP BY with COUNT.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + # Should have groups for electronics, books, accessories, stationery + categories = {r["category"] for r in results} + assert "electronics" in categories + assert "books" in categories + + def test_group_by_with_avg(self, sql_index): + """Test GROUP BY with AVG.""" + sql_query = SQLQuery( + f""" + SELECT category, AVG(price) as avg_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + # All results should have category and avg_price + for result in results: + assert "category" in result + assert "avg_price" in result + assert float(result["avg_price"]) > 0 + + def test_group_by_with_filter(self, sql_index): + """Test GROUP BY with WHERE filter.""" + sql_query = SQLQuery( + f""" + SELECT category, AVG(price) as avg_price + FROM {sql_index.name} + WHERE stock > 50 + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "avg_price" in result + + def test_group_by_with_sum(self, sql_index): + """Test GROUP BY with SUM reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, SUM(price) as total_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "total_price" in result + assert float(result["total_price"]) > 0 + + def test_group_by_with_min(self, sql_index): + """Test GROUP BY with MIN reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, MIN(price) as min_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "min_price" in result + assert float(result["min_price"]) > 0 + + def test_group_by_with_max(self, sql_index): + """Test GROUP BY with MAX reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, MAX(price) as max_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "max_price" in result + assert float(result["max_price"]) > 0 + + def test_global_sum(self, sql_index): + """Test global SUM aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT SUM(price) as total + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "total" in results[0] + assert float(results[0]["total"]) > 0 + + def test_global_min(self, sql_index): + """Test global MIN aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT MIN(price) as min_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "min_price" in results[0] + assert float(results[0]["min_price"]) > 0 + + def test_global_max(self, sql_index): + """Test global MAX aggregation (no GROUP BY).""" + sql_query = SQLQuery( + f""" + SELECT MAX(price) as max_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "max_price" in results[0] + assert float(results[0]["max_price"]) > 0 + + def test_multiple_reducers(self, sql_index): + """Test multiple reducers in a single query.""" + sql_query = SQLQuery( + f""" + SELECT category, COUNT(*) as count, SUM(price) as total, AVG(price) as avg_price, MIN(price) as min_price, MAX(price) as max_price + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "category" in result + assert "count" in result + assert "total" in result + assert "avg_price" in result + assert "min_price" in result + assert "max_price" in result + + def test_count_distinct(self, sql_index): + """Test COUNT_DISTINCT reducer using Redis-specific syntax.""" + sql_query = SQLQuery( + f""" + SELECT COUNT_DISTINCT(category) as unique_categories + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "unique_categories" in results[0] + # Should have 4 unique categories: electronics, books, accessories, stationery + assert int(results[0]["unique_categories"]) == 4 + + def test_stddev(self, sql_index): + """Test STDDEV reducer.""" + sql_query = SQLQuery( + f""" + SELECT STDDEV(price) as price_stddev + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "price_stddev" in results[0] + # Verify it's a valid numeric value + stddev_value = float(results[0]["price_stddev"]) + assert stddev_value >= 0 # Standard deviation is always non-negative + + def test_quantile(self, sql_index): + """Test QUANTILE reducer.""" + sql_query = SQLQuery( + f""" + SELECT QUANTILE(price, 0.5) as median_price + FROM {sql_index.name} + """ + ) + results = sql_index.query(sql_query) + + assert len(results) == 1 + assert "median_price" in results[0] + # Verify it's a valid numeric value + median_value = float(results[0]["median_price"]) + assert median_value >= 0 + + def test_tolist(self, sql_index): + """Test TOLIST reducer via ARRAY_AGG SQL function.""" + sql_query = SQLQuery( + f""" + SELECT category, ARRAY_AGG(title) as titles + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "titles" in result + # TOLIST returns a comma-separated string or list of values + assert result["titles"] is not None + + def test_first_value(self, sql_index): + """Test FIRST_VALUE reducer.""" + sql_query = SQLQuery( + f""" + SELECT category, FIRST_VALUE(title) as first_title + FROM {sql_index.name} + GROUP BY category + """ + ) + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert "first_title" in result + # Verify it's a non-empty string + assert isinstance(result["first_title"], str) + assert len(result["first_title"]) > 0 + + +class TestSQLQueryIntegration: + """End-to-end integration tests matching proposal examples.""" + + def test_proposal_example_basic(self, sql_index): + """Test the basic example from the MLP proposal.""" + # Example from proposal doc (adapted for our test data) + sql_query = SQLQuery( + f""" + SELECT title, price, category + FROM {sql_index.name} + WHERE category = 'books' + """ + ) + + results = sql_index.query(sql_query) + + assert len(results) > 0 + for result in results: + assert result["category"] == "books" + assert "title" in result + assert "price" in result + + +@pytest.fixture +def vector_index(redis_url, worker_id): + """Create a books index with vector embeddings for SQL query testing.""" + import numpy as np + + unique_id = str(uuid.uuid4())[:8] + index_name = f"sql_books_{worker_id}_{unique_id}" + + index = SearchIndex.from_dict( + { + "index": { + "name": index_name, + "prefix": f"book_{worker_id}_{unique_id}", + "storage_type": "hash", + }, + "fields": [ + {"name": "title", "type": "text", "attrs": {"sortable": True}}, + {"name": "genre", "type": "tag", "attrs": {"sortable": True}}, + {"name": "price", "type": "numeric", "attrs": {"sortable": True}}, + { + "name": "embedding", + "type": "vector", + "attrs": { + "dims": 4, + "distance_metric": "cosine", + "algorithm": "flat", + "datatype": "float32", + }, + }, + ], + }, + redis_url=redis_url, + ) + + index.create(overwrite=True) + + # Create test books with embeddings + books = [ + { + "title": "Dune", + "genre": "Science Fiction", + "price": 15, + "embedding": np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes(), + }, + { + "title": "Foundation", + "genre": "Science Fiction", + "price": 18, + "embedding": np.array([0.15, 0.25, 0.35, 0.45], dtype=np.float32).tobytes(), + }, + { + "title": "Neuromancer", + "genre": "Science Fiction", + "price": 12, + "embedding": np.array([0.2, 0.3, 0.4, 0.5], dtype=np.float32).tobytes(), + }, + { + "title": "The Hobbit", + "genre": "Fantasy", + "price": 14, + "embedding": np.array([0.9, 0.8, 0.7, 0.6], dtype=np.float32).tobytes(), + }, + { + "title": "1984", + "genre": "Dystopian", + "price": 25, + "embedding": np.array([0.5, 0.5, 0.5, 0.5], dtype=np.float32).tobytes(), + }, + ] + + index.load(books) + + yield index + + # Cleanup + index.delete(drop=True) + + +class TestSQLQueryVectorSearch: + """Tests for SQL vector similarity search using cosine_distance() and vector_distance().""" + + def test_vector_distance_function(self, vector_index): + """Test vector search with vector_distance() function.""" + import numpy as np + + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, vector_distance(embedding, :vec) AS score + FROM {vector_index.name} + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + results = vector_index.query(sql_query) + + assert len(results) > 0 + assert len(results) <= 3 + for result in results: + assert "title" in result + assert "score" in result + # Score should be a valid non-negative distance value + score = float(result["score"]) + assert score >= 0 + + def test_vector_cosine_similarity(self, vector_index): + """Test vector search with cosine_distance() function - pgvector style.""" + import numpy as np + + # Query vector similar to Science Fiction books + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT + title, + genre, + price, + cosine_distance(embedding, :query_vector) AS vector_distance + FROM {vector_index.name} + WHERE genre = 'Science Fiction' + AND price <= 20 + ORDER BY cosine_distance(embedding, :query_vector) + LIMIT 3 + """, + params={"query_vector": query_vector}, + ) + + results = vector_index.query(sql_query) + + # Should return Science Fiction books under $20 + assert len(results) > 0 + assert len(results) <= 3 + for result in results: + assert result["genre"] == "Science Fiction" + assert float(result["price"]) <= 20 + # Verify vector_distance is returned (like VectorQuery with return_score=True) + assert "vector_distance" in result + # Distance should be a valid non-negative value + distance = float(result["vector_distance"]) + assert distance >= 0 + + def test_vector_redis_query_string(self, vector_index, redis_url): + """Test redis_query_string() returns correct KNN query for vector search.""" + import numpy as np + + # Query vector + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, cosine_distance(embedding, :vec) AS vector_distance + FROM {vector_index.name} + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + # Get the Redis command string + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify it's a valid FT.SEARCH with KNN syntax + assert redis_cmd.startswith("FT.SEARCH") + assert vector_index.name in redis_cmd + assert "KNN 3" in redis_cmd + assert "@embedding" in redis_cmd + assert "$vector" in redis_cmd + assert "vector_distance" in redis_cmd + + def test_vector_search_with_prefilter_redis_query_string( + self, vector_index, redis_url + ): + """Test redis_query_string() returns correct prefiltered KNN query.""" + import numpy as np + + query_vector = np.array([0.1, 0.2, 0.3, 0.4], dtype=np.float32).tobytes() + + sql_query = SQLQuery( + f""" + SELECT title, genre, cosine_distance(embedding, :vec) AS vector_distance + FROM {vector_index.name} + WHERE genre = 'Science Fiction' + LIMIT 3 + """, + params={"vec": query_vector}, + ) + + redis_cmd = sql_query.redis_query_string(redis_url=redis_url) + + # Verify prefilter syntax: (filter)=>[KNN ...] + assert redis_cmd.startswith("FT.SEARCH") + assert "Science Fiction" in redis_cmd or "Science\\ Fiction" in redis_cmd + assert "=>[KNN" in redis_cmd + assert "KNN 3" in redis_cmd diff --git a/tests/unit/test_sql_parameter_substitution.py b/tests/unit/test_sql_parameter_substitution.py new file mode 100644 index 00000000..086b658c --- /dev/null +++ b/tests/unit/test_sql_parameter_substitution.py @@ -0,0 +1,225 @@ +"""Unit tests for SQL parameter substitution in SQLQuery. + +These tests verify that parameter substitution correctly handles: +1. Partial matching bug: :id should not replace inside :product_id +2. Quote escaping bug: Single quotes in values should be SQL-escaped +3. Edge cases: Multiple occurrences, similar names, special characters +""" + +import pytest + +from redisvl.query.sql import SQLQuery + + +def buggy_substitute_params(sql: str, params: dict) -> str: + """Simulate the CURRENT buggy implementation for comparison. + + This is the exact code from redisvl/query/sql.py lines 105-113. + """ + for key, value in params.items(): + placeholder = f":{key}" + if isinstance(value, (int, float)): + sql = sql.replace(placeholder, str(value)) + elif isinstance(value, str): + sql = sql.replace(placeholder, f"'{value}'") + return sql + + +class TestBuggyBehaviorDemonstration: + """Tests that DEMONSTRATE the bugs in the current implementation. + + These tests show what goes wrong with the naive str.replace() approach. + They should PASS (demonstrating the bug exists) before the fix, + and some assertions will need to change after the fix. + """ + + def test_partial_match_bug_exists(self): + """Demonstrate that :id incorrectly replaces inside :product_id.""" + sql = "SELECT * FROM idx WHERE id = :id AND product_id = :product_id" + params = {"id": 123, "product_id": 456} + + result = buggy_substitute_params(sql, params) + + # BUG: :id gets replaced inside :product_id first (dict ordering dependent) + # This demonstrates the bug - the result is corrupted + # Depending on dict ordering, we might get "product_123" corruption + assert ":id" not in result or "product_" in result # Some substitution happened + + def test_quote_escaping_bug_exists(self): + """Demonstrate that quotes are NOT escaped in current implementation.""" + sql = "SELECT * FROM idx WHERE name = :name" + params = {"name": "O'Brien"} + + result = buggy_substitute_params(sql, params) + + # BUG: The quote is NOT escaped - this produces invalid SQL + assert "O'Brien" in result # Raw quote, not escaped + assert "O''Brien" not in result # Proper escaping is missing + + +class TestParameterSubstitutionPartialMatching: + """Tests for the partial string matching bug. + + The bug: Using str.replace(':id', '123') would also replace + ':id' inside ':product_id', resulting in 'product_123'. + """ + + def test_similar_param_names_no_partial_match(self): + """Test that :id doesn't replace inside :product_id.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE id = :id AND product_id = :product_id", + params={"id": 123, "product_id": 456}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "id = 123" in substituted + assert "product_id = 456" in substituted + # Should NOT have "product_123" + assert "product_123" not in substituted + + def test_prefix_param_names(self): + """Test params where one is a prefix of another: :user, :user_id, :user_name.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE user = :user AND user_id = :user_id AND user_name = :user_name", + params={"user": "alice", "user_id": 42, "user_name": "Alice Smith"}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "user = 'alice'" in substituted + assert "user_id = 42" in substituted + assert "user_name = 'Alice Smith'" in substituted + # Should NOT have corrupted values + assert "'alice'_id" not in substituted + assert "'alice'_name" not in substituted + + def test_suffix_param_names(self): + """Test params where one is a suffix pattern: :vec, :query_vec.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE vec = :vec AND query_vec = :query_vec", + params={"vec": 1.0, "query_vec": 2.0}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "vec = 1.0" in substituted or "vec = 1" in substituted + assert "query_vec = 2.0" in substituted or "query_vec = 2" in substituted + + +class TestParameterSubstitutionQuoteEscaping: + """Tests for the quote escaping bug. + + The bug: String values with single quotes like "O'Brien" would + produce invalid SQL: 'O'Brien' instead of 'O''Brien'. + """ + + def test_single_quote_in_value(self): + """Test that single quotes are properly escaped.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE name = :name", + params={"name": "O'Brien"}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + # SQL standard escaping: ' becomes '' + assert "name = 'O''Brien'" in substituted + + def test_multiple_quotes_in_value(self): + """Test multiple single quotes in a value.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE phrase = :phrase", + params={"phrase": "It's a 'test' string"}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "phrase = 'It''s a ''test'' string'" in substituted + + def test_apostrophe_names(self): + """Test common names with apostrophes.""" + test_cases = [ + ("McDonald's", "'McDonald''s'"), + ("O'Reilly", "'O''Reilly'"), + ("D'Angelo", "'D''Angelo'"), + ] + + for name, expected in test_cases: + sql_query = SQLQuery( + "SELECT * FROM idx WHERE name = :name", + params={"name": name}, + ) + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + assert f"name = {expected}" in substituted, f"Failed for {name}" + + +class TestParameterSubstitutionEdgeCases: + """Tests for edge cases in parameter substitution.""" + + def test_multiple_occurrences_same_param(self): + """Test that a parameter used multiple times is substituted everywhere.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE category = :cat OR subcategory = :cat", + params={"cat": "electronics"}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert substituted.count("'electronics'") == 2 + + def test_empty_string_value(self): + """Test empty string parameter value.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE name = :name", + params={"name": ""}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "name = ''" in substituted + + def test_numeric_types(self): + """Test integer and float parameter values.""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE count = :count AND price = :price", + params={"count": 42, "price": 99.99}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + assert "count = 42" in substituted + assert "price = 99.99" in substituted + + def test_bytes_param_not_substituted(self): + """Test that bytes parameters are not substituted (handled separately).""" + sql_query = SQLQuery( + "SELECT * FROM idx WHERE embedding = :vec", + params={"vec": b"\x00\x01\x02\x03"}, + ) + + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + + # Bytes should remain as placeholder + assert ":vec" in substituted + + def test_special_characters_in_value(self): + """Test special characters that might interfere with regex.""" + special_values = [ + "hello@world.com", + "path/to/file", + "price: $100", + "regex.*pattern", + "back\\slash", + ] + + for value in special_values: + sql_query = SQLQuery( + "SELECT * FROM idx WHERE field = :field", + params={"field": value}, + ) + substituted = sql_query._substitute_params(sql_query.sql, sql_query.params) + # Should contain the value wrapped in quotes (with any necessary escaping) + assert ( + ":field" not in substituted + ), f"Failed to substitute for value: {value}" diff --git a/uv.lock b/uv.lock index ad61bd62..aa18dac1 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9.2, <3.14" resolution-markers = [ "python_full_version >= '3.13'", @@ -4255,7 +4255,7 @@ wheels = [ [[package]] name = "redisvl" -version = "0.13.2" +version = "0.14.0" source = { editable = "." } dependencies = [ { name = "jsonpath-ng" }, @@ -4299,6 +4299,9 @@ pillow = [ sentence-transformers = [ { name = "sentence-transformers" }, ] +sql-redis = [ + { name = "sql-redis" }, +] vertexai = [ { name = "google-cloud-aiplatform" }, { name = "protobuf" }, @@ -4355,11 +4358,12 @@ requires-dist = [ { name = "pyyaml", specifier = ">=5.4,<7.0" }, { name = "redis", specifier = ">=5.0,<7.2" }, { name = "sentence-transformers", marker = "extra == 'sentence-transformers'", specifier = ">=3.4.0,<4" }, + { name = "sql-redis", marker = "extra == 'sql-redis'", specifier = ">=0.1.2" }, { name = "tenacity", specifier = ">=8.2.2" }, { name = "urllib3", marker = "extra == 'bedrock'", specifier = "<2.2.0" }, { name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.2.2" }, ] -provides-extras = ["mistralai", "openai", "nltk", "cohere", "voyageai", "sentence-transformers", "langcache", "vertexai", "bedrock", "pillow"] +provides-extras = ["mistralai", "openai", "nltk", "cohere", "voyageai", "sentence-transformers", "langcache", "vertexai", "bedrock", "pillow", "sql-redis"] [package.metadata.requires-dev] dev = [ @@ -5263,6 +5267,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, ] +[[package]] +name = "sql-redis" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "redis", version = "7.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "redis", version = "7.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sqlglot" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/d0/d16fccfa2c526f86fa14de2b24b000ff6f9e9d3eb50585326e8a12a74895/sql_redis-0.1.2.tar.gz", hash = "sha256:abdef256af90e4f2815c8983ec23563104cee7b8469234012fc1688df00bb499", size = 107321, upload-time = "2026-02-06T15:30:15.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/55/4fbdfa4fdceab9ba97a08e61c2294d41a3203cb90bcfc657fc83c7342d0b/sql_redis-0.1.2-py3-none-any.whl", hash = "sha256:2efae4b1e25cbc9e27d1e2f3e4dbc3bce277a534475167c6c821a2e037a7e516", size = 19905, upload-time = "2026-02-06T15:30:13.973Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.44" @@ -5316,6 +5334,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/5e/6a29fa884d9fb7ddadf6b69490a9d45fded3b38541713010dad16b77d015/sqlalchemy-2.0.44-py3-none-any.whl", hash = "sha256:19de7ca1246fbef9f9d1bff8f1ab25641569df226364a0e40457dc5457c54b05", size = 1928718, upload-time = "2025-10-10T15:29:45.32Z" }, ] +[[package]] +name = "sqlglot" +version = "28.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/b6/f188b9616bef49943353f3622d726af30fdb08acbd081deef28ba43ceb48/sqlglot-28.6.0.tar.gz", hash = "sha256:8c0a432a6745c6c7965bbe99a17667c5a3ca1d524a54b31997cf5422b1727f6a", size = 5676522, upload-time = "2026-01-13T17:39:24.389Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/a6/21b1e19994296ba4a34bc7abaf4fcb40d7e7787477bdfde58cd843594459/sqlglot-28.6.0-py3-none-any.whl", hash = "sha256:8af76e825dc8456a49f8ce049d69bbfcd116655dda3e53051754789e2edf8eba", size = 575186, upload-time = "2026-01-13T17:39:22.327Z" }, +] + [[package]] name = "stack-data" version = "0.6.3"