From 76e48a5d7718f1d77415223529a91f0c834fe623 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Wed, 31 Dec 2025 22:42:03 +0000 Subject: [PATCH 1/8] iAdding adaptive search notebook --- .../adaptive_video_search.ipynb | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 notebooks/advanced_techniques/adaptive_video_search.ipynb diff --git a/notebooks/advanced_techniques/adaptive_video_search.ipynb b/notebooks/advanced_techniques/adaptive_video_search.ipynb new file mode 100644 index 0000000..e2469fc --- /dev/null +++ b/notebooks/advanced_techniques/adaptive_video_search.ipynb @@ -0,0 +1,276 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3743e5aa-5a08-4b37-a545-4e7461e14468", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/advanced_techniques/adaptive_video_search.ipynb)\n", + "\n", + "[![View Article](https://img.shields.io/badge/View%20Article-blue)](https://www.mongodb.com/company/blog/technical/adaptive-video-search/?utm_campaign=devrel&utm_source=cross-post&utm_medium=organic_social&utm_content=https%3A%2F%2Fgithub.com%2Fmongodb-developer%2FGenAI-Showcase&utm_term=apoorva.joshi)" + ] + }, + { + "cell_type": "markdown", + "id": "becc445d-5a51-4a8a-abeb-ceab0b54c167", + "metadata": {}, + "source": [ + "# Building an Adaptive Video Search System using Voyage AI and MongoDB" + ] + }, + { + "cell_type": "markdown", + "id": "8beabeb0-f231-4e56-9af3-3178be3c92cf", + "metadata": {}, + "source": [ + "## Step 1: Install required libraries\n", + "\n", + "- **voyageai**: Voyage AI's Python SDK\n", + "- **pymongo**: MongoDB's Python driver\n", + "- **anthropic**: Anthropic's Python SDK\n", + "- **huggingface_hub**: Python library for interacting with the Hugging Face Hub" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e9d5d1c4-614b-42de-9e15-c02146e8be89", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qU voyageai==0.3.7 pymongo==4.15.5 anthropic==0.75.0 huggingface-hub==1.2.3" + ] + }, + { + "cell_type": "markdown", + "id": "aafea857-7c11-48ff-a5cc-21a44de2f02b", + "metadata": {}, + "source": [ + "## Step 2: Setup prerequisites\n", + "\n", + "**Voyage AI**\n", + "- [Obtain a Voyage AI API key](https://dashboard.voyageai.com/organization/api-keys)\n", + "\n", + "**MongoDB**\n", + "- Register for a [free MongoDB Atlas account](https://www.mongodb.com/cloud/atlas/register)\n", + "- [Create a new database cluster](https://www.mongodb.com/docs/guides/atlas/cluster/)\n", + "- [Obtain the connection string](https://www.mongodb.com/docs/guides/atlas/connection-string/) for your database cluster\n", + "\n", + "**Anthropic**\n", + "- [Obtain an Anthropic API key](https://platform.claude.com/settings/keys)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "12b1c0b3-b18a-4fda-b730-0693b5259a8f", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "import anthropic\n", + "import voyageai\n", + "from pymongo import MongoClient" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d05bb72f-bebc-4255-8206-7a3b70b3d302", + "metadata": {}, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Enter your VoyageAI API key: ········\n" + ] + } + ], + "source": [ + "# Set Voyage AI API key as an environment variable\n", + "os.environ[\"VOYAGE_API_KEY\"] = getpass.getpass(\"Enter your VoyageAI API key:\")\n", + "# Initialize the Voyage AI client\n", + "voyage_client = voyageai.Client()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "662fc622-7d64-4d12-acb4-8eaa425aa829", + "metadata": {}, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Enter your MongoDB connection string: ········\n" + ] + }, + { + "data": { + "text/plain": [ + "{'ok': 1.0,\n", + " '$clusterTime': {'clusterTime': Timestamp(1767219327, 1),\n", + " 'signature': {'hash': b'\\xc0[1\\xf8\\xdfM\\xd5\\x943\\xb06jV\\xdbu$=\\x0c\\xb7p',\n", + " 'keyId': 7558184680432861186}},\n", + " 'operationTime': Timestamp(1767219327, 1)}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Set the MongoDB connection string\n", + "MONGODB_URI = getpass.getpass(\"Enter your MongoDB connection string:\")\n", + "# Initialize the MongoDB client\n", + "mongodb_client = MongoClient(\n", + " MONGODB_URI, appname=\"devrel.showcase.adaptive_video_search\"\n", + ")\n", + "# Check MongoDB connection\n", + "mongodb_client.admin.command(\"ping\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f0baa8a5-8625-4d41-a2ab-13404b13f3cd", + "metadata": {}, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Enter your Anthropic API key: ········\n" + ] + } + ], + "source": [ + "# Set Anthropic API key as an environment variable\n", + "os.environ[\"ANTHROPIC_API_KEY\"] = getpass.getpass(\"Enter your Anthropic API key:\")\n", + "# Initialize the Anthropic client\n", + "anthropic_client = anthropic.Anthropic()" + ] + }, + { + "cell_type": "markdown", + "id": "c58c9820-c53b-49e5-8023-b5234cf7d817", + "metadata": {}, + "source": [ + "## Step 3: Download the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6b1cef44-0d97-44fb-878e-7e3872c0d8fe", + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import snapshot_download" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "d698eba9-fa77-4714-b7d5-558aecd9e93d", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b775014dbb5248edadc4f237dd3ca3b4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading (incomplete total...): 0.00B [00:00, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "25c2c5535b8e410ebac1c7b6d908ece1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 18 files: 0%| | 0/18 [00:00 Date: Fri, 2 Jan 2026 22:38:16 +0000 Subject: [PATCH 2/8] Added everything else --- .../adaptive_video_search.ipynb | 728 +++++++++++++++++- 1 file changed, 703 insertions(+), 25 deletions(-) diff --git a/notebooks/advanced_techniques/adaptive_video_search.ipynb b/notebooks/advanced_techniques/adaptive_video_search.ipynb index e2469fc..093160d 100644 --- a/notebooks/advanced_techniques/adaptive_video_search.ipynb +++ b/notebooks/advanced_techniques/adaptive_video_search.ipynb @@ -23,22 +23,49 @@ "id": "8beabeb0-f231-4e56-9af3-3178be3c92cf", "metadata": {}, "source": [ - "## Step 1: Install required libraries\n", + "## Step 1: Install required packages\n", "\n", "- **voyageai**: Voyage AI's Python SDK\n", "- **pymongo**: MongoDB's Python driver\n", "- **anthropic**: Anthropic's Python SDK\n", - "- **huggingface_hub**: Python library for interacting with the Hugging Face Hub" + "- **huggingface_hub**: Python library for interacting with the Hugging Face Hub\n", + "- **ffmpeg-python**: Python wrapper for `ffmpeg`\n", + "- **tqdm**: Python library to display progress bars for loops" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 1, "id": "e9d5d1c4-614b-42de-9e15-c02146e8be89", "metadata": {}, "outputs": [], "source": [ - "!pip install -qU voyageai==0.3.7 pymongo==4.15.5 anthropic==0.75.0 huggingface-hub==1.2.3" + "!pip install -qU voyageai==0.3.7 pymongo==4.15.5 anthropic==0.75.0 huggingface-hub==1.2.3 ffmpeg-python==0.2.0 tqdm==4.67.1" + ] + }, + { + "cell_type": "markdown", + "id": "d4803c8c-eb59-44aa-9306-5e49088add6b", + "metadata": {}, + "source": [ + "You'll also need to install `ffmpeg` binary itself. To do this, run the following commands from the terminal:\n", + "\n", + "#### MacOS\n", + "\n", + "```\n", + "brew install ffmpeg\n", + "```\n", + "\n", + "#### Linux\n", + "\n", + "```\n", + "sudo apt-get install ffmpeg\n", + "```\n", + "\n", + "#### Windows\n", + "* Download the executable from [ffmpeg.org](https://ffmpeg.org/download.html#build-windows)\n", + "* Extract the downloaded zip file\n", + "* Note the path to the `bin` folder" ] }, { @@ -62,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 2, "id": "12b1c0b3-b18a-4fda-b730-0693b5259a8f", "metadata": {}, "outputs": [], @@ -77,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "d05bb72f-bebc-4255-8206-7a3b70b3d302", "metadata": {}, "outputs": [ @@ -98,7 +125,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "id": "662fc622-7d64-4d12-acb4-8eaa425aa829", "metadata": {}, "outputs": [ @@ -113,13 +140,13 @@ "data": { "text/plain": [ "{'ok': 1.0,\n", - " '$clusterTime': {'clusterTime': Timestamp(1767219327, 1),\n", - " 'signature': {'hash': b'\\xc0[1\\xf8\\xdfM\\xd5\\x943\\xb06jV\\xdbu$=\\x0c\\xb7p',\n", + " '$clusterTime': {'clusterTime': Timestamp(1767387291, 1),\n", + " 'signature': {'hash': b'\\xf8\\xbcI\\xcf\\x81DR\\xc1\\xcdO\\xcf\\xa8\\x1d\\xc9\\x1do\\x14dH\\xf2',\n", " 'keyId': 7558184680432861186}},\n", - " 'operationTime': Timestamp(1767219327, 1)}" + " 'operationTime': Timestamp(1767387291, 1)}" ] }, - "execution_count": 7, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -137,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 5, "id": "f0baa8a5-8625-4d41-a2ab-13404b13f3cd", "metadata": {}, "outputs": [ @@ -156,6 +183,18 @@ "anthropic_client = anthropic.Anthropic()" ] }, + { + "cell_type": "code", + "execution_count": 17, + "id": "b252c917-c714-4e6b-bd55-a420161bb96f", + "metadata": {}, + "outputs": [], + "source": [ + "# Make ffmpeg accessible from the notebook\n", + "# Replace /path/to/ffmpeg with your ffmpeg path\n", + "os.environ[\"PATH\"] = f\"/path/to/ffmpeg:{os.environ['PATH']}\"" + ] + }, { "cell_type": "markdown", "id": "c58c9820-c53b-49e5-8023-b5234cf7d817", @@ -166,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "id": "6b1cef44-0d97-44fb-878e-7e3872c0d8fe", "metadata": {}, "outputs": [], @@ -176,14 +215,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 40, "id": "d698eba9-fa77-4714-b7d5-558aecd9e93d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b775014dbb5248edadc4f237dd3ca3b4", + "model_id": "a2d76df25d2541088516b066b8105aad", "version_major": 2, "version_minor": 0 }, @@ -197,12 +236,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "25c2c5535b8e410ebac1c7b6d908ece1", + "model_id": "47fc0f8b4e004f2ba95bbbd8f5cbde5b", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Fetching 18 files: 0%| | 0/18 [00:00 Date: Fri, 2 Jan 2026 23:06:42 +0000 Subject: [PATCH 3/8] Formatting timestamps --- .../adaptive_video_search.ipynb | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/notebooks/advanced_techniques/adaptive_video_search.ipynb b/notebooks/advanced_techniques/adaptive_video_search.ipynb index 093160d..5a0ffd7 100644 --- a/notebooks/advanced_techniques/adaptive_video_search.ipynb +++ b/notebooks/advanced_techniques/adaptive_video_search.ipynb @@ -648,7 +648,20 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 162, + "id": "f7af9203-cc46-4361-bedd-bed3cc1d87d2", + "metadata": {}, + "outputs": [], + "source": [ + "def format_time(seconds):\n", + " mins = int(seconds // 60)\n", + " secs = int(seconds % 60)\n", + " return f\"{mins}:{secs:02d}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 164, "id": "360aa134-ca0f-4a21-84ba-0aa4edc12692", "metadata": {}, "outputs": [], @@ -679,13 +692,13 @@ " results = collection.aggregate(pipeline)\n", " for result in results:\n", " print(\n", - " f\"{result.get('video_title')} ({result.get('start')} to {result.get('end')})\"\n", + " f\"{result.get('video_title')} ({format_time(result.get('start'))} - {format_time(result.get('end'))})\"\n", " )" ] }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 166, "id": "765fb1f3-a410-4ea6-89b4-689b57c83e15", "metadata": {}, "outputs": [], @@ -741,13 +754,13 @@ " results = collection.aggregate(pipeline)\n", " for result in results:\n", " print(\n", - " f\"{result.get('video_title')} ({result.get('start')} to {result.get('end')})\"\n", + " f\"{result.get('video_title')} ({format_time(result.get('start'))} - {format_time(result.get('end'))})\"\n", " )" ] }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 167, "id": "17923831-57bf-4aed-9ee1-ed8196ddaa8c", "metadata": {}, "outputs": [ @@ -755,9 +768,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Classic French Croissants with Chef Marguerite Dubois (24 to 37)\n", - "Classic French Croissants with Chef Marguerite Dubois (59 to 61)\n", - "Classic French Croissants with Chef Marguerite Dubois (0 to 7)\n" + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:59 - 1:01)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:00 - 0:07)\n" ] } ], @@ -767,7 +780,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 168, "id": "3490f293-0b1d-4a66-82f3-30f9f443318f", "metadata": {}, "outputs": [ @@ -775,9 +788,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Artisan Sourdough Bread Folding Technique (10 to 18)\n", - "Classic French Croissants with Chef Marguerite Dubois (24 to 37)\n", - "Artisan Sourdough Bread Folding Technique (19 to 20)\n" + "Artisan Sourdough Bread Folding Technique (0:10 - 0:18)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", + "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n" ] } ], @@ -884,7 +897,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 169, "id": "19357309-57ec-42b1-896e-b45bb147661d", "metadata": {}, "outputs": [ @@ -893,9 +906,9 @@ "output_type": "stream", "text": [ "Using search type: vector\n", - "Classic French Croissants with Chef Marguerite Dubois (24 to 37)\n", - "Classic French Croissants with Chef Marguerite Dubois (59 to 61)\n", - "Classic French Croissants with Chef Marguerite Dubois (0 to 7)\n" + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:59 - 1:01)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:00 - 0:07)\n" ] } ], @@ -905,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 170, "id": "0da06647-0fb9-4d6a-a7af-a9ef9cea32a0", "metadata": {}, "outputs": [ @@ -914,9 +927,9 @@ "output_type": "stream", "text": [ "Using search type: hybrid\n", - "Artisan Sourdough Bread Folding Technique (10 to 18)\n", - "Classic French Croissants with Chef Marguerite Dubois (24 to 37)\n", - "Artisan Sourdough Bread Folding Technique (19 to 20)\n" + "Artisan Sourdough Bread Folding Technique (0:10 - 0:18)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", + "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n" ] } ], From 7bc70dc37e962ecefd644f5dfb32e47b665b4a17 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Mon, 5 Jan 2026 17:59:04 +0000 Subject: [PATCH 4/8] Adding comments --- ...earch.ipynb => agentic_video_search.ipynb} | 153 +++++++++++------- 1 file changed, 97 insertions(+), 56 deletions(-) rename notebooks/advanced_techniques/{adaptive_video_search.ipynb => agentic_video_search.ipynb} (87%) diff --git a/notebooks/advanced_techniques/adaptive_video_search.ipynb b/notebooks/advanced_techniques/agentic_video_search.ipynb similarity index 87% rename from notebooks/advanced_techniques/adaptive_video_search.ipynb rename to notebooks/advanced_techniques/agentic_video_search.ipynb index 5a0ffd7..57c700e 100644 --- a/notebooks/advanced_techniques/adaptive_video_search.ipynb +++ b/notebooks/advanced_techniques/agentic_video_search.ipynb @@ -5,9 +5,9 @@ "id": "3743e5aa-5a08-4b37-a545-4e7461e14468", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/advanced_techniques/adaptive_video_search.ipynb)\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mongodb-developer/GenAI-Showcase/blob/main/notebooks/advanced_techniques/agentic_video_search.ipynb)\n", "\n", - "[![View Article](https://img.shields.io/badge/View%20Article-blue)](https://www.mongodb.com/company/blog/technical/adaptive-video-search/?utm_campaign=devrel&utm_source=cross-post&utm_medium=organic_social&utm_content=https%3A%2F%2Fgithub.com%2Fmongodb-developer%2FGenAI-Showcase&utm_term=apoorva.joshi)" + "[![View Article](https://img.shields.io/badge/View%20Article-blue)](https://www.mongodb.com/company/blog/technical/agentic-video-search/?utm_campaign=devrel&utm_source=cross-post&utm_medium=organic_social&utm_content=https%3A%2F%2Fgithub.com%2Fmongodb-developer%2FGenAI-Showcase&utm_term=apoorva.joshi)" ] }, { @@ -15,7 +15,7 @@ "id": "becc445d-5a51-4a8a-abeb-ceab0b54c167", "metadata": {}, "source": [ - "# Building an Adaptive Video Search System using Voyage AI and MongoDB" + "# Building an Agentic Video Search System using Voyage AI and MongoDB" ] }, { @@ -48,7 +48,7 @@ "id": "d4803c8c-eb59-44aa-9306-5e49088add6b", "metadata": {}, "source": [ - "You'll also need to install `ffmpeg` binary itself. To do this, run the following commands from the terminal:\n", + "You'll also need to install the `ffmpeg` binary itself. To do this, run the following commands from the terminal and note the path to the `ffmpeg` installation:\n", "\n", "#### MacOS\n", "\n", @@ -104,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 171, "id": "d05bb72f-bebc-4255-8206-7a3b70b3d302", "metadata": {}, "outputs": [ @@ -112,13 +112,13 @@ "name": "stdin", "output_type": "stream", "text": [ - "Enter your VoyageAI API key: ········\n" + "Enter your Voyage API key: ········\n" ] } ], "source": [ - "# Set Voyage AI API key as an environment variable\n", - "os.environ[\"VOYAGE_API_KEY\"] = getpass.getpass(\"Enter your VoyageAI API key:\")\n", + "# Set Voyage API key as an environment variable\n", + "os.environ[\"VOYAGE_API_KEY\"] = getpass.getpass(\"Enter your Voyage API key:\")\n", "# Initialize the Voyage AI client\n", "voyage_client = voyageai.Client()" ] @@ -156,7 +156,7 @@ "MONGODB_URI = getpass.getpass(\"Enter your MongoDB connection string:\")\n", "# Initialize the MongoDB client\n", "mongodb_client = MongoClient(\n", - " MONGODB_URI, appname=\"devrel.showcase.adaptive_video_search\"\n", + " MONGODB_URI, appname=\"devrel.showcase.agentic_video_search\"\n", ")\n", "# Check MongoDB connection\n", "mongodb_client.admin.command(\"ping\")" @@ -205,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 172, "id": "6b1cef44-0d97-44fb-878e-7e3872c0d8fe", "metadata": {}, "outputs": [], @@ -215,14 +215,14 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 173, "id": "d698eba9-fa77-4714-b7d5-558aecd9e93d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a2d76df25d2541088516b066b8105aad", + "model_id": "5153fe076e7c460eb032a40783accb03", "version_major": 2, "version_minor": 0 }, @@ -236,7 +236,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "47fc0f8b4e004f2ba95bbbd8f5cbde5b", + "model_id": "71591b99eceb486b808022f317715839", "version_major": 2, "version_minor": 0 }, @@ -331,9 +331,10 @@ " captions = data[\"captions\"]\n", " title = data[\"title\"]\n", "\n", - " # Segment this video\n", + " # Segment the video based on captions\n", " for i, caption in enumerate(captions):\n", " segment_id = f\"segment_{i:03d}\"\n", + " # Create segment\n", " output_file = os.path.join(segments_dir, f\"{video_id}_{segment_id}.mp4\")\n", " (\n", " ffmpeg.input(video_path, ss=caption[\"start\"], to=caption[\"end\"])\n", @@ -341,7 +342,7 @@ " .overwrite_output()\n", " .run(quiet=True)\n", " )\n", - " # Create document to write to MongoDB\n", + " # Create segment document to write to MongoDB\n", " doc = {\n", " \"segment_id\": segment_id,\n", " \"video_id\": video_id,\n", @@ -378,7 +379,7 @@ } ], "source": [ - "# Preview a MongoDB doc\n", + "# Preview a segment doc\n", "docs[0]" ] }, @@ -413,12 +414,22 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 189, "id": "5874fbd5-5340-4d26-ae1d-44e20b4ff8f6", "metadata": {}, "outputs": [], "source": [ - "def generate_embedding(inputs, input_type):\n", + "def generate_embeddings(inputs: list[list], input_type: str) -> list[list]:\n", + " \"\"\"\n", + " Generate embeddings using Voyage AI's latest multimodal embedding model.\n", + "\n", + " Args:\n", + " inputs (list[list]): Inputs as a list of lists\n", + " input_type (str): Type of input. Can be one of \"document\" or \"query\"\n", + "\n", + " Returns:\n", + " list[list]: List of embeddings\n", + " \"\"\"\n", " embeddings = voyage_client.multimodal_embed(\n", " inputs=inputs, model=MODEL_NAME, input_type=input_type\n", " ).embeddings\n", @@ -463,7 +474,9 @@ " path=f\"{segments_dir}/{doc['video_id']}_{doc['segment_id']}.mp4\",\n", " model=MODEL_NAME,\n", " )\n", - " embeddings = generate_embedding([[video_obj, doc[\"caption\"]]], \"document\")\n", + " # Embed the video segment and its caption together\n", + " embeddings = generate_embeddings([[video_obj, doc[\"caption\"]]], \"document\")\n", + " # Add the embedding to the MongoDB document\n", " doc[\"embedding\"] = embeddings[0]" ] }, @@ -586,6 +599,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Full-text search index definition\n", "fts_model = SearchIndexModel(\n", " name=\"fts-index\",\n", " definition={\n", @@ -601,6 +615,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Vector search index definition\n", "vs_model = SearchIndexModel(\n", " name=\"vector-index\",\n", " type=\"vectorSearch\",\n", @@ -653,7 +668,16 @@ "metadata": {}, "outputs": [], "source": [ - "def format_time(seconds):\n", + "def format_time(seconds: int) -> str:\n", + " \"\"\"\n", + " Format a second timestamp as min:sec.\n", + "\n", + " Args:\n", + " seconds (int): Time in seconds\n", + "\n", + " Returns:\n", + " str: Formatted timestamp\n", + " \"\"\"\n", " mins = int(seconds // 60)\n", " secs = int(seconds % 60)\n", " return f\"{mins}:{secs:02d}\"" @@ -661,13 +685,19 @@ }, { "cell_type": "code", - "execution_count": 164, + "execution_count": 194, "id": "360aa134-ca0f-4a21-84ba-0aa4edc12692", "metadata": {}, "outputs": [], "source": [ - "def vector_search(query):\n", - " query_embedding = generate_embedding([[query]], \"query\")[0]\n", + "def vector_search(query: str) -> None:\n", + " \"\"\"\n", + " Retrieve relevant video segments using vector search.\n", + "\n", + " Args:\n", + " query (str): User query string\n", + " \"\"\"\n", + " query_embedding = generate_embeddings([[query]], \"query\")[0]\n", " pipeline = [\n", " {\n", " \"$vectorSearch\": {\n", @@ -698,13 +728,19 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 195, "id": "765fb1f3-a410-4ea6-89b4-689b57c83e15", "metadata": {}, "outputs": [], "source": [ - "def hybrid_search(query):\n", - " query_embedding = generate_embedding([[query]], \"query\")[0]\n", + "def hybrid_search(query: str) -> None:\n", + " \"\"\"\n", + " Retrieve relevant video segments using hybrid search.\n", + "\n", + " Args:\n", + " query (str): User query string\n", + " \"\"\"\n", + " query_embedding = generate_embeddings([[query]], \"query\")[0]\n", " pipeline = [\n", " {\n", " \"$rankFusion\": {\n", @@ -760,7 +796,7 @@ }, { "cell_type": "code", - "execution_count": 167, + "execution_count": 196, "id": "17923831-57bf-4aed-9ee1-ed8196ddaa8c", "metadata": {}, "outputs": [ @@ -780,7 +816,7 @@ }, { "cell_type": "code", - "execution_count": 168, + "execution_count": 197, "id": "3490f293-0b1d-4a66-82f3-30f9f443318f", "metadata": {}, "outputs": [ @@ -803,7 +839,7 @@ "id": "61a57c6f-7981-48b7-a0c8-0e5306241ecb", "metadata": {}, "source": [ - "## Step 9: Building the Adaptive Search Pipeline" + "## Step 9: Building the Agentic Search Pipeline" ] }, { @@ -813,6 +849,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Define structured output schema\n", "output_schema = {\n", " \"type\": \"object\",\n", " \"properties\": {\"search\": {\"type\": \"string\", \"enum\": [\"vector\", \"hybrid\"]}},\n", @@ -844,12 +881,22 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 182, "id": "93fc66bb-ae15-4c4b-a868-7bad3bc3426c", "metadata": {}, "outputs": [], "source": [ - "def get_search_type(query):\n", + "def get_search_type(query: str) -> str:\n", + " \"\"\"\n", + " Use an LLM to determine the search strategy based on the query.\n", + "\n", + " Args:\n", + " query (str): User query string\n", + "\n", + " Returns:\n", + " str: Search type. One of \"vector\" or \"hybrid\"\n", + " \"\"\"\n", + " print(\"Determining search type...\")\n", " response = anthropic_client.beta.messages.create(\n", " model=\"claude-sonnet-4-5\",\n", " max_tokens=50,\n", @@ -859,45 +906,37 @@ " messages=[{\"role\": \"user\", \"content\": f\"Query: {query}\"}],\n", " output_format={\"type\": \"json_schema\", \"schema\": output_schema},\n", " )\n", - "\n", - " return json.loads(response.content[0].text)[\"search\"]" + " search_type = json.loads(response.content[0].text).get(\"search\", \"unknown\")\n", + " print(f\"Using search type: {search_type}\")\n", + " return search_type" ] }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 183, "id": "b92e7a4d-fb05-4838-bc42-2b00a21fe1da", "metadata": {}, "outputs": [], "source": [ - "def execute_search(search_type, query):\n", - " results = []\n", + "def search(query: str) -> None:\n", + " \"\"\"\n", + " Given a query, determine the search type and execute the search.\n", + "\n", + " Args:\n", + " query (str): User quqery string\n", + " \"\"\"\n", + " search_type = get_search_type(query)\n", " if search_type == \"vector\":\n", - " results = vector_search(query)\n", + " vector_search(query)\n", " elif search_type == \"hybrid\":\n", - " results = hybrid_search(query)\n", + " hybrid_search(query)\n", " else:\n", - " print(\"Search type not supported.\")\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": 154, - "id": "91b2b465-38a2-4542-80cc-de6140d778d9", - "metadata": {}, - "outputs": [], - "source": [ - "def search(query):\n", - " search_type = get_search_type(query)\n", - " print(f\"Using search type: {search_type}\")\n", - " results = execute_search(search_type, query)\n", - " return results" + " print(f\"Not a supported search type: {search_type}\")" ] }, { "cell_type": "code", - "execution_count": 169, + "execution_count": 184, "id": "19357309-57ec-42b1-896e-b45bb147661d", "metadata": {}, "outputs": [ @@ -905,6 +944,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Determining search type...\n", "Using search type: vector\n", "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", "Classic French Croissants with Chef Marguerite Dubois (0:59 - 1:01)\n", @@ -918,7 +958,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 185, "id": "0da06647-0fb9-4d6a-a7af-a9ef9cea32a0", "metadata": {}, "outputs": [ @@ -926,6 +966,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Determining search type...\n", "Using search type: hybrid\n", "Artisan Sourdough Bread Folding Technique (0:10 - 0:18)\n", "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", From 77bb3b11a7c0ac3ae654e50b9019d3d2a5d3523f Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Tue, 6 Jan 2026 00:24:54 +0000 Subject: [PATCH 5/8] Adding comment --- notebooks/advanced_techniques/agentic_video_search.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notebooks/advanced_techniques/agentic_video_search.ipynb b/notebooks/advanced_techniques/agentic_video_search.ipynb index 57c700e..f6c6202 100644 --- a/notebooks/advanced_techniques/agentic_video_search.ipynb +++ b/notebooks/advanced_techniques/agentic_video_search.ipynb @@ -261,7 +261,9 @@ "id": "5c33737d-30bb-4d41-add8-01e4fcd32b61", "metadata": {}, "source": [ - "## Step 4: Segment the videos using captions" + "## Step 4: Segment the videos using captions\n", + "\n", + "The `voyage-multimodal-3.5` model has a 32k token limit or a 20 MB file size limit for video inputs. When working with large videos, we recommend splitting them into smaller segments prior to embedding. Splitting videos based on natural breaks in captions/transcripts ensures that related frames stay together, resulting in more focused embeddings." ] }, { From 2a06959677f2ced07c09c206ef48cfd2fefce872 Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Tue, 6 Jan 2026 00:39:59 +0000 Subject: [PATCH 6/8] Nitpicking --- notebooks/advanced_techniques/agentic_video_search.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/advanced_techniques/agentic_video_search.ipynb b/notebooks/advanced_techniques/agentic_video_search.ipynb index f6c6202..94ffb9d 100644 --- a/notebooks/advanced_techniques/agentic_video_search.ipynb +++ b/notebooks/advanced_techniques/agentic_video_search.ipynb @@ -263,7 +263,7 @@ "source": [ "## Step 4: Segment the videos using captions\n", "\n", - "The `voyage-multimodal-3.5` model has a 32k token limit or a 20 MB file size limit for video inputs. When working with large videos, we recommend splitting them into smaller segments prior to embedding. Splitting videos based on natural breaks in captions/transcripts ensures that related frames stay together, resulting in more focused embeddings." + "`voyage-multimodal-3.5` has a 32k token limit or a 20 MB file size limit for video inputs. When working with large videos, split them into smaller segments prior to embedding to keep them within the model’s limits. Splitting videos at natural breaks in captions/transcripts ensures that related frames remain together, resulting in more focused embeddings." ] }, { From 6a98cbf6e90fdcca76d9afa6ff2c6de80bb189ce Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Tue, 6 Jan 2026 20:56:09 +0000 Subject: [PATCH 7/8] Updating FTS --- .../agentic_video_search.ipynb | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/notebooks/advanced_techniques/agentic_video_search.ipynb b/notebooks/advanced_techniques/agentic_video_search.ipynb index 94ffb9d..67e21b5 100644 --- a/notebooks/advanced_techniques/agentic_video_search.ipynb +++ b/notebooks/advanced_techniques/agentic_video_search.ipynb @@ -730,7 +730,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 201, "id": "765fb1f3-a410-4ea6-89b4-689b57c83e15", "metadata": {}, "outputs": [], @@ -763,7 +763,7 @@ " {\n", " \"$search\": {\n", " \"index\": \"fts-index\",\n", - " \"phrase\": {\"query\": query, \"path\": \"caption\"},\n", + " \"text\": {\"query\": query, \"path\": \"caption\"},\n", " }\n", " },\n", " {\"$limit\": 10},\n", @@ -776,7 +776,6 @@ " \"scoreDetails\": True,\n", " }\n", " },\n", - " {\"$addFields\": {\"scoreDetails\": {\"$meta\": \"scoreDetails\"}}},\n", " {\n", " \"$project\": {\n", " \"_id\": 0,\n", @@ -818,7 +817,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 202, "id": "3490f293-0b1d-4a66-82f3-30f9f443318f", "metadata": {}, "outputs": [ @@ -827,8 +826,8 @@ "output_type": "stream", "text": [ "Artisan Sourdough Bread Folding Technique (0:10 - 0:18)\n", - "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", - "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n" + "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n" ] } ], @@ -960,7 +959,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 203, "id": "0da06647-0fb9-4d6a-a7af-a9ef9cea32a0", "metadata": {}, "outputs": [ @@ -971,8 +970,8 @@ "Determining search type...\n", "Using search type: hybrid\n", "Artisan Sourdough Bread Folding Technique (0:10 - 0:18)\n", - "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n", - "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n" + "Artisan Sourdough Bread Folding Technique (0:19 - 0:20)\n", + "Classic French Croissants with Chef Marguerite Dubois (0:24 - 0:37)\n" ] } ], From 670e3fed01d05075ba7828c28fa3e303603ec9fa Mon Sep 17 00:00:00 2001 From: ajosh0504 Date: Wed, 7 Jan 2026 00:09:14 +0000 Subject: [PATCH 8/8] Nit --- notebooks/advanced_techniques/agentic_video_search.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/notebooks/advanced_techniques/agentic_video_search.ipynb b/notebooks/advanced_techniques/agentic_video_search.ipynb index 67e21b5..6c4c50d 100644 --- a/notebooks/advanced_techniques/agentic_video_search.ipynb +++ b/notebooks/advanced_techniques/agentic_video_search.ipynb @@ -871,7 +871,6 @@ "vector\n", "- Best for: Visual actions and details, methods, concepts or general descriptions.\n", "- Examples: \"How to chop onions\", \"Grilling vegetables\"\n", - "- Uses: Multimodal embeddings that capture both video and caption meaning\n", "\n", "hybrid\n", "- Best for: Specific names and terms such as techniques, chef names, dietary restrictions etc.\n",