reduce default chunksize for reactome embedding (#93)

GFJHogue · web-flow · commit e398a3714565 · 2025-09-27T13:09:26.000-04:00
diff --git a/src/data_generation/reactome/__init__.py b/src/data_generation/reactome/__init__.py
@@ -53,11 +53,13 @@ def upload_to_chromadb(
     embeddings_instance: Embeddings
     if hf_model is None:  # Use OpenAI
         embeddings_instance = OpenAIEmbeddings(
+            chunk_size=500,
             show_progress_bar=True,
         )
     elif hf_model.startswith("openai/text-embedding-"):
         embeddings_instance = OpenAIEmbeddings(
             model=hf_model[len("openai/") :],
+            chunk_size=500,
             show_progress_bar=True,
         )
     elif "HUGGINGFACEHUB_API_TOKEN" in os.environ:

Original file line number	Diff line number	Diff line change
`@@ -53,11 +53,13 @@ def upload_to_chromadb(`
`53`	`53`	`embeddings_instance: Embeddings`
`54`	`54`	`if hf_model is None: # Use OpenAI`
`55`	`55`	`embeddings_instance = OpenAIEmbeddings(`
	`56`	`+ chunk_size=500,`
`56`	`57`	`show_progress_bar=True,`
`57`	`58`	`)`
`58`	`59`	`elif hf_model.startswith("openai/text-embedding-"):`
`59`	`60`	`embeddings_instance = OpenAIEmbeddings(`
`60`	`61`	`model=hf_model[len("openai/") :],`
	`62`	`+ chunk_size=500,`
`61`	`63`	`show_progress_bar=True,`
`62`	`64`	`)`
`63`	`65`	`elif "HUGGINGFACEHUB_API_TOKEN" in os.environ:`