Added Effective search ratio

backend/example.env

@@ -42,4 +42,5 @@ LLM_MODEL_CONFIG_fireworks_llama_v3_70b="model_name,fireworks_api_key"
 LLM_MODEL_CONFIG_bedrock_claude_3_5_sonnet="model_name,aws_access_key_id,aws_secret__access_key,region_name"
 LLM_MODEL_CONFIG_ollama_llama3="model_name,model_local_url"
 YOUTUBE_TRANSCRIPT_PROXY="https://user:pass@domain:port"
+EFFECTIVE_SEARCH_RATIO=5
 

backend/requirements.txt

@@ -1,6 +1,6 @@
 asyncio==3.4.3
-boto3==1.35.69
-botocore==1.35.69
+boto3==1.35.90
+botocore==1.35.90
 certifi==2024.8.30
 fastapi==0.115.6
 fastapi-health==0.4.0
@@ -10,24 +10,24 @@ google_auth_oauthlib==1.2.1
 google-cloud-core==2.4.1
 json-repair==0.30.2
 pip-install==1.3.5
-langchain==0.3.8
-langchain-aws==0.2.7
+langchain==0.3.13
+langchain-aws==0.2.10
 langchain-anthropic==0.3.0
 langchain-fireworks==0.2.5
-langchain-community==0.3.8
-langchain-core==0.3.21
-langchain-experimental==0.3.3
+langchain-community==0.3.13
+langchain-core==0.3.28
+langchain-experimental==0.3.4
 langchain-google-vertexai==2.0.7
 langchain-groq==0.2.1
-langchain-openai==0.2.9
-langchain-text-splitters==0.3.2
+langchain-openai==0.2.14
+langchain-text-splitters==0.3.4
 langchain-huggingface==0.1.2
 langdetect==1.0.9
-langsmith==0.1.146
+langsmith==0.2.4
 langserve==0.3.0
 neo4j-rust-ext
 nltk==3.9.1
-openai==1.55.1
+openai==1.58.1
 opencv-python==4.10.0.84
 psutil==6.1.0
 pydantic==2.9.2
@@ -58,4 +58,4 @@ graphdatascience==1.12
 Secweb==1.11.0
 ragas==0.2.6
 rouge_score==0.1.2
-langchain-neo4j==0.1.1
+langchain-neo4j==0.2.0

backend/src/QA_integration.py

@@ -364,12 +364,13 @@ def initialize_neo4j_vector(graph, chat_mode_settings):
         raise
     return neo_db
 
-def create_retriever(neo_db, document_names, chat_mode_settings,search_k, score_threshold):
+def create_retriever(neo_db, document_names, chat_mode_settings,search_k, score_threshold,ef_ratio):
     if document_names and chat_mode_settings["document_filter"]:
         retriever = neo_db.as_retriever(
             search_type="similarity_score_threshold",
             search_kwargs={
                 'k': search_k,
+                'effective_search_ratio': ef_ratio,
                 'score_threshold': score_threshold,
                 'filter': {'fileName': {'$in': document_names}}
             }
@@ -378,7 +379,7 @@ def create_retriever(neo_db, document_names, chat_mode_settings,search_k, score_
     else:
         retriever = neo_db.as_retriever(
             search_type="similarity_score_threshold",
-            search_kwargs={'k': search_k, 'score_threshold': score_threshold}
+            search_kwargs={'k': search_k,'effective_search_ratio': ef_ratio, 'score_threshold': score_threshold}
         )
         logging.info(f"Successfully created retriever with search_k={search_k}, score_threshold={score_threshold}")
     return retriever
@@ -389,7 +390,8 @@ def get_neo4j_retriever(graph, document_names,chat_mode_settings, score_threshol
         neo_db = initialize_neo4j_vector(graph, chat_mode_settings)
         # document_names= list(map(str.strip, json.loads(document_names)))
         search_k = chat_mode_settings["top_k"]
-        retriever = create_retriever(neo_db, document_names,chat_mode_settings, search_k, score_threshold)
+        ef_ratio = int(os.getenv("EFFECTIVE_SEARCH_RATIO", "2")) if os.getenv("EFFECTIVE_SEARCH_RATIO", "2").isdigit() else 2
+        retriever = create_retriever(neo_db, document_names,chat_mode_settings, search_k, score_threshold,ef_ratio)
         return retriever
     except Exception as e:
         index_name = chat_mode_settings.get("index_name")