diff --git a/README.md b/README.md index 800721ec3..e0b59346d 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Allow unauthenticated request : Yes | VITE_LLM_MODELS | Mandatory | diffbot,openai-gpt-3.5,openai-gpt-4o | Models available for selection on the frontend, used for entities extraction and Q&A | VITE_CHAT_MODES | Mandatory | vector,graph+vector,graph,hybrid | Chat modes available for Q&A | VITE_ENV | Mandatory | DEV or PROD | Environment variable for the app | -| VITE_TIME_PER_CHUNK | Optional | 4 | Time per chunk for processing | +| VITE_TIME_PER_PAGE | Optional | 50 | Time per page for processing | | VITE_CHUNK_SIZE | Optional | 5242880 | Size of each chunk of file for upload | | VITE_GOOGLE_CLIENT_ID | Optional | | Client ID for Google authentication | | GCS_FILE_CACHE | Optional | False | If set to True, will save the files to process into GCS. If set to False, will save the files locally | diff --git a/backend/Performance_test.py b/backend/Performance_test.py index fc0aee66f..712d3daf1 100644 --- a/backend/Performance_test.py +++ b/backend/Performance_test.py @@ -94,6 +94,7 @@ def performance_main(): for _ in range(CONCURRENT_REQUESTS): futures.append(executor.submit(post_request_chunk)) + # Chatbot request futures # Chatbot request futures # for message in CHATBOT_MESSAGES: # futures.append(executor.submit(chatbot_request, message)) diff --git a/backend/requirements.txt b/backend/requirements.txt index ab42a749d..46c57aea5 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -69,18 +69,18 @@ jsonpath-python==1.0.6 jsonpointer==2.4 json-repair==0.25.2 kiwisolver==1.4.5 -langchain==0.2.8 -langchain-aws==0.1.9 -langchain-anthropic==0.1.19 -langchain-fireworks==0.1.4 -langchain-google-genai==1.0.7 -langchain-community==0.2.7 -langchain-core==0.2.19 -langchain-experimental==0.0.62 -langchain-google-vertexai==1.0.6 -langchain-groq==0.1.6 -langchain-openai==0.1.14 -langchain-text-splitters==0.2.2 +langchain +langchain-aws +langchain-anthropic +langchain-fireworks +langchain-google-genai +langchain-community +langchain-core +langchain-experimental +langchain-google-vertexai +langchain-groq +langchain-openai +langchain-text-splitters langdetect==1.0.9 langsmith==0.1.83 layoutparser==0.3.4 diff --git a/backend/score.py b/backend/score.py index 97274ecca..00650b56e 100644 --- a/backend/score.py +++ b/backend/score.py @@ -105,8 +105,8 @@ async def create_source_knowledge_graph_url( return create_api_response('Failed',message='source_type is other than accepted source') message = f"Source Node created successfully for source type: {source_type} and source: {source}" - josn_obj = {'api_name':'url_scan','db_url':uri,'url_scanned_file':lst_file_name, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'url_scan','db_url':uri,'url_scanned_file':lst_file_name, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response("Success",message=message,success_count=success_count,failed_count=failed_count,file_name=lst_file_name) except Exception as e: error_message = str(e) @@ -208,9 +208,9 @@ async def extract_knowledge_graph_from_file( else: logging.info(f'Deleted File Path: {merged_file_path} and Deleted File Name : {file_name}') delete_uploaded_local_file(merged_file_path,file_name) - josn_obj = {'message':message,'error_message':error_message, 'file_name': file_name,'status':'Failed','db_url':uri,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) - logging.exception(f'File Failed in extraction: {josn_obj}') + json_obj = {'message':message,'error_message':error_message, 'file_name': file_name,'status':'Failed','db_url':uri,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) + logging.exception(f'File Failed in extraction: {json_obj}') return create_api_response('Failed', message=message + error_message[:100], error=error_message, file_name = file_name) finally: gc.collect() @@ -225,8 +225,8 @@ async def get_source_list(uri:str, userName:str, password:str, database:str=None if " " in uri: uri = uri.replace(" ","+") result = await asyncio.to_thread(get_source_list_from_graph,uri,userName,decoded_password,database) - josn_obj = {'api_name':'sources_list','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'sources_list','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response("Success",data=result) except Exception as e: job_status = "Failed" @@ -243,19 +243,20 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database if "materialize_text_chunk_similarities" in tasks: await asyncio.to_thread(update_graph, graph) - josn_obj = {'api_name': 'post_processing/materialize_text_chunk_similarities', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name': 'post_processing/update_similarity_graph', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) logging.info(f'Updated KNN Graph') + if "enable_hybrid_search_and_fulltext_search_in_bloom" in tasks: await asyncio.to_thread(create_fulltext, uri=uri, username=userName, password=password, database=database,type="entities") - await asyncio.to_thread(create_fulltext, uri=uri, username=userName, password=password, database=database,type="keyword") + # await asyncio.to_thread(create_fulltext, uri=uri, username=userName, password=password, database=database,type="keyword") josn_obj = {'api_name': 'post_processing/enable_hybrid_search_and_fulltext_search_in_bloom', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} logger.log_struct(josn_obj) logging.info(f'Full Text index created') if os.environ.get('ENTITY_EMBEDDING','False').upper()=="TRUE" and "materialize_entity_similarities" in tasks: await asyncio.to_thread(create_entity_embedding, graph) - josn_obj = {'api_name': 'post_processing/materialize_entity_similarities', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name': 'post_processing/create_entity_embedding', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) logging.info(f'Entity Embeddings created') return create_api_response('Success', message='All tasks completed successfully') @@ -284,8 +285,8 @@ async def chat_bot(uri=Form(),model=Form(None),userName=Form(), password=Form(), logging.info(f"Total Response time is {total_call_time:.2f} seconds") result["info"]["response_time"] = round(total_call_time, 2) - josn_obj = {'api_name':'chat_bot','db_url':uri,'session_id':session_id, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'chat_bot','db_url':uri,'session_id':session_id, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response('Success',data=result) except Exception as e: job_status = "Failed" @@ -301,8 +302,8 @@ async def chunk_entities(uri=Form(),userName=Form(), password=Form(), chunk_ids= try: logging.info(f"URI: {uri}, Username: {userName}, chunk_ids: {chunk_ids}") result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, chunk_ids=chunk_ids) - josn_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response('Success',data=result) except Exception as e: job_status = "Failed" @@ -329,8 +330,8 @@ async def graph_query( password=password, document_names=document_names ) - josn_obj = {'api_name':'graph_query','db_url':uri,'document_names':document_names, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'graph_query','db_url':uri,'document_names':document_names, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response('Success', data=result) except Exception as e: job_status = "Failed" @@ -379,8 +380,8 @@ async def upload_large_file_into_chunks(file:UploadFile = File(...), chunkNumber try: graph = create_graph_database_connection(uri, userName, password, database) result = await asyncio.to_thread(upload_file, graph, model, file, chunkNumber, totalChunks, originalname, uri, CHUNK_DIR, MERGED_DIR) - josn_obj = {'api_name':'upload','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'upload','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) if int(chunkNumber) == int(totalChunks): return create_api_response('Success',data=result, message='Source Node Created Successfully') else: @@ -401,8 +402,8 @@ async def get_structured_schema(uri=Form(), userName=Form(), password=Form(), da graph = create_graph_database_connection(uri, userName, password, database) result = await asyncio.to_thread(get_labels_and_relationtypes, graph) logging.info(f'Schema result from DB: {result}') - josn_obj = {'api_name':'schema','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'schema','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response('Success', data=result) except Exception as e: message="Unable to get the labels and relationtypes from neo4j database" @@ -470,8 +471,8 @@ async def delete_document_and_entities(uri=Form(), result, files_list_size = await asyncio.to_thread(graphDb_data_Access.delete_file_from_graph, filenames, source_types, deleteEntities, MERGED_DIR, uri) # entities_count = result[0]['deletedEntities'] if 'deletedEntities' in result[0] else 0 message = f"Deleted {files_list_size} documents with entities from database" - josn_obj = {'api_name':'delete_document_and_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} - logger.log_struct(josn_obj) + json_obj = {'api_name':'delete_document_and_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} + logger.log_struct(json_obj) return create_api_response('Success',message=message) except Exception as e: job_status = "Failed" @@ -627,4 +628,4 @@ async def merge_duplicate_nodes(uri=Form(), userName=Form(), password=Form(), da gc.collect() if __name__ == "__main__": - uvicorn.run(app) \ No newline at end of file + uvicorn.run(app) diff --git a/backend/src/QA_integration_new.py b/backend/src/QA_integration_new.py index 1c0bc254c..eeac78c1e 100644 --- a/backend/src/QA_integration_new.py +++ b/backend/src/QA_integration_new.py @@ -41,26 +41,26 @@ def get_neo4j_retriever(graph, retrieval_query,document_names,mode,index_name="vector",keyword_index="keyword", search_k=CHAT_SEARCH_KWARG_K, score_threshold=CHAT_SEARCH_KWARG_SCORE_THRESHOLD): try: - if mode == "hybrid": - # neo_db = Neo4jVector.from_existing_graph( - # embedding=EMBEDDING_FUNCTION, - # index_name=index_name, - # retrieval_query=retrieval_query, - # graph=graph, - # search_type="hybrid", - # node_label="Chunk", - # embedding_node_property="embedding", - # text_node_properties=["text"] - # # keyword_index_name=keyword_index - # ) - neo_db = Neo4jVector.from_existing_index( + if mode == "fulltext" or mode == "graph + vector + fulltext": + neo_db = Neo4jVector.from_existing_graph( embedding=EMBEDDING_FUNCTION, index_name=index_name, retrieval_query=retrieval_query, graph=graph, search_type="hybrid", + node_label="Chunk", + embedding_node_property="embedding", + text_node_properties=["text"], keyword_index_name=keyword_index ) + # neo_db = Neo4jVector.from_existing_index( + # embedding=EMBEDDING_FUNCTION, + # index_name=index_name, + # retrieval_query=retrieval_query, + # graph=graph, + # search_type="hybrid", + # keyword_index_name=keyword_index + # ) logging.info(f"Successfully retrieved Neo4jVector index '{index_name}' and keyword index '{keyword_index}'") else: neo_db = Neo4jVector.from_existing_index( @@ -374,7 +374,7 @@ def QA_RAG(graph, model, question, document_names,session_id, mode): "user": "chatbot" } return result - elif mode == "vector" or mode == "hybrid": + elif mode == "vector" or mode == "fulltext": retrieval_query = VECTOR_SEARCH_QUERY else: retrieval_query = VECTOR_GRAPH_SEARCH_QUERY.format(no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT) diff --git a/backend/src/main.py b/backend/src/main.py index f7dd190ef..a7d5058a0 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -264,6 +264,7 @@ def processing_source(uri, userName, password, database, model, file_name, pages graphDb_data_Access = graphDBdataAccess(graph) result = graphDb_data_Access.get_current_status_document_node(file_name) + print(result) logging.info("Break down file into chunks") bad_chars = ['"', "\n", "'"] for i in range(0,len(pages)): @@ -277,91 +278,97 @@ def processing_source(uri, userName, password, database, model, file_name, pages create_chunks_obj = CreateChunksofDocument(pages, graph) chunks = create_chunks_obj.split_file_into_chunks() chunkId_chunkDoc_list = create_relation_between_chunks(graph,file_name,chunks) - if result[0]['Status'] != 'Processing': - obj_source_node = sourceNode() - status = "Processing" - obj_source_node.file_name = file_name - obj_source_node.status = status - obj_source_node.total_chunks = len(chunks) - obj_source_node.total_pages = len(pages) - obj_source_node.model = model - logging.info(file_name) - logging.info(obj_source_node) - graphDb_data_Access.update_source_node(obj_source_node) - - logging.info('Update the status as Processing') - update_graph_chunk_processed = int(os.environ.get('UPDATE_GRAPH_CHUNKS_PROCESSED')) - # selected_chunks = [] - is_cancelled_status = False - job_status = "Completed" - node_count = 0 - rel_count = 0 - for i in range(0, len(chunkId_chunkDoc_list), update_graph_chunk_processed): - select_chunks_upto = i+update_graph_chunk_processed - logging.info(f'Selected Chunks upto: {select_chunks_upto}') - if len(chunkId_chunkDoc_list) <= select_chunks_upto: - select_chunks_upto = len(chunkId_chunkDoc_list) - selected_chunks = chunkId_chunkDoc_list[i:select_chunks_upto] + + if len(result) > 0: + if result[0]['Status'] != 'Processing': + obj_source_node = sourceNode() + status = "Processing" + obj_source_node.file_name = file_name + obj_source_node.status = status + obj_source_node.total_chunks = len(chunks) + obj_source_node.total_pages = len(pages) + obj_source_node.model = model + logging.info(file_name) + logging.info(obj_source_node) + graphDb_data_Access.update_source_node(obj_source_node) + + logging.info('Update the status as Processing') + update_graph_chunk_processed = int(os.environ.get('UPDATE_GRAPH_CHUNKS_PROCESSED')) + # selected_chunks = [] + is_cancelled_status = False + job_status = "Completed" + node_count = 0 + rel_count = 0 + for i in range(0, len(chunkId_chunkDoc_list), update_graph_chunk_processed): + select_chunks_upto = i+update_graph_chunk_processed + logging.info(f'Selected Chunks upto: {select_chunks_upto}') + if len(chunkId_chunkDoc_list) <= select_chunks_upto: + select_chunks_upto = len(chunkId_chunkDoc_list) + selected_chunks = chunkId_chunkDoc_list[i:select_chunks_upto] + result = graphDb_data_Access.get_current_status_document_node(file_name) + is_cancelled_status = result[0]['is_cancelled'] + logging.info(f"Value of is_cancelled : {result[0]['is_cancelled']}") + if bool(is_cancelled_status) == True: + job_status = "Cancelled" + logging.info('Exit from running loop of processing file') + exit + else: + node_count,rel_count = processing_chunks(selected_chunks,graph,uri, userName, password, database,file_name,model,allowedNodes,allowedRelationship,node_count, rel_count) + end_time = datetime.now() + processed_time = end_time - start_time + + obj_source_node = sourceNode() + obj_source_node.file_name = file_name + obj_source_node.updated_at = end_time + obj_source_node.processing_time = processed_time + obj_source_node.node_count = node_count + obj_source_node.processed_chunk = select_chunks_upto + obj_source_node.relationship_count = rel_count + graphDb_data_Access.update_source_node(obj_source_node) + result = graphDb_data_Access.get_current_status_document_node(file_name) is_cancelled_status = result[0]['is_cancelled'] - logging.info(f"Value of is_cancelled : {result[0]['is_cancelled']}") if bool(is_cancelled_status) == True: - job_status = "Cancelled" - logging.info('Exit from running loop of processing file') - exit - else: - node_count,rel_count = processing_chunks(selected_chunks,graph,uri, userName, password, database,file_name,model,allowedNodes,allowedRelationship,node_count, rel_count) - end_time = datetime.now() - processed_time = end_time - start_time - - obj_source_node = sourceNode() - obj_source_node.file_name = file_name - obj_source_node.updated_at = end_time - obj_source_node.processing_time = processed_time - obj_source_node.node_count = node_count - obj_source_node.processed_chunk = select_chunks_upto - obj_source_node.relationship_count = rel_count - graphDb_data_Access.update_source_node(obj_source_node) - - result = graphDb_data_Access.get_current_status_document_node(file_name) - is_cancelled_status = result[0]['is_cancelled'] - if bool(is_cancelled_status) == True: - logging.info(f'Is_cancelled True at the end extraction') - job_status = 'Cancelled' - logging.info(f'Job Status at the end : {job_status}') - end_time = datetime.now() - processed_time = end_time - start_time - obj_source_node = sourceNode() - obj_source_node.file_name = file_name - obj_source_node.status = job_status - obj_source_node.processing_time = processed_time + logging.info(f'Is_cancelled True at the end extraction') + job_status = 'Cancelled' + logging.info(f'Job Status at the end : {job_status}') + end_time = datetime.now() + processed_time = end_time - start_time + obj_source_node = sourceNode() + obj_source_node.file_name = file_name + obj_source_node.status = job_status + obj_source_node.processing_time = processed_time - graphDb_data_Access.update_source_node(obj_source_node) - logging.info('Updated the nodeCount and relCount properties in Document node') - logging.info(f'file:{file_name} extraction has been completed') + graphDb_data_Access.update_source_node(obj_source_node) + logging.info('Updated the nodeCount and relCount properties in Document node') + logging.info(f'file:{file_name} extraction has been completed') - # merged_file_path have value only when file uploaded from local - - if is_uploaded_from_local: - gcs_file_cache = os.environ.get('GCS_FILE_CACHE') - if gcs_file_cache == 'True': - folder_name = create_gcs_bucket_folder_name_hashed(uri, file_name) - delete_file_from_gcs(BUCKET_UPLOAD,folder_name,file_name) - else: - delete_uploaded_local_file(merged_file_path, file_name) + # merged_file_path have value only when file uploaded from local - return { - "fileName": file_name, - "nodeCount": node_count, - "relationshipCount": rel_count, - "processingTime": round(processed_time.total_seconds(),2), - "status" : job_status, - "model" : model, - "success_count" : 1 - } + if is_uploaded_from_local: + gcs_file_cache = os.environ.get('GCS_FILE_CACHE') + if gcs_file_cache == 'True': + folder_name = create_gcs_bucket_folder_name_hashed(uri, file_name) + delete_file_from_gcs(BUCKET_UPLOAD,folder_name,file_name) + else: + delete_uploaded_local_file(merged_file_path, file_name) + + return { + "fileName": file_name, + "nodeCount": node_count, + "relationshipCount": rel_count, + "processingTime": round(processed_time.total_seconds(),2), + "status" : job_status, + "model" : model, + "success_count" : 1 + } + else: + logging.info('File does not process because it\'s already in Processing status') else: - logging.info('File does not process because it\'s already in Processing status') + error_message = "Unable to get the status of docuemnt node." + logging.error(error_message) + raise Exception(error_message) def processing_chunks(chunkId_chunkDoc_list,graph,uri, userName, password, database,file_name,model,allowedNodes,allowedRelationship, node_count, rel_count): #create vector index and update chunk node with embedding diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 903999a51..c5f8e98a4 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -276,4 +276,4 @@ RETURN text, avg_score as score, {{length:size(text), source: COALESCE( CASE WHEN d.url CONTAINS "None" THEN d.fileName ELSE d.url END, d.fileName), chunkdetails: chunkdetails}} AS metadata """ -YOUTUBE_CHUNK_SIZE_SECONDS = 60 \ No newline at end of file +YOUTUBE_CHUNK_SIZE_SECONDS = 60 diff --git a/backend/test_integrationqa.py b/backend/test_integrationqa.py index cd662cbdc..821cc6b5c 100644 --- a/backend/test_integrationqa.py +++ b/backend/test_integrationqa.py @@ -1,270 +1,243 @@ +import json +import os +import shutil +import logging +import pandas as pd +from datetime import datetime as dt +from dotenv import load_dotenv + from score import * from src.main import * -import logging from src.QA_integration_new import QA_RAG from langserve import add_routes -import asyncio -import os -from dotenv import load_dotenv -import pandas as pd -from datetime import datetime as dt -uri = '' -userName = '' -password = '' -# model = 'openai-gpt-3.5' -database = 'neo4j' +# Load environment variables if needed +load_dotenv() + +# Constants +URI = '' +USERNAME = '' +PASSWORD = '' +DATABASE = 'neo4j' CHUNK_DIR = os.path.join(os.path.dirname(__file__), "chunks") MERGED_DIR = os.path.join(os.path.dirname(__file__), "merged_files") -graph = create_graph_database_connection(uri, userName, password, database) - - -def test_graph_from_file_local_file(model_name): - model = model_name - file_name = 'About Amazon.pdf' - # shutil.copyfile('data/Bank of America Q23.pdf', 'backend/src/merged_files/Bank of America Q23.pdf') - shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf', - '/workspaces/llm-graph-builder/backend/merged_files/About Amazon.pdf') - obj_source_node = sourceNode() - obj_source_node.file_name = file_name - obj_source_node.file_type = 'pdf' - obj_source_node.file_size = '1087' - obj_source_node.file_source = 'local file' - obj_source_node.model = model - obj_source_node.created_at = datetime.now() - graphDb_data_Access = graphDBdataAccess(graph) - graphDb_data_Access.create_source_node(obj_source_node) - merged_file_path = os.path.join(MERGED_DIR, file_name) - print(merged_file_path) - - - local_file_result = extract_graph_from_file_local_file(uri, userName, password, database, model, merged_file_path, file_name, '', '') - # final_list.append(local_file_result) - print(local_file_result) - - logging.info("Info: ") - try: - assert local_file_result['status'] == 'Completed' and local_file_result['nodeCount'] > 0 and local_file_result[ - 'relationshipCount'] > 0 - return local_file_result - print("Success") - except AssertionError as e: - print("Fail: ", e) - return local_file_result - - -def test_graph_from_file_local_file_failed(model_name): - model = model_name - file_name = 'Not_exist.pdf' - try: - obj_source_node = sourceNode() - obj_source_node.file_name = file_name - obj_source_node.file_type = 'pdf' - obj_source_node.file_size = '0' - obj_source_node.file_source = 'local file' - obj_source_node.model = model - obj_source_node.created_at = datetime.now() - graphDb_data_Access = graphDBdataAccess(graph) - graphDb_data_Access.create_source_node(obj_source_node) - - local_file_result = extract_graph_from_file_local_file(graph, model, file_name, merged_file_path, '', '') - - print(local_file_result) - except AssertionError as e: - print('Failed due to file does not exist means not uploaded or accidentaly deleteled from server') - print("Failed: Error from extract function ", e) - -# Check for Wikipedia file to be test -def test_graph_from_Wikipedia(model_name): - model = model_name - wiki_query = 'https://en.wikipedia.org/wiki/Ram_Mandir' - source_type = 'Wikipedia' - file_name = "Ram_Mandir" - create_source_node_graph_url_wikipedia(graph, model, wiki_query, source_type) - wikiresult = extract_graph_from_file_Wikipedia(uri, userName, password, database, model, file_name, 1, 'en', '', '') - logging.info("Info: Wikipedia test done") - print(wikiresult) - + +# Initialize database connection +graph = create_graph_database_connection(URI, USERNAME, PASSWORD, DATABASE) + +def create_source_node_local(graph, model, file_name): + """Creates a source node for a local file.""" + source_node = sourceNode() + source_node.file_name = file_name + source_node.file_type = 'pdf' + source_node.file_size = '1087' + source_node.file_source = 'local file' + source_node.model = model + source_node.created_at = dt.now() + graphDB_data_Access = graphDBdataAccess(graph) + graphDB_data_Access.create_source_node(source_node) + return source_node + +def test_graph_from_file_local(model_name): + """Test graph creation from a local file.""" + file_name = 'About Amazon.pdf' + shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf', + os.path.join(MERGED_DIR, file_name)) + + create_source_node_local(graph, model_name, file_name) + merged_file_path = os.path.join(MERGED_DIR, file_name) + + local_file_result = extract_graph_from_file_local_file( + URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '' + ) + logging.info("Local file processing complete") + print(local_file_result) + try: - assert wikiresult['status'] == 'Completed' and wikiresult['nodeCount'] > 0 and wikiresult['relationshipCount'] > 0 - return wikiresult + assert local_file_result['status'] == 'Completed' + assert local_file_result['nodeCount'] > 0 + assert local_file_result['relationshipCount'] > 0 print("Success") except AssertionError as e: - print("Fail ", e) - return wikiresult - + print("Fail: ", e) -def test_graph_from_Wikipedia_failed(): - wiki_query = 'Test QA 123456' - source_type = 'Wikipedia' - try: - logging.info("Created source node for wikipedia") - create_source_node_graph_url_wikipedia(graph, model, wiki_query, source_type) - except AssertionError as e: - print("Fail ", e) + return local_file_result -# Check for Youtube_video to be Success -def test_graph_from_youtube_video(model_name): - model = model_name - source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA' - source_type = 'youtube' +def test_graph_from_wikipedia(model_name): + """Test graph creation from a Wikipedia page.""" + wiki_query = 'https://en.wikipedia.org/wiki/Ram_Mandir' + source_type = 'Wikipedia' + file_name = "Ram_Mandir" + create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type) - create_source_node_graph_url_youtube(graph, model, source_url, source_type) - youtuberesult = extract_graph_from_file_youtube(uri, userName, password, database, model, source_url, '', '') + wiki_result = extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 1, 'en', '', '') + logging.info("Wikipedia test done") + print(wiki_result) - logging.info("Info: Youtube Video test done") - print(youtuberesult) try: - assert youtuberesult['status'] == 'Completed' and youtuberesult['nodeCount'] > 1 and youtuberesult[ - 'relationshipCount'] > 1 - return youtuberesult + assert wiki_result['status'] == 'Completed' + assert wiki_result['nodeCount'] > 0 + assert wiki_result['relationshipCount'] > 0 print("Success") except AssertionError as e: - print("Failed ", e) - return youtuberesult - -# Check for Youtube_video to be Failed - -def test_graph_from_youtube_video_failed(): - url = 'https://www.youtube.com/watch?v=U9mJuUkhUzk' - source_type = 'youtube' - - create_source_node_graph_url_youtube(graph, model, url, source_type) - youtuberesult = extract_graph_from_file_youtube(graph, model, url, ',', ',') - # print(result) - print(youtuberesult) - try: - assert youtuberesult['status'] == 'Completed' - return youtuberesult - except AssertionError as e: - print("Failed ", e) - - -# Check for the GCS file to be uploaded, process and completed - -def test_graph_from_file_test_gcs(): - bucket_name = 'test' - folder_name = 'test' - source_type = 'gcs test bucket' - file_name = 'Neuralink brain chip patient playing chess.pdf' - create_source_node_graph_url_gcs(graph, model, bucket_name, folder_name, source_type) - gcsresult = extract_graph_from_file_gcs(graph, model, bucket_name, folder_name, file_name, '', '') - - logging.info("Info") - print(gcsresult) - - try: - assert gcsresult['status'] == 'Completed' and gcsresult['nodeCount'] > 10 and gcsresult['relationshipCount'] > 5 - print("Success") - except AssertionError as e: - print("Failed ", e) - - -def test_graph_from_file_test_gcs_failed(): - bucket_name = 'llm_graph_test' - folder_name = 'test' - source_type = 'gcs bucket' - # file_name = 'Neuralink brain chip patient playing chess.pdf' - try: - create_source_node_graph_url_gcs(graph, model, bucket_name, folder_name, source_type) - print("GCS: Create source node failed due to bucket not exist") - except AssertionError as e: - print("Failed ", e) - - -def test_graph_from_file_test_s3_failed(): - source_url = 's3://development-llm-test/' - try: - create_source_node_graph_url_s3(graph, model, source_url, 'test123', 'pwd123') - # assert result['status'] == 'Failed' - # print("S3 created source node failed die to wrong access key id and secret") - except AssertionError as e: - print("Failed ", e) - - -# Check the Functionality of Chatbot QnA for mode 'graph+vector' -def test_chatbot_QnA(model_name): - model = model_name - QA_n_RAG = QA_RAG(graph, model, 'Tell me about amazon', '[]', 1, 'graph+vector') + print("Fail: ", e) + + return wiki_result + +def test_graph_website(model_name): + """Test graph creation from a Website page.""" + #graph, model, source_url, source_type + source_url = 'https://www.amazon.com/' + source_type = 'web-url' + create_source_node_graph_web_url(graph, model_name, source_url, source_type) + + weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '') + logging.info("WebUrl test done") + print(weburl_result) - print(QA_n_RAG) - print(len(QA_n_RAG['message'])) try: - assert len(QA_n_RAG['message']) > 20 - return QA_n_RAG + assert weburl_result['status'] == 'Completed' + assert weburl_result['nodeCount'] > 0 + assert weburl_result['relationshipCount'] > 0 print("Success") except AssertionError as e: - print("Failed ", e) - return QA_n_RAG + print("Fail: ", e) + return weburl_result -# Check the Functionality of Chatbot QnA for mode 'vector' -def test_chatbot_QnA_vector(model_name): - model = model_name - QA_n_RAG_vector = QA_RAG(graph, model, 'Tell me about amazon', '[]', 1, 'vector') +def test_graph_from_youtube_video(model_name): + """Test graph creation from a YouTube video.""" + source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA' + source_type = 'youtube' + create_source_node_graph_url_youtube(graph, model_name, source_url, source_type) + youtube_result = extract_graph_from_file_youtube( + URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '' + ) + logging.info("YouTube Video test done") + print(youtube_result) - print(QA_n_RAG_vector) - print(len(QA_n_RAG_vector['message'])) try: - assert len(QA_n_RAG_vector['message']) > 20 - return QA_n_RAG_vector + assert youtube_result['status'] == 'Completed' + assert youtube_result['nodeCount'] > 1 + assert youtube_result['relationshipCount'] > 1 print("Success") except AssertionError as e: - print("Failed ", e) - return QA_n_RAG_vector - -# Check the Functionality of Chatbot QnA for mode 'hybrid' + print("Failed: ", e) -def test_chatbot_QnA_hybrid(model_name): - model = model_name - QA_n_RAG_hybrid = QA_RAG(graph, model, 'Tell me about amazon', '[]', 1, 'hybrid') + return youtube_result +def test_chatbot_qna(model_name, mode='vector'): + """Test chatbot QnA functionality for different modes.""" + QA_n_RAG = QA_RAG(graph, model_name, 'Tell me about amazon', '[]', 1, mode) + print(QA_n_RAG) + print(len(QA_n_RAG['message'])) - print(QA_n_RAG_hybrid) - print(len(QA_n_RAG_hybrid['message'])) try: - assert len(QA_n_RAG_hybrid['message']) > 20 - return QA_n_RAG_hybrid + assert len(QA_n_RAG['message']) > 20 + return QA_n_RAG print("Success") except AssertionError as e: print("Failed ", e) - return QA_n_RAG_hybrid - - + return QA_n_RAG + +#Get Test disconnected_nodes list +def disconected_nodes(): + #graph = create_graph_database_connection(uri, userName, password, database) + graphDb_data_Access = graphDBdataAccess(graph) + nodes_list, total_nodes = graphDb_data_Access.list_unconnected_nodes() + print(nodes_list[0]["e"]["elementId"]) + status = "False" + + if total_nodes['total']>0: + status = "True" + else: + status = "False" + + return nodes_list[0]["e"]["elementId"], status + +#Test Delete delete_disconnected_nodes list +def delete_disconected_nodes(lst_element_id): + print(f'disconnect elementid list {lst_element_id}') + #graph = create_graph_database_connection(uri, userName, password, database) + graphDb_data_Access = graphDBdataAccess(graph) + result = graphDb_data_Access.delete_unconnected_nodes(json.dumps(lst_element_id)) + print(f'delete disconnect api result {result}') + if not result: + return "True" + else: + return "False" + +#Test Get Duplicate_nodes +def get_duplicate_nodes(): + #graph = create_graph_database_connection(uri, userName, password, database) + graphDb_data_Access = graphDBdataAccess(graph) + nodes_list, total_nodes = graphDb_data_Access.get_duplicate_nodes_list() + if total_nodes['total']>0: + return "True" + else: + return "False" + +#Test populate_graph_schema +def test_populate_graph_schema_from_text(model): + result_schema = populate_graph_schema_from_text('When Amazon was founded in 1993 by creator Jeff Benzos, it was mostly an online bookstore. Initially Amazon’s growth was very slow, not turning a profit until over 7 years after its founding. This was thanks to the great momentum provided by the dot-com bubble.', model, True) + print(result_schema) + return result_schema + +# def compare_graph_results(results): +# """ +# Compare graph results across different models. +# Add custom logic here to compare graph data, nodes, and relationships. +# """ +# # Placeholder logic for comparison +# print("Comparing results...") +# for i in range(len(results) - 1): +# result_a = results[i] +# result_b = results[i + 1] +# if result_a == result_b: +# print(f"Result {i} is identical to result {i+1}") +# else: +# print(f"Result {i} differs from result {i+1}") + +def run_tests(): + final_list = [] + error_list = [] + models = ['openai-gpt-3.5', 'openai-gpt-4o'] + + for model_name in models: + try: + final_list.append(test_graph_from_file_local(model_name)) + final_list.append(test_graph_from_wikipedia(model_name)) + final_list.append(test_populate_graph_schema_from_text(model_name)) + final_list.append(test_graph_website(model_name)) + final_list.append(test_graph_from_youtube_video(model_name)) + final_list.append(test_chatbot_qna(model_name)) + final_list.append(test_chatbot_qna(model_name, mode='vector')) + final_list.append(test_chatbot_qna(model_name, mode='graph+vector+fulltext')) + except Exception as e: + error_list.append((model_name, str(e))) + # #Compare and log diffrences in graph results + # # compare_graph_results(final_list) # Pass the final_list to comapre_graph_results + # test_populate_graph_schema_from_text('openai-gpt-4o') + dis_elementid, dis_status = disconected_nodes() + lst_element_id = [dis_elementid] + delt = delete_disconected_nodes(lst_element_id) + dup = get_duplicate_nodes() + # schma = test_populate_graph_schema_from_text(model) + # Save final results to CSV + df = pd.DataFrame(final_list) + print(df) + df['execution_date'] = dt.today().strftime('%Y-%m-%d') + df['disconnected_nodes']=dis_status + df['get_duplicate_nodes']=dup + df['delete_disconected_nodes']=delt + # df['test_populate_graph_schema_from_text'] = schma + df.to_csv(f"Integration_TestResult_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False) + + # Save error details to CSV + df_errors = pd.DataFrame(error_list, columns=['Model', 'Error']) + df_errors['execution_date'] = dt.today().strftime('%Y-%m-%d') + df_errors.to_csv(f"Error_details_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False) if __name__ == "__main__": - final_list = [] - for model_name in ['openai-gpt-3.5','azure_ai_gpt_35','azure_ai_gpt_4o','anthropic_claude_3_5_sonnet','fireworks_llama_v3_70b','bedrock_claude_3_5_sonnet']: - - # local file - response = test_graph_from_file_local_file(model_name) - final_list.append(response) - - # # Wikipedia Test - response = test_graph_from_Wikipedia(model_name) - final_list.append(response) - - # # Youtube Test - response= test_graph_from_youtube_video(model_name) - final_list.append(response) - # # print(final_list) - - # # test_graph_from_file_test_gcs(model_name) # GCS Test - - # #chatbot 'graph+vector' - response = test_chatbot_QnA(model_name) - final_list.append(response) - - # #chatbot 'vector' - response = test_chatbot_QnA_vector(model_name) - final_list.append(response) - - # #chatbot 'hybrid' - response = test_chatbot_QnA_hybrid(model_name) - final_list.append(response) - - # test_graph_from_file_test_s3_failed() # S3 Failed Test Case - df = pd.DataFrame(final_list) - df['execution_date']= datetime.today().strftime('%Y-%m-%d') - df.to_csv(f"Integration_TestResult_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False) \ No newline at end of file + run_tests() \ No newline at end of file diff --git a/example.env b/example.env index b443fd18d..23bcc6e06 100644 --- a/example.env +++ b/example.env @@ -1,27 +1,27 @@ # Mandatory -OPENAI_API_KEY = "" -DIFFBOT_API_KEY = "" +OPENAI_API_KEY="" +DIFFBOT_API_KEY="" # Optional Backend -EMBEDDING_MODEL = "all-MiniLM-L6-v2" -IS_EMBEDDING = "true" -KNN_MIN_SCORE = "0.94" +EMBEDDING_MODEL="all-MiniLM-L6-v2" +IS_EMBEDDING="true" +KNN_MIN_SCORE="0.94" # Enable Gemini (default is False) | Can be False or True -GEMINI_ENABLED = False +GEMINI_ENABLED=False # LLM_MODEL_CONFIG_ollama_llama3="llama3,http://host.docker.internal:11434" # Enable Google Cloud logs (default is False) | Can be False or True -GCP_LOG_METRICS_ENABLED = False -NUMBER_OF_CHUNKS_TO_COMBINE = 6 -UPDATE_GRAPH_CHUNKS_PROCESSED = 20 -NEO4J_URI = "neo4j://database:7687" -NEO4J_USERNAME = "neo4j" -NEO4J_PASSWORD = "password" -LANGCHAIN_API_KEY = "" -LANGCHAIN_PROJECT = "" -LANGCHAIN_TRACING_V2 = "true" -LANGCHAIN_ENDPOINT = "https://api.smith.langchain.com" -GCS_FILE_CACHE = False +GCP_LOG_METRICS_ENABLED=False +NUMBER_OF_CHUNKS_TO_COMBINE=6 +UPDATE_GRAPH_CHUNKS_PROCESSED=20 +NEO4J_URI="neo4j://database:7687" +NEO4J_USERNAME="neo4j" +NEO4J_PASSWORD="password" +LANGCHAIN_API_KEY="" +LANGCHAIN_PROJECT="" +LANGCHAIN_TRACING_V2="true" +LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" +GCS_FILE_CACHE=False ENTITY_EMBEDDING=True # Optional Frontend @@ -30,9 +30,8 @@ VITE_BLOOM_URL="https://workspace-preview.neo4j.io/workspace/explore?connectURL= VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,web" VITE_LLM_MODELS="diffbot,openai-gpt-3.5,openai-gpt-4o" # ",ollama_llama3" VITE_ENV="DEV" -VITE_TIME_PER_CHUNK=4 VITE_TIME_PER_PAGE=50 VITE_CHUNK_SIZE=5242880 VITE_GOOGLE_CLIENT_ID="" VITE_CHAT_MODES="" -VITE_BATCH_SIZE=2 \ No newline at end of file +VITE_BATCH_SIZE=2 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 7a31d5bcf..c3a7c1c82 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -6,7 +6,6 @@ ARG VITE_REACT_APP_SOURCES="" ARG VITE_LLM_MODELS="" ARG VITE_GOOGLE_CLIENT_ID="" ARG VITE_BLOOM_URL="https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true" -ARG VITE_TIME_PER_CHUNK=4 ARG VITE_TIME_PER_PAGE=50 ARG VITE_LARGE_FILE_SIZE=5242880 ARG VITE_CHUNK_SIZE=5242880 @@ -23,8 +22,8 @@ RUN VITE_BACKEND_API_URL=$VITE_BACKEND_API_URL \ VITE_LLM_MODELS=$VITE_LLM_MODELS \ VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID \ VITE_BLOOM_URL=$VITE_BLOOM_URL \ - VITE_TIME_PER_CHUNK=$VITE_TIME_PER_CHUNK \ VITE_CHUNK_SIZE=$VITE_CHUNK_SIZE \ + VITE_TIME_PER_PAGE=$VITE_TIME_PER_PAGE \ VITE_ENV=$VITE_ENV \ VITE_LARGE_FILE_SIZE=${VITE_LARGE_FILE_SIZE} \ VITE_CHAT_MODES=$VITE_CHAT_MODES \ diff --git a/frontend/example.env b/frontend/example.env index 05b8cdf60..63bd3e7c3 100644 --- a/frontend/example.env +++ b/frontend/example.env @@ -3,7 +3,6 @@ VITE_BLOOM_URL="https://workspace-preview.neo4j.io/workspace/explore?connectURL= VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,web" VITE_LLM_MODELS="diffbot,openai-gpt-3.5,openai-gpt-4o" VITE_ENV="DEV" -VITE_TIME_PER_CHUNK=4 VITE_TIME_PER_PAGE=50 VITE_CHUNK_SIZE=5242880 VITE_LARGE_FILE_SIZE=5242880 diff --git a/frontend/src/API/Index.ts b/frontend/src/API/Index.ts new file mode 100644 index 000000000..f4ad15cbe --- /dev/null +++ b/frontend/src/API/Index.ts @@ -0,0 +1,7 @@ +import axios from 'axios'; +import { url } from '../utils/Utils'; + +const api = axios.create({ + baseURL: url(), +}); +export default api; diff --git a/frontend/src/App.css b/frontend/src/App.css index ff084ffae..fe285c972 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -233,10 +233,8 @@ letter-spacing: 0; line-height: 1.25rem; width: max-content; - height: 30px; text-overflow: ellipsis; white-space: nowrap; - overflow: hidden; } .ndl-widget-content>div { @@ -365,4 +363,13 @@ .widthunset{ width: initial !important; height: initial !important; +} + +.text-input-container { + transition: width 1.5s ease; + /* width: 100dvh; */ +} + +.text-input-container.search-initiated { + width: 60dvh; } \ No newline at end of file diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index 7bd837299..89c15ea72 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -18,13 +18,20 @@ import wikipedialogo from '../../assets/images/wikipedia.svg'; import youtubelogo from '../../assets/images/youtube.svg'; import gcslogo from '../../assets/images/gcs.webp'; import s3logo from '../../assets/images/s3logo.png'; -import { Chunk, Entity, ExtendedNode, GroupedEntity, UserCredentials, chatInfoMessage } from '../../types'; +import { + Chunk, + Entity, + ExtendedNode, + ExtendedRelationship, + GroupedEntity, + UserCredentials, + chatInfoMessage, +} from '../../types'; import { useContext, useEffect, useMemo, useState } from 'react'; import HoverableLink from '../UI/HoverableLink'; import GraphViewButton from '../Graph/GraphViewButton'; import { chunkEntitiesAPI } from '../../services/ChunkEntitiesInfo'; import { useCredentials } from '../../context/UserCredentials'; -import type { Relationship } from '@neo4j-nvl/base'; import { calcWordColor } from '@neo4j-devtools/word-color'; import ReactMarkdown from 'react-markdown'; import { GlobeAltIconOutline } from '@neo4j-ndl/react/icons'; @@ -51,7 +58,7 @@ const ChatInfoModal: React.FC = ({ const [loading, setLoading] = useState(false); const { userCredentials } = useCredentials(); const [nodes, setNodes] = useState([]); - const [relationships, setRelationships] = useState([]); + const [relationships, setRelationships] = useState([]); const [chunks, setChunks] = useState([]); const themeUtils = useContext(ThemeWrapperContext); const [, copy] = useCopyToClipboard(); @@ -168,7 +175,11 @@ const ChatInfoModal: React.FC = ({ ) : ( {mode != 'graph' ? Sources used : <>} - {mode === 'graph+vector' || mode === 'graph' ? Top Entities used : <>} + {mode === 'graph+vector' || mode === 'graph' || mode === 'graph+vector+fulltext' ? ( + Top Entities used + ) : ( + <> + )} {mode === 'graph' && cypher_query?.trim().length ? ( Generated Cypher Query ) : ( diff --git a/frontend/src/components/ChatBot/ChatModeToggle.tsx b/frontend/src/components/ChatBot/ChatModeToggle.tsx index 4c0d54bc7..e82dfea4d 100644 --- a/frontend/src/components/ChatBot/ChatModeToggle.tsx +++ b/frontend/src/components/ChatBot/ChatModeToggle.tsx @@ -31,7 +31,12 @@ export default function ChatModeToggle({ () => chatModes?.map((m) => { return { - title: capitalize(m), + title: m.includes('+') + ? m + .split('+') + .map((s) => capitalize(s)) + .join('+') + : capitalize(m), onClick: () => { setchatMode(m); }, diff --git a/frontend/src/components/ChatBot/Info/InfoModal.tsx b/frontend/src/components/ChatBot/Info/InfoModal.tsx index 0dd325731..cf1bbca47 100644 --- a/frontend/src/components/ChatBot/Info/InfoModal.tsx +++ b/frontend/src/components/ChatBot/Info/InfoModal.tsx @@ -6,13 +6,20 @@ import wikipedialogo from '../../../assets/images/Wikipedia-logo-v2.svg'; import youtubelogo from '../../../assets/images/youtube.png'; import gcslogo from '../../../assets/images/gcs.webp'; import s3logo from '../../../assets/images/s3logo.png'; -import { Chunk, Entity, ExtendedNode, GroupedEntity, UserCredentials, chatInfoMessage } from '../../../types'; +import { + Chunk, + Entity, + ExtendedNode, + ExtendedRelationship, + GroupedEntity, + UserCredentials, + chatInfoMessage, +} from '../../../types'; import { useEffect, useMemo, useState } from 'react'; import HoverableLink from '../../UI/HoverableLink'; import GraphViewButton from '../../Graph/GraphViewButton'; import { chunkEntitiesAPI } from '../../../services/ChunkEntitiesInfo'; import { useCredentials } from '../../../context/UserCredentials'; -import type { Relationship } from '@neo4j-nvl/base'; import { calcWordColor } from '@neo4j-devtools/word-color'; import ReactMarkdown from 'react-markdown'; import { GlobeAltIconOutline } from '@neo4j-ndl/react/icons'; @@ -23,7 +30,7 @@ const InfoModal: React.FC = ({ sources, model, total_tokens, re const [loading, setLoading] = useState(false); const { userCredentials } = useCredentials(); const [nodes, setNodes] = useState([]); - const [relationships, setRelationships] = useState([]); + const [relationships, setRelationships] = useState([]); const [chunks, setChunks] = useState([]); const parseEntity = (entity: Entity) => { const { labels, properties } = entity; diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index ec4fad9bb..4bde5a9b4 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -31,6 +31,8 @@ import FallBackDialog from './UI/FallBackDialog'; import DeletePopUp from './Popups/DeletePopUp/DeletePopUp'; import GraphEnhancementDialog from './Popups/GraphEnhancementDialog'; import { tokens } from '@neo4j-ndl/base'; +import axios from 'axios'; + const ConnectionModal = lazy(() => import('./Popups/ConnectionModal/ConnectionModal')); const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog')); let afterFirstRender = false; @@ -180,6 +182,7 @@ const Content: React.FC = ({ }; const extractHandler = async (fileItem: CustomFile, uid: string) => { + queue.remove(fileItem.name as string); try { setFilesData((prevfiles) => prevfiles.map((curfile) => { @@ -252,28 +255,45 @@ const Content: React.FC = ({ }); } } catch (err: any) { - const error = JSON.parse(err.message); - if (Object.keys(error).includes('fileName')) { - const { message } = error; - const { fileName } = error; - const errorMessage = error.message; - setalertDetails({ - showAlert: true, - alertType: 'error', - alertMessage: message, - }); - setFilesData((prevfiles) => - prevfiles.map((curfile) => { - if (curfile.name == fileName) { - return { - ...curfile, - status: 'Failed', - errorMessage, - }; - } - return curfile; - }) - ); + if (err instanceof Error) { + try { + const error = JSON.parse(err.message); + if (Object.keys(error).includes('fileName')) { + setProcessedCount((prev) => { + if (prev == batchSize) { + return batchSize - 1; + } + return prev + 1; + }); + const { message, fileName } = error; + queue.remove(fileName); + const errorMessage = error.message; + setalertDetails({ + showAlert: true, + alertType: 'error', + alertMessage: message, + }); + setFilesData((prevfiles) => + prevfiles.map((curfile) => { + if (curfile.name == fileName) { + return { ...curfile, status: 'Failed', errorMessage }; + } + return curfile; + }) + ); + } else { + console.error('Unexpected error format:', error); + } + } catch (parseError) { + if (axios.isAxiosError(err)) { + const axiosErrorMessage = err.response?.data?.message || err.message; + console.error('Axios error occurred:', axiosErrorMessage); + } else { + console.error('An unexpected error occurred:', err.message); + } + } + } else { + console.error('An unknown error occurred:', err); } } }; @@ -302,7 +322,10 @@ const Content: React.FC = ({ return data; }; - const addFilesToQueue = (remainingFiles: CustomFile[]) => { + const addFilesToQueue = async (remainingFiles: CustomFile[]) => { + if (!remainingFiles.length) { + await postProcessing(userCredentials as UserCredentials, postProcessingTasks); + } remainingFiles.forEach((f) => { setFilesData((prev) => prev.map((pf) => { @@ -379,13 +402,11 @@ const Content: React.FC = ({ } Promise.allSettled(data).then(async (_) => { setextractLoading(false); - await postProcessing(userCredentials as UserCredentials, postProcessingTasks); }); - } else if (queueFiles && !queue.isEmpty()) { + } else if (queueFiles && !queue.isEmpty() && processingFilesCount < batchSize) { data = scheduleBatchWiseProcess(queue.items, true); Promise.allSettled(data).then(async (_) => { setextractLoading(false); - await postProcessing(userCredentials as UserCredentials, postProcessingTasks); }); } else { addFilesToQueue(filesTobeProcessed as CustomFile[]); @@ -405,7 +426,6 @@ const Content: React.FC = ({ } Promise.allSettled(data).then(async (_) => { setextractLoading(false); - await postProcessing(userCredentials as UserCredentials, postProcessingTasks); }); } else { const selectedNewFiles = childRef.current?.getSelectedRows().filter((f) => f.status === 'New'); diff --git a/frontend/src/components/Dropdown.tsx b/frontend/src/components/Dropdown.tsx index f046bb8ff..ba949aec0 100644 --- a/frontend/src/components/Dropdown.tsx +++ b/frontend/src/components/Dropdown.tsx @@ -1,6 +1,6 @@ import { Dropdown, Tip } from '@neo4j-ndl/react'; import { OptionType, ReusableDropdownProps } from '../types'; -import { useMemo } from 'react'; +import { useMemo, useReducer } from 'react'; import { capitalize } from '../utils/Utils'; const DropdownComponent: React.FC = ({ @@ -13,6 +13,7 @@ const DropdownComponent: React.FC = ({ isDisabled, value, }) => { + const [disableTooltip, toggleDisableState] = useReducer((state) => !state, false); const handleChange = (selectedOption: OptionType | null | void) => { onSelect(selectedOption); }; @@ -20,7 +21,7 @@ const DropdownComponent: React.FC = ({ return ( <>
- + = ({ menuPlacement: 'auto', isDisabled: isDisabled, value: value, + onMenuOpen: () => { + toggleDisableState(); + }, + onMenuClose: () => { + toggleDisableState(); + }, }} size='medium' fluid diff --git a/frontend/src/components/FileTable.tsx b/frontend/src/components/FileTable.tsx index 70dff8f1a..23310eaf4 100644 --- a/frontend/src/components/FileTable.tsx +++ b/frontend/src/components/FileTable.tsx @@ -761,6 +761,13 @@ const FileTable = forwardRef((props, ref) => { return curfile; }) ); + setProcessedCount((prev) => { + if (prev == batchSize) { + return batchSize - 1; + } + return prev + 1; + }); + queue.remove(fileName); } else { let errorobj = { error: res.data.error, message: res.data.message, fileName }; throw new Error(JSON.stringify(errorobj)); diff --git a/frontend/src/components/Graph/GraphViewModal.tsx b/frontend/src/components/Graph/GraphViewModal.tsx index 9c0d5d190..39438b788 100644 --- a/frontend/src/components/Graph/GraphViewModal.tsx +++ b/frontend/src/components/Graph/GraphViewModal.tsx @@ -1,6 +1,23 @@ -import { Banner, Dialog, Flex, IconButtonArray, LoadingSpinner, Typography } from '@neo4j-ndl/react'; +import { + Banner, + Dialog, + Flex, + IconButton, + IconButtonArray, + LoadingSpinner, + TextInput, + Typography, + useDebounce, +} from '@neo4j-ndl/react'; import { useCallback, useEffect, useRef, useState } from 'react'; -import { ExtendedNode, GraphType, GraphViewModalProps, Scheme, UserCredentials } from '../../types'; +import { + ExtendedNode, + ExtendedRelationship, + GraphType, + GraphViewModalProps, + Scheme, + UserCredentials, +} from '../../types'; import { InteractiveNvlWrapper } from '@neo4j-nvl/react'; import NVL from '@neo4j-nvl/base'; import type { Node, Relationship } from '@neo4j-nvl/base'; @@ -9,16 +26,26 @@ import { ArrowPathIconOutline, DragIcon, FitToScreenIcon, + MagnifyingGlassIconOutline, MagnifyingGlassMinusIconOutline, MagnifyingGlassPlusIconOutline, } from '@neo4j-ndl/react/icons'; import IconButtonWithToolTip from '../UI/IconButtonToolTip'; -import { filterData, processGraphData } from '../../utils/Utils'; +import { filterData, processGraphData, sortAlphabetically } from '../../utils/Utils'; import { useCredentials } from '../../context/UserCredentials'; import { LegendsChip } from './LegendsChip'; import graphQueryAPI from '../../services/GraphQuery'; -import { graphLabels, intitalGraphType, mouseEventCallbacks, nvlOptions, queryMap } from '../../utils/Constants'; +import { + graphLabels, + intitalGraphType, + mouseEventCallbacks, + nvlOptions, + queryMap, + RESULT_STEP_SIZE, +} from '../../utils/Constants'; import CheckboxSelection from './CheckboxSelection'; +import { ShowAll } from '../UI/ShowAll'; + const GraphViewModal: React.FunctionComponent = ({ open, inspectedName, @@ -40,6 +67,10 @@ const GraphViewModal: React.FunctionComponent = ({ const { userCredentials } = useCredentials(); const [scheme, setScheme] = useState({}); const [newScheme, setNewScheme] = useState({}); + const [searchQuery, setSearchQuery] = useState(''); + const debouncedQuery = useDebounce(searchQuery, 300); + + // the checkbox selection const handleCheckboxChange = (graph: GraphType) => { const currentIndex = graphType.indexOf(graph); const newGraphSelected = [...graphType]; @@ -50,9 +81,18 @@ const GraphViewModal: React.FunctionComponent = ({ newGraphSelected.splice(currentIndex, 1); initGraph(newGraphSelected, allNodes, allRelationships, scheme); } + setSearchQuery(''); setGraphType(newGraphSelected); }; + const nodeCount = (nodes: ExtendedNode[], label: string): number => { + return [...new Set(nodes?.filter((n) => n.labels?.includes(label)).map((i) => i.id))].length; + }; + + const relationshipCount = (relationships: ExtendedRelationship[], label: string): number => { + return [...new Set(relationships?.filter((r) => r.caption?.includes(label)).map((i) => i.id))].length; + }; + const graphQuery: string = graphType.includes('DocumentChunk') && graphType.includes('Entities') ? queryMap.DocChunkEntities @@ -62,6 +102,7 @@ const GraphViewModal: React.FunctionComponent = ({ ? queryMap.Entities : ''; + // fit graph to original position const handleZoomToFit = () => { nvlRef.current?.fit( allNodes.map((node) => node.id), @@ -75,7 +116,9 @@ const GraphViewModal: React.FunctionComponent = ({ handleZoomToFit(); }, 10); return () => { - nvlRef.current?.destroy(); + if (nvlRef.current) { + nvlRef.current?.destroy(); + } setGraphType(intitalGraphType); clearTimeout(timeoutId); setScheme({}); @@ -83,6 +126,7 @@ const GraphViewModal: React.FunctionComponent = ({ setRelationships([]); setAllNodes([]); setAllRelationships([]); + setSearchQuery(''); }; }, []); @@ -151,6 +195,71 @@ const GraphViewModal: React.FunctionComponent = ({ } }, [open]); + // The search and update nodes + const handleSearch = useCallback( + (value: string) => { + const query = value.toLowerCase(); + const updatedNodes = nodes.map((node) => { + if (query === '') { + return { + ...node, + activated: false, + selected: false, + size: graphLabels.nodeSize, + }; + } + const { id, properties, caption } = node; + const propertiesMatch = properties?.id?.toLowerCase().includes(query); + const match = id.toLowerCase().includes(query) || propertiesMatch || caption?.toLowerCase().includes(query); + return { + ...node, + activated: match, + selected: match, + size: + match && viewPoint === graphLabels.showGraphView + ? 100 + : match && viewPoint !== graphLabels.showGraphView + ? 50 + : graphLabels.nodeSize, + }; + }); + // deactivating any active relationships + const updatedRelationships = relationships.map((rel) => { + return { + ...rel, + activated: false, + selected: false, + }; + }); + setNodes(updatedNodes); + setRelationships(updatedRelationships); + }, + [nodes] + ); + + useEffect(() => { + handleSearch(debouncedQuery); + }, [debouncedQuery]); + + const initGraph = ( + graphType: GraphType[], + finalNodes: ExtendedNode[], + finalRels: Relationship[], + schemeVal: Scheme + ) => { + if (allNodes.length > 0 && allRelationships.length > 0) { + const { filteredNodes, filteredRelations, filteredScheme } = filterData( + graphType, + finalNodes ?? [], + finalRels ?? [], + schemeVal + ); + setNodes(filteredNodes); + setRelationships(filteredRelations); + setNewScheme(filteredScheme); + } + }; + // Unmounting the component if (!open) { return <>; @@ -201,10 +310,11 @@ const GraphViewModal: React.FunctionComponent = ({ setRelationships([]); setAllNodes([]); setAllRelationships([]); + setSearchQuery(''); }; // sort the legends in with Chunk and Document always the first two values - const legendCheck = Object.keys(newScheme).sort((a, b) => { + const nodeCheck = Object.keys(newScheme).sort((a, b) => { if (a === graphLabels.document || a === graphLabels.chunk) { return -1; } else if (b === graphLabels.document || b === graphLabels.chunk) { @@ -213,23 +323,75 @@ const GraphViewModal: React.FunctionComponent = ({ return a.localeCompare(b); }); - const initGraph = ( - graphType: GraphType[], - finalNodes: ExtendedNode[], - finalRels: Relationship[], - schemeVal: Scheme - ) => { - if (allNodes.length > 0 && allRelationships.length > 0) { - const { filteredNodes, filteredRelations, filteredScheme } = filterData( - graphType, - finalNodes ?? [], - finalRels ?? [], - schemeVal - ); - setNodes(filteredNodes); - setRelationships(filteredRelations); - setNewScheme(filteredScheme); + // get sorted relationships + const relationshipsSorted = relationships.sort(sortAlphabetically); + + // To get the relationship count + const groupedAndSortedRelationships: ExtendedRelationship[] = Object.values( + relationshipsSorted.reduce((acc: { [key: string]: ExtendedRelationship }, relType: Relationship) => { + const key = relType.caption || ''; + if (!acc[key]) { + acc[key] = { ...relType, count: 0 }; + } + + acc[key]!.count += relationshipCount(relationships as ExtendedRelationship[], key); + return acc; + }, {}) + ); + + // On Node Click, highlighting the nodes and deactivating any active relationships + const handleNodeClick = (nodeLabel: string) => { + const updatedNodes = nodes.map((node) => { + const isActive = node.labels.includes(nodeLabel); + return { + ...node, + activated: isActive, + selected: isActive, + size: + isActive && viewPoint === graphLabels.showGraphView + ? 100 + : isActive && viewPoint !== graphLabels.showGraphView + ? 50 + : graphLabels.nodeSize, + }; + }); + // deactivating any active relationships + const updatedRelationships = relationships.map((rel) => { + return { + ...rel, + activated: false, + selected: false, + }; + }); + if (searchQuery !== '') { + setSearchQuery(''); } + setNodes(updatedNodes); + setRelationships(updatedRelationships); + }; + // On Relationship Legend Click, highlight the relationships and deactivating any active nodes + const handleRelationshipClick = (nodeLabel: string) => { + const updatedRelations = relationships.map((rel) => { + return { + ...rel, + activated: rel?.caption?.includes(nodeLabel), + selected: rel?.caption?.includes(nodeLabel), + }; + }); + // // deactivating any active nodes + const updatedNodes = nodes.map((node) => { + return { + ...node, + activated: false, + selected: false, + size: graphLabels.nodeSize, + }; + }); + if (searchQuery !== '') { + setSearchQuery(''); + } + setRelationships(updatedRelations); + setNodes(updatedNodes); }; return ( @@ -334,17 +496,79 @@ const GraphViewModal: React.FunctionComponent = ({ handleClasses={{ left: 'ml-1' }} >
- - {graphLabels.resultOverview} - - {graphLabels.totalNodes} ({nodes.length}) - - -
- {legendCheck.map((key, index) => ( - - ))} -
+ {nodeCheck.length > 0 && ( + <> + + {graphLabels.resultOverview} +
+ { + setSearchQuery(e.target.value); + }} + placeholder='Search On Node Properties' + fluid={true} + leftIcon={ + + + + } + /> +
+ + {graphLabels.totalNodes} ({nodes.length}) + + +
+ + {nodeCheck.map((nodeLabel, index) => ( + handleNodeClick(nodeLabel)} + /> + ))} + +
+ + )} + {relationshipsSorted.length > 0 && ( + <> + + + {graphLabels.totalRelationships} ({relationships.length}) + + +
+ + {groupedAndSortedRelationships.map((relType, index) => ( + handleRelationshipClick(relType.caption || '')} + /> + ))} + +
+ + )}
diff --git a/frontend/src/components/Graph/LegendsChip.tsx b/frontend/src/components/Graph/LegendsChip.tsx index 7e121dc67..2ab9c7b40 100644 --- a/frontend/src/components/Graph/LegendsChip.tsx +++ b/frontend/src/components/Graph/LegendsChip.tsx @@ -1,12 +1,6 @@ -import { useMemo } from 'react'; import { LegendChipProps } from '../../types'; import Legend from '../UI/Legend'; -export const LegendsChip: React.FunctionComponent = ({ scheme, title, nodes }) => { - const chunkcount = useMemo( - () => [...new Set(nodes?.filter((n) => n?.labels?.includes(title)).map((i) => i.id))].length, - [nodes] - ); - - return ; +export const LegendsChip: React.FunctionComponent = ({ scheme, label, type, count, onClick }) => { + return ; }; diff --git a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx index 63e2d7883..ab9fe2399 100644 --- a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx +++ b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx @@ -60,41 +60,43 @@ export default function ConnectionModal({ }, [open]); const recreateVectorIndex = useCallback( - async (isNewVectorIndex: boolean) => { - try { - setVectorIndexLoading(true); - const response = await createVectorIndex(userCredentials as UserCredentials, isNewVectorIndex); - setVectorIndexLoading(false); - if (response.data.status === 'Failed') { - throw new Error(response.data.error); - } else { - setMessage({ - type: 'success', - content: 'Successfully created the vector index', - }); - setConnectionStatus(true); - localStorage.setItem( - 'neo4j.connection', - JSON.stringify({ - uri: userCredentials?.uri, - user: userCredentials?.userName, - password: userCredentials?.password, - database: userCredentials?.database, - userDbVectorIndex: 384, - }) - ); - } - } catch (error) { - setVectorIndexLoading(false); - if (error instanceof Error) { - console.log('Error in recreating the vector index', error.message); - setMessage({ type: 'danger', content: error.message }); + async (isNewVectorIndex: boolean, usercredential: UserCredentials) => { + if (usercredential != null && Object.values(usercredential).length) { + try { + setVectorIndexLoading(true); + const response = await createVectorIndex(usercredential as UserCredentials, isNewVectorIndex); + setVectorIndexLoading(false); + if (response.data.status === 'Failed') { + throw new Error(response.data.error); + } else { + setMessage({ + type: 'success', + content: 'Successfully created the vector index', + }); + setConnectionStatus(true); + localStorage.setItem( + 'neo4j.connection', + JSON.stringify({ + uri: usercredential?.uri, + user: usercredential?.userName, + password: usercredential?.password, + database: usercredential?.database, + userDbVectorIndex: 384, + }) + ); + } + } catch (error) { + setVectorIndexLoading(false); + if (error instanceof Error) { + console.log('Error in recreating the vector index', error.message); + setMessage({ type: 'danger', content: error.message }); + } } + setTimeout(() => { + setMessage({ type: 'unknown', content: '' }); + setOpenConnection((prev) => ({ ...prev, openPopUp: false })); + }, 3000); } - setTimeout(() => { - setMessage({ type: 'unknown', content: '' }); - setOpenConnection((prev) => ({ ...prev, openPopUp: false })); - }, 3000); }, [userCredentials, userDbVectorIndex] ); @@ -104,8 +106,9 @@ export default function ConnectionModal({ type: 'danger', content: ( recreateVectorIndex(chunksExistsWithDifferentEmbedding)} + recreateVectorIndex={() => + recreateVectorIndex(chunksExistsWithDifferentEmbedding, userCredentials as UserCredentials) + } isVectorIndexAlreadyExists={chunksExistsWithDifferentEmbedding || isVectorIndexMatch} userVectorIndexDimension={JSON.parse(localStorage.getItem('neo4j.connection') ?? 'null').userDbVectorIndex} chunksExists={chunksExistsWithoutEmbedding} @@ -113,7 +116,7 @@ export default function ConnectionModal({ ), }); } - }, [isVectorIndexMatch, vectorIndexLoading, chunksExistsWithDifferentEmbedding, chunksExistsWithoutEmbedding]); + }, [isVectorIndexMatch, chunksExistsWithDifferentEmbedding, chunksExistsWithoutEmbedding, userCredentials]); const parseAndSetURI = (uri: string, urlparams = false) => { const uriParts: string[] = uri.split('://'); @@ -189,7 +192,8 @@ export default function ConnectionModal({ const submitConnection = async () => { const connectionURI = `${protocol}://${URI}${URI.split(':')[1] ? '' : `:${port}`}`; - setUserCredentials({ uri: connectionURI, userName: username, password: password, database: database, port: port }); + const credential = { uri: connectionURI, userName: username, password: password, database: database, port: port }; + setUserCredentials(credential); setIsLoading(true); try { const response = await connectAPI(connectionURI, username, password, database); @@ -197,6 +201,16 @@ export default function ConnectionModal({ if (response?.data?.status !== 'Success') { throw new Error(response.data.error); } else { + localStorage.setItem( + 'neo4j.connection', + JSON.stringify({ + uri: connectionURI, + user: username, + password: password, + database: database, + userDbVectorIndex, + }) + ); setUserDbVectorIndex(response.data.data.db_vector_dimension); if ( (response.data.data.application_dimension === response.data.data.db_vector_dimension || @@ -214,27 +228,19 @@ export default function ConnectionModal({ type: 'danger', content: ( - recreateVectorIndex( - !( - response.data.data.db_vector_dimension > 0 && - response.data.data.db_vector_dimension != response.data.data.application_dimension - ) - ) - } + recreateVectorIndex={() => recreateVectorIndex(false, credential)} isVectorIndexAlreadyExists={response.data.data.db_vector_dimension != 0} chunksExists={true} /> ), }); + return; } else { setMessage({ type: 'danger', content: ( recreateVectorIndex(true)} + recreateVectorIndex={() => recreateVectorIndex(true, credential)} isVectorIndexAlreadyExists={ response.data.data.db_vector_dimension != 0 && response.data.data.db_vector_dimension != response.data.data.application_dimension @@ -244,17 +250,8 @@ export default function ConnectionModal({ /> ), }); + return; } - localStorage.setItem( - 'neo4j.connection', - JSON.stringify({ - uri: connectionURI, - user: username, - password: password, - database: database, - userDbVectorIndex, - }) - ); } } catch (error) { setIsLoading(false); @@ -266,6 +263,9 @@ export default function ConnectionModal({ } } setTimeout(() => { + if (connectionMessage?.type != 'danger') { + setMessage({ type: 'unknown', content: '' }); + } setPassword(''); }, 3000); }; diff --git a/frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx b/frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx index f9e85b63e..3c2965f44 100644 --- a/frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx +++ b/frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx @@ -2,21 +2,22 @@ import { Box, Flex } from '@neo4j-ndl/react'; import Markdown from 'react-markdown'; import ButtonWithToolTip from '../../UI/ButtonWithToolTip'; import { useCredentials } from '../../../context/UserCredentials'; +import { useState } from 'react'; export default function VectorIndexMisMatchAlert({ - vectorIndexLoading, recreateVectorIndex, isVectorIndexAlreadyExists, userVectorIndexDimension, chunksExists, }: { - vectorIndexLoading: boolean; recreateVectorIndex: () => Promise; isVectorIndexAlreadyExists: boolean; userVectorIndexDimension?: number; chunksExists: boolean; }) { const { userCredentials } = useCredentials(); + const [vectorIndexLoading, setVectorIndexLoading] = useState(false); + return ( @@ -42,7 +43,11 @@ To proceed, please choose one of the following options: label='creates the supported vector index' placement='top' loading={vectorIndexLoading} - onClick={() => recreateVectorIndex()} + onClick={async () => { + setVectorIndexLoading(true); + await recreateVectorIndex(); + setVectorIndexLoading(false); + }} className='!w-full' color='danger' disabled={userCredentials === null} diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/Deduplication/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/Deduplication/index.tsx index bf8345550..f5a021e30 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/Deduplication/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/Deduplication/index.tsx @@ -160,7 +160,7 @@ export default function DeduplicationTab() { return ( {info.getValue().map((l, index) => ( - + ))} ); diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx index 2dc8a13ff..373007e95 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx @@ -111,7 +111,7 @@ export default function DeletePopUpForOrphanNodes({ return ( {info.getValue().map((l, index) => ( - + ))} ); diff --git a/frontend/src/components/QuickStarter.tsx b/frontend/src/components/QuickStarter.tsx index 29832fee1..db2cba7e7 100644 --- a/frontend/src/components/QuickStarter.tsx +++ b/frontend/src/components/QuickStarter.tsx @@ -42,4 +42,4 @@ const QuickStarter: React.FunctionComponent = () => { ); }; -export default QuickStarter; \ No newline at end of file +export default QuickStarter; diff --git a/frontend/src/components/UI/Legend.tsx b/frontend/src/components/UI/Legend.tsx index d798ff4f4..a3c806669 100644 --- a/frontend/src/components/UI/Legend.tsx +++ b/frontend/src/components/UI/Legend.tsx @@ -3,16 +3,20 @@ import { GraphLabel } from '@neo4j-ndl/react'; export default function Legend({ bgColor, title, - chunkCount, + count, + type, + onClick, }: { bgColor: string; title: string; - chunkCount?: number; + count?: number; + type: 'node' | 'relationship' | 'propertyKey'; + tabIndex?: number; + onClick?: (e: React.MouseEvent) => void; }) { return ( - - {title} - {chunkCount && `(${chunkCount})`} + + {title} {count !== undefined && `(${count})`} ); } diff --git a/frontend/src/components/UI/ShowAll.tsx b/frontend/src/components/UI/ShowAll.tsx new file mode 100644 index 000000000..726146723 --- /dev/null +++ b/frontend/src/components/UI/ShowAll.tsx @@ -0,0 +1,38 @@ +import { Button } from '@neo4j-ndl/react'; +import type { ReactNode } from 'react'; +import { useState } from 'react'; + +// import { ButtonGroup } from '../button-group/button-group'; + +type ShowAllProps = { + initiallyShown: number; + /* pass thunk to enable rendering only shown components */ + children: ((() => ReactNode) | ReactNode)[]; + ariaLabel?: string; +}; +const isThunkComponent = (t: (() => ReactNode) | ReactNode): t is () => ReactNode => typeof t === 'function'; + +export function ShowAll({ initiallyShown, children }: ShowAllProps) { + const [expanded, setExpanded] = useState(false); + const toggleExpanded = () => setExpanded((e) => !e); + const itemCount = children.length; + const controlsNeeded = itemCount > initiallyShown; + const shown = expanded ? itemCount : initiallyShown; + const leftToShow = itemCount - shown; + + if (itemCount === 0) { + return null; + } + + const currentChildren = children.slice(0, shown).map((c) => (isThunkComponent(c) ? c() : c)); + return ( + <> +
{currentChildren}
+ {controlsNeeded && ( + + )} + + ); +} diff --git a/frontend/src/context/UsersFiles.tsx b/frontend/src/context/UsersFiles.tsx index 75f965d0a..fd3531403 100644 --- a/frontend/src/context/UsersFiles.tsx +++ b/frontend/src/context/UsersFiles.tsx @@ -58,7 +58,7 @@ const FileContextProvider: FC = ({ children }) => { const [selectedSchemas, setSelectedSchemas] = useState([]); const [rowSelection, setRowSelection] = useState>({}); const [selectedRows, setSelectedRows] = useState([]); - const [chatMode, setchatMode] = useState('graph+vector'); + const [chatMode, setchatMode] = useState('graph+vector+fulltext'); const [isSchema, setIsSchema] = useState(false); const [showTextFromSchemaDialog, setShowTextFromSchemaDialog] = useState({ triggeredFrom: '', diff --git a/frontend/src/hooks/useSse.tsx b/frontend/src/hooks/useSse.tsx index f8a07f61e..8b063751c 100644 --- a/frontend/src/hooks/useSse.tsx +++ b/frontend/src/hooks/useSse.tsx @@ -7,7 +7,7 @@ export default function useServerSideEvent( alertHandler: (inMinutes: boolean, minutes: number, filename: string) => void, errorHandler: (filename: string) => void ) { - const { setFilesData, setProcessedCount, queue } = useFileContext(); + const { setFilesData, setProcessedCount } = useFileContext(); function updateStatusForLargeFiles(eventSourceRes: eventResponsetypes) { const { fileName, @@ -45,7 +45,7 @@ export default function useServerSideEvent( }); }); } - } else if (status === 'Completed' || status === 'Cancelled') { + } else if (status === 'Completed') { setFilesData((prevfiles) => { return prevfiles.map((curfile) => { if (curfile.name == fileName) { @@ -67,7 +67,6 @@ export default function useServerSideEvent( } return prev + 1; }); - queue.remove(fileName); } else if (eventSourceRes.status === 'Failed') { setFilesData((prevfiles) => { return prevfiles.map((curfile) => { diff --git a/frontend/src/services/CancelAPI.ts b/frontend/src/services/CancelAPI.ts index a162bed83..de3ea9ba0 100644 --- a/frontend/src/services/CancelAPI.ts +++ b/frontend/src/services/CancelAPI.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { UserCredentials, commonserverresponse } from '../types'; +import api from '../API/Index'; const cancelAPI = async (filenames: string[], source_types: string[]) => { try { @@ -14,7 +13,7 @@ const cancelAPI = async (filenames: string[], source_types: string[]) => { } formData.append('filenames', JSON.stringify(filenames)); formData.append('source_types', JSON.stringify(source_types)); - const response = await axios.post(`${url()}/cancelled_job`, formData); + const response = await api.post(`/cancelled_job`, formData); return response; } catch (error) { console.log('Error Posting the Question:', error); diff --git a/frontend/src/services/ChunkEntitiesInfo.ts b/frontend/src/services/ChunkEntitiesInfo.ts index 3b4323197..aa133c815 100644 --- a/frontend/src/services/ChunkEntitiesInfo.ts +++ b/frontend/src/services/ChunkEntitiesInfo.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { ChatInfo_APIResponse, UserCredentials } from '../types'; +import api from '../API/Index'; const chunkEntitiesAPI = async (userCredentials: UserCredentials, chunk_ids: string) => { try { @@ -10,7 +9,7 @@ const chunkEntitiesAPI = async (userCredentials: UserCredentials, chunk_ids: str formData.append('password', userCredentials?.password ?? ''); formData.append('chunk_ids', chunk_ids); - const response: ChatInfo_APIResponse = await axios.post(`${url()}/chunk_entities`, formData, { + const response: ChatInfo_APIResponse = await api.post(`/chunk_entities`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/CommonAPI.ts b/frontend/src/services/CommonAPI.ts index 78d324248..bbae83032 100644 --- a/frontend/src/services/CommonAPI.ts +++ b/frontend/src/services/CommonAPI.ts @@ -1,5 +1,6 @@ -import axios, { AxiosResponse, Method } from 'axios'; +import { AxiosResponse, Method } from 'axios'; import { UserCredentials, FormDataParams } from '../types'; +import api from '../API/Index'; // API Call const apiCall = async ( @@ -16,7 +17,7 @@ const apiCall = async ( for (const key in additionalParams) { formData.append(key, additionalParams[key]); } - const response: AxiosResponse = await axios({ + const response: AxiosResponse = await api({ method: method, url: url, data: formData, diff --git a/frontend/src/services/ConnectAPI.ts b/frontend/src/services/ConnectAPI.ts index 73d997b1e..026c41c44 100644 --- a/frontend/src/services/ConnectAPI.ts +++ b/frontend/src/services/ConnectAPI.ts @@ -1,5 +1,4 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; +import api from '../API/Index'; const connectAPI = async (connectionURI: string, username: string, password: string, database: string) => { try { @@ -8,7 +7,7 @@ const connectAPI = async (connectionURI: string, username: string, password: str formData.append('database', database ?? ''); formData.append('userName', username ?? ''); formData.append('password', password ?? ''); - const response = await axios.post(`${url()}/connect`, formData, { + const response = await api.post(`/connect`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/DeleteFiles.ts b/frontend/src/services/DeleteFiles.ts index 36ae28cbd..a86ef187e 100644 --- a/frontend/src/services/DeleteFiles.ts +++ b/frontend/src/services/DeleteFiles.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { CustomFile, UserCredentials } from '../types'; +import api from '../API/Index'; const deleteAPI = async (userCredentials: UserCredentials, selectedFiles: CustomFile[], deleteEntities: boolean) => { try { @@ -14,7 +13,7 @@ const deleteAPI = async (userCredentials: UserCredentials, selectedFiles: Custom formData.append('deleteEntities', JSON.stringify(deleteEntities)); formData.append('filenames', JSON.stringify(filenames)); formData.append('source_types', JSON.stringify(source_types)); - const response = await axios.post(`${url()}/delete_document_and_entities`, formData); + const response = await api.post(`/delete_document_and_entities`, formData); return response; } catch (error) { console.log('Error Posting the Question:', error); diff --git a/frontend/src/services/DeleteOrphanNodes.ts b/frontend/src/services/DeleteOrphanNodes.ts index 135dfcdeb..2cebc572c 100644 --- a/frontend/src/services/DeleteOrphanNodes.ts +++ b/frontend/src/services/DeleteOrphanNodes.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { UserCredentials } from '../types'; +import api from '../API/Index'; const deleteOrphanAPI = async (userCredentials: UserCredentials, selectedNodes: string[]) => { try { @@ -10,7 +9,7 @@ const deleteOrphanAPI = async (userCredentials: UserCredentials, selectedNodes: formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('unconnected_entities_list', JSON.stringify(selectedNodes)); - const response = await axios.post(`${url()}/delete_unconnected_nodes`, formData); + const response = await api.post(`/delete_unconnected_nodes`, formData); return response; } catch (error) { console.log('Error Posting the Question:', error); diff --git a/frontend/src/services/GetDuplicateNodes.ts b/frontend/src/services/GetDuplicateNodes.ts index 3cb27a970..b7ea0c426 100644 --- a/frontend/src/services/GetDuplicateNodes.ts +++ b/frontend/src/services/GetDuplicateNodes.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { duplicateNodesData, UserCredentials } from '../types'; +import api from '../API/Index'; export const getDuplicateNodes = async (userCredentials: UserCredentials) => { const formData = new FormData(); @@ -9,7 +8,7 @@ export const getDuplicateNodes = async (userCredentials: UserCredentials) => { formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); try { - const response = await axios.post(`${url()}/get_duplicate_nodes`, formData); + const response = await api.post(`/get_duplicate_nodes`, formData); return response; } catch (error) { console.log(error); diff --git a/frontend/src/services/GetFiles.ts b/frontend/src/services/GetFiles.ts index fd8d60fba..056a9cc05 100644 --- a/frontend/src/services/GetFiles.ts +++ b/frontend/src/services/GetFiles.ts @@ -1,14 +1,11 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { SourceListServerData, UserCredentials } from '../types'; +import api from '../API/Index'; export const getSourceNodes = async (userCredentials: UserCredentials) => { try { const encodedstr = btoa(userCredentials.password); - const response = await axios.get( - `${url()}/sources_list?uri=${userCredentials.uri}&database=${userCredentials.database}&userName=${ - userCredentials.userName - }&password=${encodedstr}` + const response = await api.get( + `/sources_list?uri=${userCredentials.uri}&database=${userCredentials.database}&userName=${userCredentials.userName}&password=${encodedstr}` ); return response; } catch (error) { diff --git a/frontend/src/services/GetNodeLabelsRelTypes.ts b/frontend/src/services/GetNodeLabelsRelTypes.ts index acc05f267..8c7345c2a 100644 --- a/frontend/src/services/GetNodeLabelsRelTypes.ts +++ b/frontend/src/services/GetNodeLabelsRelTypes.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { ServerData, UserCredentials } from '../types'; +import api from '../API/Index'; export const getNodeLabelsAndRelTypes = async (userCredentials: UserCredentials) => { const formData = new FormData(); @@ -9,7 +8,7 @@ export const getNodeLabelsAndRelTypes = async (userCredentials: UserCredentials) formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); try { - const response = await axios.post(`${url()}/schema`, formData); + const response = await api.post(`/schema`, formData); return response; } catch (error) { console.log(error); diff --git a/frontend/src/services/GetOrphanNodes.ts b/frontend/src/services/GetOrphanNodes.ts index 70cfe8cda..3578214d1 100644 --- a/frontend/src/services/GetOrphanNodes.ts +++ b/frontend/src/services/GetOrphanNodes.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { OrphanNodeResponse, UserCredentials } from '../types'; +import api from '../API/Index'; export const getOrphanNodes = async (userCredentials: UserCredentials) => { const formData = new FormData(); @@ -9,7 +8,7 @@ export const getOrphanNodes = async (userCredentials: UserCredentials) => { formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); try { - const response = await axios.post(`${url()}/get_unconnected_nodes_list`, formData); + const response = await api.post(`/get_unconnected_nodes_list`, formData); return response; } catch (error) { console.log(error); diff --git a/frontend/src/services/GraphQuery.ts b/frontend/src/services/GraphQuery.ts index 55d22a5ee..f792f4aec 100644 --- a/frontend/src/services/GraphQuery.ts +++ b/frontend/src/services/GraphQuery.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { UserCredentials } from '../types'; +import api from '../API/Index'; const graphQueryAPI = async ( userCredentials: UserCredentials, @@ -16,7 +15,7 @@ const graphQueryAPI = async ( formData.append('query_type', query_type ?? 'entities'); formData.append('document_names', JSON.stringify(document_names)); - const response = await axios.post(`${url()}/graph_query`, formData, { + const response = await api.post(`/graph_query`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/HealthStatus.ts b/frontend/src/services/HealthStatus.ts index 69a77f0fe..a59badbe0 100644 --- a/frontend/src/services/HealthStatus.ts +++ b/frontend/src/services/HealthStatus.ts @@ -1,9 +1,8 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; +import api from '../API/Index'; const healthStatus = async () => { try { - const healthUrl = `${url()}/health`; - const response = await axios.get(healthUrl); + const healthUrl = `/health`; + const response = await api.get(healthUrl); return response; } catch (error) { console.log('API status error', error); diff --git a/frontend/src/services/MergeDuplicateEntities.ts b/frontend/src/services/MergeDuplicateEntities.ts index ee4e0fffb..1fdb9b387 100644 --- a/frontend/src/services/MergeDuplicateEntities.ts +++ b/frontend/src/services/MergeDuplicateEntities.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { commonserverresponse, selectedDuplicateNodes, UserCredentials } from '../types'; +import api from '../API/Index'; const mergeDuplicateNodes = async (userCredentials: UserCredentials, selectedNodes: selectedDuplicateNodes[]) => { try { @@ -10,7 +9,7 @@ const mergeDuplicateNodes = async (userCredentials: UserCredentials, selectedNod formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('duplicate_nodes_list', JSON.stringify(selectedNodes)); - const response = await axios.post(`${url()}/merge_duplicate_nodes`, formData); + const response = await api.post(`/merge_duplicate_nodes`, formData); return response; } catch (error) { console.log('Error Merging the duplicate nodes:', error); diff --git a/frontend/src/services/PollingAPI.ts b/frontend/src/services/PollingAPI.ts index 08e6a11bb..f14ed0580 100644 --- a/frontend/src/services/PollingAPI.ts +++ b/frontend/src/services/PollingAPI.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { PollingAPI_Response, statusupdate } from '../types'; +import api from '../API/Index'; export default async function subscribe( fileName: string, @@ -15,12 +14,12 @@ export default async function subscribe( const MAX_POLLING_ATTEMPTS = 10; let pollingAttempts = 0; - let delay = 2000; + let delay = 1000; while (pollingAttempts < MAX_POLLING_ATTEMPTS) { let currentdelay = delay; - let response: PollingAPI_Response = await axios.get( - `${url()}/document_status/${fileName}?url=${uri}&userName=${username}&password=${encodedstr}&database=${database}` + let response: PollingAPI_Response = await api.get( + `/document_status/${fileName}?url=${uri}&userName=${username}&password=${encodedstr}&database=${database}` ); if (response.data?.file_name?.status === 'Processing') { diff --git a/frontend/src/services/PostProcessing.ts b/frontend/src/services/PostProcessing.ts index 9f45cc6bd..98c94c238 100644 --- a/frontend/src/services/PostProcessing.ts +++ b/frontend/src/services/PostProcessing.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { UserCredentials } from '../types'; +import api from '../API/Index'; const postProcessing = async (userCredentials: UserCredentials, taskParam: string[]) => { try { @@ -10,7 +9,7 @@ const postProcessing = async (userCredentials: UserCredentials, taskParam: strin formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('tasks', JSON.stringify(taskParam)); - const response = await axios.post(`${url()}/post_processing`, formData, { + const response = await api.post(`/post_processing`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/QnaAPI.ts b/frontend/src/services/QnaAPI.ts index 931618839..78fa240ba 100644 --- a/frontend/src/services/QnaAPI.ts +++ b/frontend/src/services/QnaAPI.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { UserCredentials } from '../types'; +import api from '../API/Index'; export const chatBotAPI = async ( userCredentials: UserCredentials, @@ -22,7 +21,7 @@ export const chatBotAPI = async ( formData.append('mode', mode); formData.append('document_names', JSON.stringify(document_names)); const startTime = Date.now(); - const response = await axios.post(`${url()}/chat_bot`, formData, { + const response = await api.post(`/chat_bot`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, @@ -44,7 +43,7 @@ export const clearChatAPI = async (userCredentials: UserCredentials, session_id: formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('session_id', session_id); - const response = await axios.post(`${url()}/clear_chat_bot`, formData, { + const response = await api.post(`/clear_chat_bot`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/SchemaFromTextAPI.ts b/frontend/src/services/SchemaFromTextAPI.ts index 785fb0d68..3d1984ccf 100644 --- a/frontend/src/services/SchemaFromTextAPI.ts +++ b/frontend/src/services/SchemaFromTextAPI.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { ScehmaFromText } from '../types'; +import api from '../API/Index'; export const getNodeLabelsAndRelTypesFromText = async (model: string, inputText: string, isSchemaText: boolean) => { const formData = new FormData(); @@ -9,7 +8,7 @@ export const getNodeLabelsAndRelTypesFromText = async (model: string, inputText: formData.append('is_schema_description_checked', JSON.stringify(isSchemaText)); try { - const response = await axios.post(`${url()}/populate_graph_schema`, formData); + const response = await api.post(`/populate_graph_schema`, formData); return response; } catch (error) { console.log(error); diff --git a/frontend/src/services/URLScan.ts b/frontend/src/services/URLScan.ts index 58ad37dfb..444022934 100644 --- a/frontend/src/services/URLScan.ts +++ b/frontend/src/services/URLScan.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { ScanProps, ServerResponse } from '../types'; +import api from '../API/Index'; const urlScanAPI = async (props: ScanProps) => { try { @@ -46,7 +45,7 @@ const urlScanAPI = async (props: ScanProps) => { formData.append('access_token', props.access_token); } - const response: ServerResponse = await axios.post(`${url()}/url/scan`, formData, { + const response: ServerResponse = await api.post(`/url/scan`, formData, { headers: { 'Content-Type': 'multipart/form-data', }, diff --git a/frontend/src/services/vectorIndexCreation.ts b/frontend/src/services/vectorIndexCreation.ts index accf85659..e89767551 100644 --- a/frontend/src/services/vectorIndexCreation.ts +++ b/frontend/src/services/vectorIndexCreation.ts @@ -1,6 +1,5 @@ -import axios from 'axios'; -import { url } from '../utils/Utils'; import { commonserverresponse, UserCredentials } from '../types'; +import api from '../API/Index'; export const createVectorIndex = async (userCredentials: UserCredentials, isVectorIndexExists: boolean) => { const formData = new FormData(); @@ -10,7 +9,7 @@ export const createVectorIndex = async (userCredentials: UserCredentials, isVect formData.append('password', userCredentials?.password ?? ''); formData.append('isVectorIndexExist', JSON.stringify(isVectorIndexExists)); try { - const response = await axios.post(`${url()}/drop_create_vector_index`, formData); + const response = await api.post(`/drop_create_vector_index`, formData); return response; } catch (error) { console.log(error); diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8e05f632b..5bf076d10 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -259,7 +259,7 @@ export interface GraphViewModalProps { setGraphViewOpen: Dispatch>; viewPoint: string; nodeValues?: ExtendedNode[]; - relationshipValues?: Relationship[]; + relationshipValues?: ExtendedRelationship[]; selectedRows?: CustomFile[] | undefined; } @@ -344,13 +344,13 @@ export type alertStateType = { export type Scheme = Record; export type LabelCount = Record; -interface NodeType extends Partial { - labels?: string[]; -} + export interface LegendChipProps { scheme: Scheme; - title: string; - nodes: NodeType[]; + label: string; + type: 'node' | 'relationship' | 'propertyKey'; + count: number; + onClick: (e: React.MouseEvent) => void; } export interface FileContextProviderProps { children: ReactNode; @@ -578,29 +578,6 @@ export type GraphStatsLabels = Record< } >; -type NodeStyling = { - backgroundColor: string; - borderColor: string; - textColor: string; - caption: string; - diameter: string; -}; - -type RelationStyling = { - fontSize: string; - lineColor: string; - textColorExternal: string; - textColorInternal: string; - caption: string; - padding: string; - width: string; -}; - -export type GraphStyling = { - node: Record>; - relationship: Record>; -}; - export interface ExtendedNode extends Node { labels: string[]; properties: { @@ -610,7 +587,7 @@ export interface ExtendedNode extends Node { } export interface ExtendedRelationship extends Relationship { - labels: string[]; + count: number; } export interface connectionState { openPopUp: boolean; @@ -655,7 +632,7 @@ export interface S3File { } export interface GraphViewButtonProps { nodeValues?: ExtendedNode[]; - relationshipValues?: Relationship[]; + relationshipValues?: ExtendedRelationship[]; } export interface DrawerChatbotProps { isExpanded: boolean; diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index b4781b929..47f1e119f 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -51,15 +51,15 @@ export const llms = 'bedrock_claude_3_5_sonnet', ]; -export const defaultLLM = llms?.includes('openai-gpt-4o-mini') - ? 'openai-gpt-4o-mini' +export const defaultLLM = llms?.includes('openai-gpt-4o') + ? 'openai-gpt-4o' : llms?.includes('gemini-1.0-pro') ? 'gemini-1.0-pro' : 'diffbot'; export const chatModes = process.env?.VITE_CHAT_MODES?.trim() != '' ? process.env.VITE_CHAT_MODES?.split(',') - : ['vector', 'graph', 'graph+vector', 'hybrid', 'hybrid+graph']; + : ['vector', 'graph', 'graph+vector', 'fulltext', 'graph+vector+fulltext']; export const chunkSize = process.env.VITE_CHUNK_SIZE ? parseInt(process.env.VITE_CHUNK_SIZE) : 1 * 1024 * 1024; export const timeperpage = process.env.VITE_TIME_PER_PAGE ? parseInt(process.env.VITE_TIME_PER_PAGE) : 50; export const timePerByte = 0.2; @@ -231,4 +231,8 @@ export const graphLabels = { totalNodes: 'Total Nodes', noEntities: 'No Entities Found', selectCheckbox: 'Select atleast one checkbox for graph view', + totalRelationships: 'Total Relationships', + nodeSize: 30, }; + +export const RESULT_STEP_SIZE = 25; diff --git a/frontend/src/utils/Utils.ts b/frontend/src/utils/Utils.ts index 9745993f6..0b8b05841 100644 --- a/frontend/src/utils/Utils.ts +++ b/frontend/src/utils/Utils.ts @@ -1,6 +1,6 @@ import { calcWordColor } from '@neo4j-devtools/word-color'; import type { Relationship } from '@neo4j-nvl/base'; -import { Entity, ExtendedNode, GraphType, Messages, Scheme } from '../types'; +import { Entity, ExtendedNode, ExtendedRelationship, GraphType, Messages, Scheme } from '../types'; // Get the Url export const url = () => { @@ -130,7 +130,7 @@ export function extractPdfFileName(url: string): string { return decodedFileName; } -export const processGraphData = (neoNodes: ExtendedNode[], neoRels: Relationship[]) => { +export const processGraphData = (neoNodes: ExtendedNode[], neoRels: ExtendedRelationship[]) => { const schemeVal: Scheme = {}; let iterator = 0; const labels: string[] = neoNodes.map((f: any) => f.labels); @@ -239,3 +239,9 @@ export const parseEntity = (entity: Entity) => { export const titleCheck = (title: string) => { return title === 'Chunk' || title === 'Document'; }; + +export const sortAlphabetically = (a: Relationship, b: Relationship) => { + const captionOne = a.caption?.toLowerCase() || ''; + const captionTwo = b.caption?.toLowerCase() || ''; + return captionOne.localeCompare(captionTwo); +}; diff --git a/frontend/yarn.lock b/frontend/yarn.lock index c91c34832..3915845ef 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -6614,4 +6614,4 @@ yocto-queue@^0.1.0: zwitch@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7" - integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== + integrity sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== \ No newline at end of file