Chunks to be created (#1015)

* Restrict chunks creation configurable

* Added chunk_to_be_created en value in connect endpoint

* Added Chunks To be Processed

* Update Content.tsx

* Conditional display

---------

Co-authored-by: Pravesh Kumar <[email protected]>

Author: kartikpersistent

backend/example.env

@@ -44,7 +44,7 @@ LLM_MODEL_CONFIG_ollama_llama3="model_name,model_local_url"
 YOUTUBE_TRANSCRIPT_PROXY="https://user:pass@domain:port"
 EFFECTIVE_SEARCH_RATIO=5
 GRAPH_CLEANUP_MODEL="openai_gpt_4o"
-CHUNKS_TO_BE_PROCESSED="50"
+CHUNKS_TO_BE_CREATED="50"
 BEDROCK_EMBEDDING_MODEL="model_name,aws_access_key,aws_secret_key,region_name"                       #model_name="amazon.titan-embed-text-v1"
 LLM_MODEL_CONFIG_bedrock_nova_micro_v1="model_name,aws_access_key,aws_secret_key,region_name"        #model_name="amazon.nova-micro-v1:0"
 LLM_MODEL_CONFIG_bedrock_nova_lite_v1="model_name,aws_access_key,aws_secret_key,region_name"         #model_name="amazon.nova-lite-v1:0"

backend/score.py

@@ -503,12 +503,14 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form())
         graph = create_graph_database_connection(uri, userName, password, database)
         result = await asyncio.to_thread(connection_check_and_get_vector_dimensions, graph, database)
         gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
+        chunk_to_be_created = int(os.environ.get('CHUNKS_TO_BE_CREATED', '50'))
         end = time.time()
         elapsed_time = end - start
         json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'}
         logger.log_struct(json_obj, "INFO")
         result['elapsed_api_time'] = f'{elapsed_time:.2f}'
         result['gcs_file_cache'] = gcs_file_cache
+        result['chunk_to_be_created']= chunk_to_be_created
         return create_api_response('Success',data=result)
     except Exception as e:
         job_status = "Failed"
@@ -981,8 +983,8 @@ async def backend_connection_configuration():
         database= os.getenv('NEO4J_DATABASE')
         password= os.getenv('NEO4J_PASSWORD')
         gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
+        chunk_to_be_created = int(os.environ.get('CHUNKS_TO_BE_CREATED', '50'))
         if all([uri, username, database, password]):
-            print(f'uri:{uri}, usrName:{username}, database :{database}, password: {password}')
             graph = Neo4jGraph()
             logging.info(f'login connection status of object: {graph}')
             if graph is not None:
@@ -996,6 +998,7 @@ async def backend_connection_configuration():
                 result["database"] = database
                 result["password"] = encoded_password
                 result['gcs_file_cache'] = gcs_file_cache
+                result['chunk_to_be_created']= chunk_to_be_created
                 end = time.time()
                 elapsed_time = end - start
                 result['api_name'] = 'backend_connection_configuration'

backend/src/create_chunks.py

@@ -4,6 +4,7 @@
 import logging
 from src.document_sources.youtube import get_chunks_with_timestamps, get_calculated_timestamps
 import re
+import os
 
 logging.basicConfig(format="%(asctime)s - %(message)s", level="INFO")
 
@@ -25,23 +26,28 @@ def split_file_into_chunks(self):
         """
         logging.info("Split file into smaller chunks")
         text_splitter = TokenTextSplitter(chunk_size=200, chunk_overlap=20)
+        chunk_to_be_created = int(os.environ.get('CHUNKS_TO_BE_CREATED', '50'))
         if 'page' in self.pages[0].metadata:
             chunks = []
             for i, document in enumerate(self.pages):
                 page_number = i + 1
-                for chunk in text_splitter.split_documents([document]):
-                    chunks.append(Document(page_content=chunk.page_content, metadata={'page_number':page_number}))    
+                if len(chunks) >= chunk_to_be_created:
+                    break
+                else:
+                    for chunk in text_splitter.split_documents([document]):
+                        chunks.append(Document(page_content=chunk.page_content, metadata={'page_number':page_number}))    
 
         elif 'length' in self.pages[0].metadata:
             if len(self.pages) == 1  or (len(self.pages) > 1 and self.pages[1].page_content.strip() == ''): 
                 match = re.search(r'(?:v=)([0-9A-Za-z_-]{11})\s*',self.pages[0].metadata['source'])
                 youtube_id=match.group(1)   
                 chunks_without_time_range = text_splitter.split_documents([self.pages[0]])
-                chunks = get_calculated_timestamps(chunks_without_time_range, youtube_id)
-
+                chunks = get_calculated_timestamps(chunks_without_time_range[:chunk_to_be_created], youtube_id)
             else: 
-                chunks_without_time_range = text_splitter.split_documents(self.pages)   
-                chunks = get_chunks_with_timestamps(chunks_without_time_range)
+                chunks_without_time_range = text_splitter.split_documents(self.pages)
+                chunks = get_chunks_with_timestamps(chunks_without_time_range[:chunk_to_be_created])
         else:
             chunks = text_splitter.split_documents(self.pages)
+            
+        chunks = chunks[:chunk_to_be_created]
         return chunks

backend/src/main.py

@@ -361,14 +361,12 @@ async def processing_source(uri, userName, password, database, model, file_name,
 
       logging.info('Update the status as Processing')
       update_graph_chunk_processed = int(os.environ.get('UPDATE_GRAPH_CHUNKS_PROCESSED'))
-      chunk_to_be_processed = int(os.environ.get('CHUNKS_TO_BE_PROCESSED', '50'))
+      
       # selected_chunks = []
       is_cancelled_status = False
       job_status = "Completed"
       for i in range(0, len(chunkId_chunkDoc_list), update_graph_chunk_processed):
         select_chunks_upto = i+update_graph_chunk_processed
-        if select_chunks_upto > chunk_to_be_processed:
-          break
         logging.info(f'Selected Chunks upto: {select_chunks_upto}')
         if len(chunkId_chunkDoc_list) <= select_chunks_upto:
           select_chunks_upto = len(chunkId_chunkDoc_list)

frontend/src/HOC/WithVisibility.tsx

@@ -1,4 +1,4 @@
-import { VisibilityProps } from "../types";
+import { VisibilityProps } from '../types';
 
 export function withVisibility<P>(WrappedComponent: React.ComponentType<P>) {
   const VisibityControlled = (props: P & VisibilityProps) => {

frontend/src/components/Content.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useState, useMemo, useRef, Suspense, useReducer, useCallback } from 'react';
 import FileTable from './FileTable';
-import { Button, Typography, Flex, StatusIndicator, useMediaQuery } from '@neo4j-ndl/react';
+import { Button, Typography, Flex, StatusIndicator, useMediaQuery, Callout } from '@neo4j-ndl/react';
 import { useCredentials } from '../context/UserCredentials';
 import { useFileContext } from '../context/UsersFiles';
 import { extractAPI } from '../utils/FileAPI';
@@ -63,8 +63,15 @@ const Content: React.FC<ContentProps> = ({
   const [openGraphView, setOpenGraphView] = useState<boolean>(false);
   const [inspectedName, setInspectedName] = useState<string>('');
   const [documentName, setDocumentName] = useState<string>('');
-  const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser, isGCSActive } =
-    useCredentials();
+  const {
+    setUserCredentials,
+    userCredentials,
+    setConnectionStatus,
+    isGdsActive,
+    isReadOnlyUser,
+    isGCSActive,
+    chunksToBeProces,
+  } = useCredentials();
   const [showConfirmationModal, setshowConfirmationModal] = useState<boolean>(false);
   const [showExpirationModal, setshowExpirationModal] = useState<boolean>(false);
   const [extractLoading, setextractLoading] = useState<boolean>(false);
@@ -106,7 +113,7 @@ const Content: React.FC<ContentProps> = ({
   );
   const [showDeletePopUp, setshowDeletePopUp] = useState<boolean>(false);
   const [deleteLoading, setdeleteLoading] = useState<boolean>(false);
-    const hasSelections = useHasSelections(selectedNodes, selectedRels);
+  const hasSelections = useHasSelections(selectedNodes, selectedRels);
 
   const { updateStatusForLargeFiles } = useServerSideEvent(
     (inMinutes, time, fileName) => {
@@ -150,8 +157,10 @@ const Content: React.FC<ContentProps> = ({
               ? postProcessingTasks.filter((task) => task !== 'graph_schema_consolidation')
               : postProcessingTasks
             : hasSelections
-              ? postProcessingTasks.filter((task) => task !== 'graph_schema_consolidation' && task !== 'enable_communities')
-              : postProcessingTasks.filter((task) => task !== 'enable_communities');
+            ? postProcessingTasks.filter(
+                (task) => task !== 'graph_schema_consolidation' && task !== 'enable_communities'
+              )
+            : postProcessingTasks.filter((task) => task !== 'enable_communities');
           const response = await postProcessing(userCredentials as UserCredentials, payload);
           if (response.data.status === 'Success') {
             const communityfiles = response.data?.data;
@@ -381,7 +390,11 @@ const Content: React.FC<ContentProps> = ({
   const addFilesToQueue = async (remainingFiles: CustomFile[]) => {
     if (!remainingFiles.length) {
       showNormalToast(
-        <PostProcessingToast isGdsActive={isGdsActive} postProcessingTasks={postProcessingTasks} isSchema={hasSelections} />
+        <PostProcessingToast
+          isGdsActive={isGdsActive}
+          postProcessingTasks={postProcessingTasks}
+          isSchema={hasSelections}
+        />
       );
       try {
         const response = await postProcessing(userCredentials as UserCredentials, postProcessingTasks);
@@ -532,8 +545,9 @@ const Content: React.FC<ContentProps> = ({
   const handleOpenGraphClick = () => {
     const bloomUrl = process.env.VITE_BLOOM_URL;
     const uriCoded = userCredentials?.uri.replace(/:\d+$/, '');
-    const connectURL = `${uriCoded?.split('//')[0]}//${userCredentials?.userName}@${uriCoded?.split('//')[1]}:${userCredentials?.port ?? '7687'
-      }`;
+    const connectURL = `${uriCoded?.split('//')[0]}//${userCredentials?.userName}@${uriCoded?.split('//')[1]}:${
+      userCredentials?.port ?? '7687'
+    }`;
     const encodedURL = encodeURIComponent(connectURL);
     const replacedUrl = bloomUrl?.replace('{CONNECT_URL}', encodedURL);
     window.open(replacedUrl, '_blank');
@@ -586,19 +600,19 @@ const Content: React.FC<ContentProps> = ({
         (response.data?.message as string).includes('Chunks are not created')
       ) {
         showNormalToast(response.data.message as string);
-        retryOnclose()
+        retryOnclose();
       } else {
         const isStartFromBegining = retryoption === RETRY_OPIONS[0] || retryoption === RETRY_OPIONS[1];
         setFilesData((prev) => {
           return prev.map((f) => {
             return f.name === filename
               ? {
-                ...f,
-                status: 'Ready to Reprocess',
-                processingProgress: isStartFromBegining ? 0 : f.processingProgress,
-                nodesCount: isStartFromBegining ? 0 : f.nodesCount,
-                relationshipsCount: isStartFromBegining ? 0 : f.relationshipsCount,
-              }
+                  ...f,
+                  status: 'Ready to Reprocess',
+                  processingProgress: isStartFromBegining ? 0 : f.processingProgress,
+                  nodesCount: isStartFromBegining ? 0 : f.nodesCount,
+                  relationshipsCount: isStartFromBegining ? 0 : f.relationshipsCount,
+                }
               : f;
           });
         });
@@ -706,7 +720,7 @@ const Content: React.FC<ContentProps> = ({
     const selectedRows = childRef.current?.getSelectedRows();
     if (selectedRows?.length) {
       const expiredFilesExists = selectedRows.some(
-        (c) => c.status !==  'Ready to Reprocess' && isExpired(c?.createdAt as Date ?? new Date())
+        (c) => c.status !== 'Ready to Reprocess' && isExpired((c?.createdAt as Date) ?? new Date())
       );
       const largeFileExists = selectedRows.some(
         (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize
@@ -715,15 +729,12 @@ const Content: React.FC<ContentProps> = ({
         setshowExpirationModal(true);
       } else if (largeFileExists && isGCSActive) {
         setshowConfirmationModal(true);
-      } else if (largeFileExists && isGCSActive) {
         setshowExpirationModal(true);
-      } else {
+      }  else {
         handleGenerateGraph(selectedRows.filter((f) => isFileReadyToProcess(f, false)));
       }
     } else if (filesData.length) {
-      const expiredFileExists = filesData.some(
-        (c) => isExpired(c?.createdAt as Date)
-      );
+      const expiredFileExists = filesData.some((c) => isExpired(c?.createdAt as Date));
       const largeFileExists = filesData.some(
         (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize
       );
@@ -863,20 +874,12 @@ const Content: React.FC<ContentProps> = ({
                 uri={userCredentials && userCredentials?.uri}
               />
               <div className='pt-1 flex gap-1 items-center'>
+                <div>{!hasSelections ? <StatusIndicator type='danger' /> : <StatusIndicator type='success' />}</div>
                 <div>
-                  {!hasSelections ? (
-                    <StatusIndicator type='danger' />
-                  ) :
-                    (<StatusIndicator type='success' />
-                    )}
-                </div>
-                <div>
-                  {hasSelections? (
+                  {hasSelections ? (
                     <span className='n-body-small'>
-                      {(hasSelections)} Graph Schema configured
-                      {hasSelections
-                        ? `(${selectedNodes.length} Labels + ${selectedRels.length} Rel Types)`
-                        : ''}
+                      {hasSelections} Graph Schema configured
+                      {hasSelections ? `(${selectedNodes.length} Labels + ${selectedRels.length} Rel Types)` : ''}
                     </span>
                   ) : (
                     <span className='n-body-small'>No Graph Schema configured</span>
@@ -913,7 +916,15 @@ const Content: React.FC<ContentProps> = ({
               )
             )}
           </div>
+          {connectionStatus && (
+            <Callout
+              className='!w-[93%] m-auto '
+              type='note'
+              description={`Large files may be partially processed up to ${chunksToBeProces} chunks due to resource limits. If you need more comprehensive processing, consider splitting larger documents.`}
+            ></Callout>
+          )}
         </Flex>
+
         <FileTable
           connectionStatus={connectionStatus}
           setConnectionStatus={setConnectionStatus}
@@ -940,6 +951,7 @@ const Content: React.FC<ContentProps> = ({
           ref={childRef}
           handleGenerateGraph={processWaitingFilesOnRefresh}
         ></FileTable>
+
         <Flex
           className={`p-2.5  mt-1.5 absolute bottom-0 w-full`}
           justifyContent='space-between'

frontend/src/components/FileTable.tsx

@@ -1009,7 +1009,9 @@ const FileTable: ForwardRefRenderFunction<ChildRef, FileTableProps> = (props, re
             }}
             isLoading={isLoading}
             rootProps={{
-              className: `absolute h-[67%] left-10 filetable ${!largedesktops ? 'top-[17%]' : 'top-[14%]'}`,
+              className: `absolute left-10 filetable ${
+                !largedesktops && connectionStatus ? 'h-[50%]' : connectionStatus ? 'h-[60%]' : 'h-[67%]'
+              }  ${!largedesktops && connectionStatus ? 'top-[29%]' : connectionStatus ? 'top-[26%]' : 'top-[14%]'}`,
             }}
             components={{
               Body: () => (

frontend/src/components/Layout/PageLayout.tsx

@@ -62,6 +62,7 @@ const PageLayout: React.FC = () => {
     setShowDisconnectButton,
     showDisconnectButton,
     setIsGCSActive,
+    setChunksToBeProces,
   } = useCredentials();
   const { cancel } = useSpeechSynthesis();
 
@@ -85,6 +86,7 @@ const PageLayout: React.FC = () => {
         setIsGCSActive(credentials.isGCSActive ?? false);
         setGdsActive(credentials.isgdsActive);
         setIsReadOnlyUser(credentials.isReadonlyUser);
+        setChunksToBeProces(credentials.chunksTobeProcess);
         localStorage.setItem(
           'neo4j.connection',
           JSON.stringify({
@@ -96,6 +98,7 @@ const PageLayout: React.FC = () => {
             isReadOnlyUser: credentials.isReadonlyUser,
             isgdsActive: credentials.isgdsActive,
             isGCSActive: credentials.isGCSActive,
+            chunksTobeProcess: credentials.chunksTobeProcess,
           })
         );
       };
@@ -158,7 +161,9 @@ const PageLayout: React.FC = () => {
             isReadonlyUser: !connectionData.data.write_access,
             isgdsActive: connectionData.data.gds_status,
             isGCSActive: connectionData?.data?.gcs_file_cache === 'True',
+            chunksTobeProcess: parseInt(connectionData.data.chunk_to_be_created),
           };
+          setChunksToBeProces(envCredentials.chunksTobeProcess);
           setIsGCSActive(envCredentials.isGCSActive);
           if (session) {
             const updated = updateSessionIfNeeded(envCredentials, session);

frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx

@@ -49,6 +49,7 @@ export default function ConnectionModal({
     errorMessage,
     setIsGCSActive,
     setShowDisconnectButton,
+    setChunksToBeProces,
   } = useCredentials();
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [searchParams, setSearchParams] = useSearchParams();
@@ -224,10 +225,11 @@ export default function ConnectionModal({
         const isgdsActive = response.data.data.gds_status;
         const isReadOnlyUser = !response.data.data.write_access;
         const isGCSActive = response.data.data.gcs_file_cache === 'True';
+        const chunksTobeProcess = parseInt(response.data.data.chunk_to_be_created);
         setIsGCSActive(isGCSActive);
         setGdsActive(isgdsActive);
         setIsReadOnlyUser(isReadOnlyUser);
-
+        setChunksToBeProces(chunksTobeProcess);
         localStorage.setItem(
           'neo4j.connection',
           JSON.stringify({
@@ -239,6 +241,7 @@ export default function ConnectionModal({
             isgdsActive,
             isReadOnlyUser,
             isGCSActive,
+            chunksTobeProcess,
           })
         );
         setUserDbVectorIndex(response.data.data.db_vector_dimension);

frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx

@@ -29,14 +29,8 @@ export default function EntityExtractionSetting({
   closeEnhanceGraphSchemaDialog?: () => void;
 }) {
   const { breakpoints } = tokens;
-  const {
-    setSelectedRels,
-    setSelectedNodes,
-    selectedNodes,
-    selectedRels,
-    selectedSchemas,
-    setSelectedSchemas,
-  } = useFileContext();
+  const { setSelectedRels, setSelectedNodes, selectedNodes, selectedRels, selectedSchemas, setSelectedSchemas } =
+    useFileContext();
   const { userCredentials } = useCredentials();
   const [loading, setLoading] = useState<boolean>(false);
   const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`);