o }!gn` @sddlZddlZddlZddlmZmZmZddlmZddl m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0m1Z1dd l2m3Z3dd l4m5Z5edZ6e e7ee8efZ9e e)e efZ:ee#j;e jGddde,Z?e@dkr ddlAmBZBeBeeCjDjDddZEe?ZFzeFjGddZHWneIyeFJddddiddiddZHYnweHKeEdd ZLeFMeHjNeOeLdSdS)!N)BufferedIOBaseBufferedReaderBytesIO)Path)ListOptionalTypeUnion CoroutineAnyTypeVar) BaseModel) ExtractAgentExtractAgentCreate ExtractConfig ExtractJobExtractJobCreate ExtractRunExtractSchemaValidateRequestExtractAgentUpdateFile ExtractMode StatusEnumProject ExtractTargetLlamaExtractSettingsPaginatedExtractRunsResponse)AsyncLlamaCloud)JSONObjectTypeaugment_async_errors) BaseComponent)run_jobs)Field PrivateAttr)DEFAULT_BASE_URL)ThreadPoolExecutorT)extraction_targetextraction_modec@s:eZdZdZ         dGd ed ed eed eed ededede de dee dee fddZ de e e efdefddZedefddZedefddZedefddZejd eddfd!dZedefd"d#Zejd$eddfd%d#Zd&edefd'd(Zd)edeefd*d+ZdHd,d-Zd.eee efd/e!dee"e e"ffd0d1Z#d.eee efdee"e e"ffd2d3Z$d.eee efdeee effd4d5Z%d.eee efdeee effd6d7Z&d)ede"fd8d9Z'd)edefd:d;Z(dZ) @dIdAedBede*fdCdDZ+defdEdFZ,dS)JExtractionAgentzTClass representing a single extraction agent with methods for extraction operations.NTF<clientagent project_idorganization_idcheck_interval max_timeout num_workers show_progressverboseverify httpx_timeoutc Csp||_||_||_||_||_||_||_||_| |_| |_ | |_ d|_ d|_ t tdtp/ddd|_dS)N r*r, max_workers)_client_agent _project_id_organization_idr2r3r4r5r7r8_verbose _data_schema_configr%minos cpu_count _thread_pool) selfr.r/r0r1r2r3r4r5r6r7r8rHf/mnt/skqttb/ctump_chatbot/chatbot/lib/python3.10/site-packages/llama_cloud_services/extract/extract.py__init__1s  zExtractionAgent.__init__cororeturnc$dtffdd }j|S)=Run coroutine in a separate thread to avoid event loop issuesrLc dtffdd }t|S)NrLc sjjj}tjjjd4IdH"}|jj_zIdHW|jj_WdIdHS|jj_w1IdHs;wYdS)Nr7timeout)r<_client_wrapper httpx_clienthttpx AsyncClientr7r8)original_clientr.rKrGrHrI wrapped_coroTs     zFExtractionAgent._run_in_thread..run_coro..wrapped_coror&asynciorunrXrWrHrIrun_coroSs z0ExtractionAgent._run_in_thread..run_coror&rFsubmitresultrGrKr]rHrWrI_run_in_threadPszExtractionAgent._run_in_threadcC|jjSN)r=idrGrHrHrIreizExtractionAgent.idcCrcrd)r=namerfrHrHrIrhmrgzExtractionAgent.namecC|js|jjS|jSrd)rAr= data_schemarfrHrHrIrjqzExtractionAgent.data_schemarjcCs^t|tr|}nt|trt|tr|}ntd||jj j t |dd}|j |_ dS)N;data_schema must be either a dictionary or a Pydantic model)rjrequest) isinstancedicttype issubclassr model_json_schema ValueErrorrbr< llama_extractvalidate_extraction_schemarrjrA)rGrjprocessed_schemavalidated_schemarHrHrIrjus   cCrird)rBr=configrfrHrHrIryrkzExtractionAgent.configrycCs ||_dSrd)rB)rGryrHrHrIrys  file_inputc st|tr |}nt|trt|}nt|ttfr t|d}ntdz|jj j |j |dIdHWt|t r<| SSt|t rG| ww)zUpload a file for extraction.rbzJfile_input must be either a file path string, file bytes, or buffer object)r0 upload_fileN)rorbytesrstrropenrtr<filesr|r>rclose)rGrzr|rHrHrI _upload_files&       zExtractionAgent._upload_filejob_idcst}d} t|jIdH|d7}|jjj|dIdH}|jt j kr1|jjj |dIdHS|jt j krZt}|||j krItd||jrY|ddkrYtdd dd qtd |d |jd |j|jjj |dIdHS)z5Wait for and return the results of an extraction job.rTNr*rz#Timeout while extracting the file: r9.)endflushzFailure in job: z , status: z , error: )time perf_counterrZsleepr2r<ruget_jobstatusrSUCCESSget_run_by_job_idPENDINGr3 Exceptionr@printwarningswarnerror)rGrstarttriesjobrrHrHrI_wait_for_job_results4   z$ExtractionAgent._wait_for_job_resultcCs,||jjj|jt|j|jdd|_dS)zPersist the extraction agent's schema and config to the database. Returns: ExtractionAgent: The updated extraction agent )rjry)extraction_agent_idrnN) rbr<ruupdate_extraction_agentrerrjryr=rfrHrHrIsaves zExtractionAgent.saverextract_settingsc s2t|ts |g}d}nd}fdd|D}tt|jdjdIdH}Wdn1s2wYdtdtffd d fd d|D}tt|jd jdIdH}Wdn1sgwYjrt ||D]\}} t|t t frt |nd } t d| d| j qt|r|dS|S)NTFcg|]}|qSrHr.0filerfrHrI z:ExtractionAgent._queue_extraction_test..Uploading filesworkersdescr5rrLcs@jjjtj|jjjddIdH}|jIdHS)Nrfile_iddata_schema_overrideconfig_override) job_creater)r<rurun_job_test_userrrerjryr)r job_queued)rrGrHrIrun_jobs  z7ExtractionAgent._queue_extraction_test..run_jobcsg|]}|qSrHrHr)rrHrIrszRunning extraction jobs Queued file extraction for file  under job_id r)rolistrr!r4r5rrr@zipr~rrre) rGrr single_file upload_tasksuploaded_files job_tasks extract_jobsrr file_reprrH)rrrGrI_queue_extraction_tests@  z&ExtractionAgent._queue_extraction_testc s t|ts |g}d}nd}fdd|D}tt|jdjdIdH}Wdn1s3wYfdd|D}tt|jd jdIdH}Wdn1s\wYjrt||D]\}}t|tt frxt|nd } t d | d |j qi|r|d S|S)z Queue multiple files for extraction. Args: files (Union[FileInput, List[FileInput]]): The files to extract Returns: Union[ExtractJob, List[ExtractJob]]: The queued extraction jobs TFcrrHrrrfrHrIrrz4ExtractionAgent.queue_extraction..rrNc s0g|]}jjjtj|jjjddqS)rrm)r<rurrrerjryrrfrHrIr$s zCreating extraction jobsrrrr) rorrr!r4r5r@rr~rrre) rGrrrrrrrrrrHrfrIqueue_extractionsD    z ExtractionAgent.queue_extractioncst|ts |g}d}nd}|IdH}fdd|D}tt|jdjdIdH}Wdn1s:wY|rE|dS|S) aAsynchronously extract data from one or more files using this agent. Args: files (Union[FileInput, List[FileInput]]): The files to extract Returns: Union[ExtractRun, List[ExtractRun]]: The extraction results TFNcsg|]}|jqSrH)rre)rrrfrHrIrVsz,ExtractionAgent.aextract..zExtracting filesrr)rorrrr!r4r5)rGrrjobs result_tasksresultsrHrfrIaextractBs  zExtractionAgent.aextractcCs|||S)aSynchronously extract data from one or more files using this agent. Args: files (Union[FileInput, List[FileInput]]): The files to extract Returns: Union[ExtractRun, List[ExtractRun]]: The extraction results )rbr)rGrrHrHrIextractas zExtractionAgent.extractcC||jjj|dS)z Get the extraction job for a given job_id. Args: job_id (str): The job_id to get the extraction job for Returns: ExtractJob: The extraction job r)rbr<rurrGrrHrHrIget_extraction_jobns z"ExtractionAgent.get_extraction_jobcCr)z Get the extraction run for a given job_id. Args: job_id (str): The job_id to get the extraction run for Returns: ExtractRun: The extraction run r)rbr<rurrrHrHrIget_extraction_run_for_jobzs  z*ExtractionAgent.get_extraction_run_for_jobrun_idcC||jjj|ddS)zxDelete an extraction run by ID. Args: run_id (str): The ID of the extraction run to delete )rN)rbr<rudelete_extraction_run)rGrrHrHrIrsz%ExtractionAgent.delete_extraction_runrdpagelimitcCs ||jjj|j|||dS)zList extraction runs for the extraction agent. Returns: PaginatedExtractRunsResponse: Paginated list of extraction runs )rskipr)rbr<rulist_extract_runsre)rGrrrHrHrIlist_extraction_runssz$ExtractionAgent.list_extraction_runscCsd|jd|jdS)NzExtractionAgent(id=z, name=))rerhrfrHrHrI__repr__szExtractionAgent.__repr__) NNr*r+r,TFTr-)rLN)rr)-__name__ __module__ __qualname____doc__rCloudExtractAgentrr~intboolfloatrJr r r&rbpropertyrerhrprjsetter SchemaInputrry FileInputrrrrrr rrrrrrrrrrrrrrHrHrHrIr).s      4 ;      r)cseZdZUdZeddZeed<eddZeed<eddd Z e ed <ed d d Z e ed <edddddZ e ed<eddd Z eed<eddd Zeed<eddd Zeeed<eddd Zeeed<eZeed<eZeed <eZeeed!<eZeeed"< # #    # #   d9deedeed e d e de ded$eed%eedeedeedeffd&d' Zd(eeeefd)efd*d+Z #d:d,ed-e d.ee!d)e"fd/d0Z# # #d;d,eed1eed)e"fd2d3Z$d)e%e"fd4d5Z&d6ed)d#fd7d8Z'Z(S)< LlamaExtractz:Factory class for creating and managing extraction agents.z%The API key for the LlamaExtract API.) descriptionapi_keyz%The base URL of the LlamaExtract API.base_urlr*z;The interval in seconds to check if the extraction is done.)defaultrr2r+zDThe maximum timeout in seconds to wait for the extraction to finish.r3r,rr9zAThe number of workers to use sending API requests for extraction.)rgtltrr4Tz-Show progress when extracting multiple files.r5Fz*Show verbose output when extracting files.r6zSimple SSL verification option.r7r-zTimeout for the httpx client.r8 _async_clientrFr>r?Nr0r1c  s|stdd}|durtd|stddpt}tj||||||| | | d tj| | d|_| |_ | |_ t |j |j |jd|_ttdtpJdd d |_|sztd d}|sztd ||jj} d d| D} | sutd| dj}||_||_dS)NLLAMA_CLOUD_API_KEYzThe API key is required.LLAMA_CLOUD_BASE_URL) rrr2r3r4r5r7r8r6rP)tokenrrSr9r*r,r:LLAMA_CLOUD_PROJECT_IDz1No project_id provided, fetching default project.cSsg|]}|jr|qSrH) is_default)rprHrHrIrrz)LlamaExtract.__init__..z6No default project found. Please provide a project_id.r)rDgetenvrtr$superrJrTrU _httpx_clientr7r8rrrrr%rCrErFrrbprojects list_projectsrer>r?)rGrrr2r3r4r5r0r1r7r8r6rdefault_project __class__rHrIrJsV      zLlamaExtract.__init__rKrLcrM)rNrLcrO)NrLc sjdus Jdtjjjd4IdH$}|jj_zIdHWjjj_WdIdHSjjj_w1IdHsAwYdS)Nz"httpx_client should be initializedrP)rrTrUr7r8rrRrS)r.rWrHrIrXs$    zCLlamaExtract._run_in_thread..run_coro..wrapped_cororYr\rWrHrIr]s z-LlamaExtract._run_in_thread..run_coror^rarHrWrIrb szLlamaExtract._run_in_threadrhrjryc Cst|tr|}nt|tr|}ntd||jjj |j |j t |||p&t dd}t|j||j |j |j|j|j|j|jd S)a]Create a new extraction agent. Args: name (str): The name of the extraction agent data_schema (SchemaInput): The data schema for the extraction agent config (Optional[ExtractConfig]): The extraction config for the agent Returns: ExtractionAgent: The created extraction agent rl)rhrjry)r0r1rn r.r/r0r1r2r3r4r5r6)rorprrr rsrtrbrrucreate_extraction_agentr>r?rDEFAULT_EXTRACT_CONFIGr)r2r3r4r5r6)rGrhrjryr/rHrHrI create_agent's:    zLlamaExtract.create_agentrec Cs|dur |dur td|r||jjj|d}n|r+||jjj||jd}ntdt |j||j|j |j |j |j |j|j|j|jd S)aGet extraction agents by name or extraction agent ID. Args: name (Optional[str]): Filter by name extraction_agent_id (Optional[str]): Filter by extraction agent ID Returns: ExtractionAgent: The extraction agent NzJBoth name and extraction_agent_id are provided. Using extraction_agent_id.r)rhr0z4Either name or extraction_agent_id must be provided.) r.r/r0r1r2r3r4r5r6r7r8)rrrbrruget_extraction_agentget_extraction_agent_by_namer>rtr)r?r2r3r4r5r6r7r8)rGrhrer/rHrHrI get_agentYs>zLlamaExtract.get_agentcs*jjjjd}fdd|DS)z%List all available extraction agents.)r0c s6g|]}tj|jjjjjjjd qS)r) r)rr>r?r2r3r4r5r6)rr/rfrHrIrs z,LlamaExtract.list_agents..)rbrrulist_extraction_agentsr>)rGagentsrHrfrI list_agentss  zLlamaExtract.list_agentsagent_idcCr)zzDelete an extraction agent by ID. Args: agent_id (str): ID of the extraction agent to delete rN)rbrrudelete_extraction_agent)rGrrHrHrI delete_agents zLlamaExtract.delete_agent) NNr*r+r,TNNTr-Frd)NN))rrrrr"rr~__annotations__rr2rr3r4r5rr6r7rr8rr#rrrFr%r>r?rJr r r&rbrrr)rrrrr __classcell__rHrHrrIrs     ?  4 2r__main__) load_dotenvtestsdataz test-agent)rhobjectrqstring)titlesummary)rq propertiesslidezconocophilips.pdf)PrZrDriorrrpathlibrtypingrrrr r r r rrTpydanticr llama_cloudrrrrrrrrrrrrrrrrllama_cloud.clientr"llama_cloud_services.extract.utilsrrllama_index.core.schemar llama_index.core.async_utilsr! llama_index.core.bridge.pydanticr"r#llama_index.core.constantsr$concurrent.futuresr%r&r~r}rrPER_DOCACCURATErr)rrdotenvr__file__parentdata_dir extractorrr/rrrrrrerrHrHrHrIsf $ D     |