heritrix3.api

exception heritrix3.api.HeritrixAPIError(message: str, *args, **kwargs)

Error as response from Heritrix3 REST API.

Parameters

• message (str) – Error description / message.

• response (Optional[requests.Response]) – Optional api response object.

class heritrix3.api.HeritrixAPI(host: str = 'https://localhost:8443/engine', user: str = 'admin', passwd: str = 'admin', verbose: bool = False, insecure: bool = True, headers: Optional[Dict[str, str]] = None, timeout: Optional[Union[int, float]] = None)

send_file(job_name: str, filepath: os.PathLike, name: Optional[str] = None) → bool

send_content(job_name: str, filecontent: Union[bytes, BinaryIO], name: str) → bool

retrieve_file(job_name: str, local_filepath: os.PathLike, job_filepath: Union[str, os.PathLike], overwrite: bool = False) → bool

info(job_name: Optional[str] = None, raw: bool = False) → Union[str, requests.models.Response]

list_jobs(status: Optional[str] = None, unbuilt: bool = False) → List[str]

get_job_state(job_name: str) → Optional[str]

get_crawl_exit_state(job_name: str) → Optional[str]

get_job_actions(job_name: str) → List[str]

wait_for_action(job_name: str, action: str, timeout: Union[int, float] = 20, poll_delay: Union[int, float] = 1) → bool

wait_for_jobstate(job_name: str, state: str, timeout: Union[int, float] = 20, poll_delay: Union[int, float] = 1) → bool

create(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

add(job_dir: str, raw: bool = False) → Union[str, requests.models.Response]

rescan(raw: bool = False) → Union[str, requests.models.Response]

copy(job_name: str, new_job_name: str, as_profile: bool = False, raw: bool = False) → Union[str, requests.models.Response]

build(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

launch(job_name: str, checkpoint: Optional[str] = None, raw: bool = False) → Union[str, requests.models.Response]

pause(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

unpause(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

terminate(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

teardown(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

checkpoint(job_name: str, raw: bool = False) → Union[str, requests.models.Response]

execute_script(job_name: str, script: str, engine: str = 'beanshell', raw: bool = False) → Union[str, requests.models.Response]

get_config(job_name: str, raw: bool = True) → str

send_config(job_name: str, cxml_filepath: os.PathLike) → bool

get_config_url(job_name: str) → str

get_launchid(job_name: str) → Optional[str]

crawl_report(job_name: str, launch_id: Optional[str] = None) → Optional[str]

seeds_report(job_name: str, launch_id: Optional[str] = None) → Optional[str]

hosts_report(job_name: str, launch_id: Optional[str] = None) → Optional[str]

mimetypes_report(job_name: str, launch_id: Optional[str] = None) → Optional[str]

responsecodes_report(job_name: str, launch_id: Optional[str] = None) → Optional[str]

job_log(job_name: str) → Optional[str]

crawl_log(job_name: str, launch_id: Optional[str] = None) → Optional[str]

list_files(job_name: str, gather_files: bool = True, gather_folders: bool = True) → List[str]

list_warcs(job_name: str, launchid: Optional[str] = None) → Optional[List[str]]

retrieve_warcs(job_name: str, local_folderpath: os.PathLike, launchid: Optional[str] = None, warcs_job_filepaths: Optional[List[Union[str, os.PathLike]]] = None, overwrite: bool = False) → Optional[int]

delete_job_dir(job_name: str) → None

heritrix3.api.disable_ssl_warnings()

Quieten SSL insecure warnings.

See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings