Document Loaders#

All different types of document loaders.

class langchain.document_loaders.AZLyricsLoader(web_path: Union[str, List[str]], header_template: Optional[dict] = None)[source]#

Loader that loads AZLyrics webpages.

load() → List[langchain.schema.Document][source]#: Load webpage.

web_paths: List[str]#

class langchain.document_loaders.AirbyteJSONLoader(file_path: str)[source]#

Loader that loads local airbyte json files.

load() → List[langchain.schema.Document][source]#: Load file.

pydantic model langchain.document_loaders.ApifyDatasetLoader[source]#

Logic for loading documents from Apify datasets.

field apify_client: Any = None#

field dataset_id: str [Required]#: The ID of the dataset on the Apify platform.

field dataset_mapping_function: Callable[[Dict], langchain.schema.Document] [Required]#: A custom function that takes a single dictionary (an Apify dataset item) and converts it to an instance of the Document class.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.AzureBlobStorageContainerLoader(conn_str: str, container: str, prefix: str = '')[source]#

Loading logic for loading documents from Azure Blob Storage.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.AzureBlobStorageFileLoader(conn_str: str, container: str, blob_name: str)[source]#

Loading logic for loading documents from Azure Blob Storage.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.BSHTMLLoader(file_path: str, open_encoding: Optional[str] = None, bs_kwargs: Optional[dict] = None)[source]#

Loader that uses beautiful soup to parse HTML files.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.BigQueryLoader(query: str, project: Optional[str] = None, page_content_columns: Optional[List[str]] = None, metadata_columns: Optional[List[str]] = None)[source]#

Loads a query result from BigQuery into a list of documents.

Each document represents one row of the result. The page_content_columns are written into the page_content of the document. The metadata_columns are written into the metadata of the document. By default, all columns are written into the page_content and none into the metadata.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.BiliBiliLoader(video_urls: List[str])[source]#

Loader that loads bilibili transcripts.

load() → List[langchain.schema.Document][source]#: Load from bilibili url.

class langchain.document_loaders.BlackboardLoader(blackboard_course_url: str, bbrouter: str, load_all_recursively: bool = True, basic_auth: Optional[Tuple[str, str]] = None, cookies: Optional[dict] = None)[source]#

Loader that loads all documents from a Blackboard course.

This loader is not compatible with all Blackboard courses. It is only compatible with courses that use the new Blackboard interface. To use this loader, you must have the BbRouter cookie. You can get this cookie by logging into the course and then copying the value of the BbRouter cookie from the browser’s developer tools.

Example

from langchain.document_loaders import BlackboardLoader

loader = BlackboardLoader(
    blackboard_course_url="https://blackboard.example.com/webapps/blackboard/execute/announcement?method=search&context=course_entry&course_id=_123456_1",
    bbrouter="expires:12345...",
)
documents = loader.load()

base_url: str#

check_bs4() → None[source]#

Check if BeautifulSoup4 is installed.

Raises: ImportError – If BeautifulSoup4 is not installed.

download(path: str) → None[source]#

Download a file from a url.

Parameters: path – Path to the file.

folder_path: str#

load() → List[langchain.schema.Document][source]#

Load data into document objects.

Returns: List of documents.

load_all_recursively: bool#

parse_filename(url: str) → str[source]#

Parse the filename from a url.

Parameters: url – Url to parse the filename from.
Returns: The filename.

class langchain.document_loaders.BlockchainDocumentLoader(contract_address: str, blockchainType: langchain.document_loaders.blockchain.BlockchainType = BlockchainType.ETH_MAINNET, api_key: str = 'docs-demo', startToken: int = 0)[source]#

Loads elements from a blockchain smart contract into Langchain documents.

The supported blockchains are: Ethereum mainnet, Ethereum Goerli testnet, Polygon mainnet, and Polygon Mumbai testnet.

If no BlockchainType is specified, the default is Ethereum mainnet.

The Loader uses the Alchemy API to interact with the blockchain.

ALCHEMY_API_KEY environment variable must be set to use this loader.

Future versions of this loader can:

Support additional Alchemy APIs (e.g. getTransactions, etc.)

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.CSVLoader(file_path: str, source_column: Optional[str] = None, csv_args: Optional[Dict] = None, encoding: Optional[str] = None)[source]#

Loads a CSV file into a list of documents.

Each document represents one row of the CSV file. Every row is converted into a key/value pair and outputted to a new line in the document’s page_content.

The source for each document loaded from csv is set to the value of the file_path argument for all doucments by default. You can override this by setting the source_column argument to the name of a column in the CSV file. The source of each document will then be set to the value of the column with the name specified in source_column.

Output Example:

column1: value1
column2: value2
column3: value3

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.ChatGPTLoader(log_file: str, num_logs: int = - 1)[source]#

Loader that loads conversations from exported ChatGPT data.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.CoNLLULoader(file_path: str)[source]#

Load CoNLL-U files.

load() → List[langchain.schema.Document][source]#: Load from file path.

class langchain.document_loaders.CollegeConfidentialLoader(web_path: Union[str, List[str]], header_template: Optional[dict] = None)[source]#

Loader that loads College Confidential webpages.

load() → List[langchain.schema.Document][source]#: Load webpage.

web_paths: List[str]#

class langchain.document_loaders.ConfluenceLoader(url: str, api_key: Optional[str] = None, username: Optional[str] = None, oauth2: Optional[dict] = None, cloud: Optional[bool] = True, number_of_retries: Optional[int] = 3, min_retry_seconds: Optional[int] = 2, max_retry_seconds: Optional[int] = 10, confluence_kwargs: Optional[dict] = None)[source]#

Load Confluence pages. Port of https://llamahub.ai/l/confluence This currently supports both username/api_key and Oauth2 login.

Specify a list page_ids and/or space_key to load in the corresponding pages into Document objects, if both are specified the union of both sets will be returned.

You can also specify a boolean include_attachments to include attachments, this is set to False by default, if set to True all attachments will be downloaded and ConfluenceReader will extract the text from the attachments and add it to the Document object. Currently supported attachment types are: PDF, PNG, JPEG/JPG, SVG, Word and Excel.

Hint: space_key and page_id can both be found in the URL of a page in Confluence - https://yoursite.atlassian.com/wiki/spaces/<space_key>/pages/<page_id>

Example

from langchain.document_loaders import ConfluenceLoader

loader = ConfluenceLoader(
    url="https://yoursite.atlassian.com/wiki",
    username="me",
    api_key="12345"
)
documents = loader.load(space_key="SPACE",limit=50)

Parameters

url (str) – _description_
api_key (str, optional) – _description_, defaults to None
username (str, optional) – _description_, defaults to None
oauth2 (dict, optional) – _description_, defaults to {}
cloud (bool, optional) – _description_, defaults to True
number_of_retries (Optional[int], optional) – How many times to retry, defaults to 3
min_retry_seconds (Optional[int], optional) – defaults to 2
max_retry_seconds (Optional[int], optional) – defaults to 10
confluence_kwargs (dict, optional) – additional kwargs to initialize confluence with

Raises

ValueError – Errors while validating input
ImportError – Required dependencies not installed.

load(space_key: Optional[str] = None, page_ids: Optional[List[str]] = None, label: Optional[str] = None, cql: Optional[str] = None, include_attachments: bool = False, include_comments: bool = False, limit: Optional[int] = 50, max_pages: Optional[int] = 1000) → List[langchain.schema.Document][source]#

Parameters

space_key (Optional[str], optional) – Space key retrieved from a confluence URL, defaults to None
page_ids (Optional[List[str]], optional) – List of specific page IDs to load, defaults to None
label (Optional[str], optional) – Get all pages with this label, defaults to None
cql (Optional[str], optional) – CQL Expression, defaults to None
include_attachments (bool, optional) – defaults to False
include_comments (bool, optional) – defaults to False
limit (int, optional) – Maximum number of pages to retrieve per request, defaults to 50
max_pages (int, optional) – Maximum number of pages to retrieve in total, defaults 1000

Raises

ValueError – _description_
ImportError – _description_

Returns

_description_

Return type

List[Document]

paginate_request(retrieval_method: Callable, **kwargs: Any) → List[source]#

Paginate the various methods to retrieve groups of pages.

Unfortunately, due to page size, sometimes the Confluence API doesn’t match the limit value. If limit is >100 confluence seems to cap the response to 100. Also, due to the Atlassian Python package, we don’t get the “next” values from the “_links” key because they only return the value from the results key. So here, the pagination starts from 0 and goes until the max_pages, getting the limit number of pages with each request. We have to manually check if there are more docs based on the length of the returned list of pages, rather than just checking for the presence of a next key in the response like this page would have you do: https://developer.atlassian.com/server/confluence/pagination-in-the-rest-api/

Parameters: retrieval_method (callable) – Function used to retrieve docs
Returns: List of documents
Return type: List

process_attachment(page_id: str) → List[str][source]#

process_doc(link: str) → str[source]#

process_image(link: str) → str[source]#

process_page(page: dict, include_attachments: bool, include_comments: bool, text_maker: Any) → langchain.schema.Document[source]#

process_pdf(link: str) → str[source]#

process_svg(link: str) → str[source]#

process_xls(link: str) → str[source]#

static validate_init_args(url: Optional[str] = None, api_key: Optional[str] = None, username: Optional[str] = None, oauth2: Optional[dict] = None) → Optional[List][source]#: Validates proper combinations of init arguments

class langchain.document_loaders.DataFrameLoader(data_frame: Any, page_content_column: str = 'text')[source]#

Load Pandas DataFrames.

load() → List[langchain.schema.Document][source]#: Load from the dataframe.

class langchain.document_loaders.DiffbotLoader(api_token: str, urls: List[str], continue_on_failure: bool = True)[source]#

Loader that loads Diffbot file json.

load() → List[langchain.schema.Document][source]#: Extract text from Diffbot on all the URLs and return Document instances

class langchain.document_loaders.DirectoryLoader(path: str, glob: str = '**/[!.]*', silent_errors: bool = False, load_hidden: bool = False, loader_cls: typing.Union[typing.Type[langchain.document_loaders.unstructured.UnstructuredFileLoader], typing.Type[langchain.document_loaders.text.TextLoader], typing.Type[langchain.document_loaders.html_bs.BSHTMLLoader]] = <class 'langchain.document_loaders.unstructured.UnstructuredFileLoader'>, loader_kwargs: typing.Optional[dict] = None, recursive: bool = False, show_progress: bool = False)[source]#

Loading logic for loading documents from a directory.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.DiscordChatLoader(chat_log: pd.DataFrame, user_id_col: str = 'ID')[source]#

Load Discord chat logs.

load() → List[langchain.schema.Document][source]#: Load all chat messages.

class langchain.document_loaders.DuckDBLoader(query: str, database: str = ':memory:', read_only: bool = False, config: Optional[Dict[str, str]] = None, page_content_columns: Optional[List[str]] = None, metadata_columns: Optional[List[str]] = None)[source]#

Loads a query result from DuckDB into a list of documents.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.EverNoteLoader(file_path: str)[source]#

Loader to load in EverNote files..

load() → List[langchain.schema.Document][source]#: Load document from EverNote file.

class langchain.document_loaders.FacebookChatLoader(path: str)[source]#

Loader that loads Facebook messages json directory dump.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.GCSDirectoryLoader(project_name: str, bucket: str, prefix: str = '')[source]#

Loading logic for loading documents from GCS.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.GCSFileLoader(project_name: str, bucket: str, blob: str)[source]#

Loading logic for loading documents from GCS.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.GitLoader(repo_path: str, clone_url: Optional[str] = None, branch: Optional[str] = 'main', file_filter: Optional[Callable[[str], bool]] = None)[source]#

Loads files from a Git repository into a list of documents. Repository can be local on disk available at repo_path, or remote at clone_url that will be cloned to repo_path. Currently supports only text files.

Each document represents one file in the repository. The path points to the local Git repository, and the branch specifies the branch to load files from. By default, it loads from the main branch.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.GitbookLoader(web_page: str, load_all_paths: bool = False, base_url: Optional[str] = None, content_selector: str = 'main')[source]#

Load GitBook data.

load from either a single page, or
load all (relative) paths in the navbar.

load() → List[langchain.schema.Document][source]#: Fetch text from one single GitBook page.

web_paths: List[str]#

class langchain.document_loaders.GoogleApiClient(credentials_path: pathlib.Path = PosixPath('/home/docs/.credentials/credentials.json'), service_account_path: pathlib.Path = PosixPath('/home/docs/.credentials/credentials.json'), token_path: pathlib.Path = PosixPath('/home/docs/.credentials/token.json'))[source]#

A Generic Google Api Client.

To use, you should have the google_auth_oauthlib,youtube_transcript_api,google python package installed. As the google api expects credentials you need to set up a google account and register your Service. “https://developers.google.com/docs/api/quickstart/python”

Example

from langchain.document_loaders import GoogleApiClient
google_api_client = GoogleApiClient(
    service_account_path=Path("path_to_your_sec_file.json")
)

credentials_path: pathlib.Path = PosixPath('/home/docs/.credentials/credentials.json')#

service_account_path: pathlib.Path = PosixPath('/home/docs/.credentials/credentials.json')#

token_path: pathlib.Path = PosixPath('/home/docs/.credentials/token.json')#

classmethod validate_channel_or_videoIds_is_set(values: Dict[str, Any]) → Dict[str, Any][source]#: Validate that either folder_id or document_ids is set, but not both.

class langchain.document_loaders.GoogleApiYoutubeLoader(google_api_client: langchain.document_loaders.youtube.GoogleApiClient, channel_name: Optional[str] = None, video_ids: Optional[List[str]] = None, add_video_info: bool = True, captions_language: str = 'en', continue_on_failure: bool = False)[source]#

Loader that loads all Videos from a Channel

To use, you should have the googleapiclient,youtube_transcript_api python package installed. As the service needs a google_api_client, you first have to initialize the GoogleApiClient.

Additionally you have to either provide a channel name or a list of videoids “https://developers.google.com/docs/api/quickstart/python”

Example

from langchain.document_loaders import GoogleApiClient
from langchain.document_loaders import GoogleApiYoutubeLoader
google_api_client = GoogleApiClient(
    service_account_path=Path("path_to_your_sec_file.json")
)
loader = GoogleApiYoutubeLoader(
    google_api_client=google_api_client,
    channel_name = "CodeAesthetic"
)
load.load()

add_video_info: bool = True#

captions_language: str = 'en'#

channel_name: Optional[str] = None#

continue_on_failure: bool = False#

google_api_client: langchain.document_loaders.youtube.GoogleApiClient#

load() → List[langchain.schema.Document][source]#: Load documents.

classmethod validate_channel_or_videoIds_is_set(values: Dict[str, Any]) → Dict[str, Any][source]#: Validate that either folder_id or document_ids is set, but not both.

video_ids: Optional[List[str]] = None#

pydantic model langchain.document_loaders.GoogleDriveLoader[source]#

Loader that loads Google Docs from Google Drive.

Validators

validate_credentials_path » credentials_path
validate_folder_id_or_document_ids » all fields

field credentials_path: pathlib.Path = PosixPath('/home/docs/.credentials/credentials.json')#

field document_ids: Optional[List[str]] = None#

field file_ids: Optional[List[str]] = None#

field folder_id: Optional[str] = None#

field recursive: bool = False#

field service_account_key: pathlib.Path = PosixPath('/home/docs/.credentials/keys.json')#

field token_path: pathlib.Path = PosixPath('/home/docs/.credentials/token.json')#

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.GutenbergLoader(file_path: str)[source]#

Loader that uses urllib to load .txt web files.

load() → List[langchain.schema.Document][source]#: Load file.

class langchain.document_loaders.HNLoader(web_path: Union[str, List[str]], header_template: Optional[dict] = None)[source]#

Load Hacker News data from either main page results or the comments page.

load() → List[langchain.schema.Document][source]#

Get important HN webpage information.

Components are:

title
content
source url,
time of post
author of the post
number of comments
rank of the post

load_comments(soup_info: Any) → List[langchain.schema.Document][source]#: Load comments from a HN post.

load_results(soup: Any) → List[langchain.schema.Document][source]#: Load items from an HN page.

web_paths: List[str]#

class langchain.document_loaders.HuggingFaceDatasetLoader(path: str, page_content_column: str = 'text', name: Optional[str] = None, data_dir: Optional[str] = None, data_files: Optional[Union[str, Sequence[str], Mapping[str, Union[str, Sequence[str]]]]] = None, cache_dir: Optional[str] = None, keep_in_memory: Optional[bool] = None, save_infos: bool = False, use_auth_token: Optional[Union[bool, str]] = None, num_proc: Optional[int] = None)[source]#

Loading logic for loading documents from the Hugging Face Hub.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.IFixitLoader(web_path: str)[source]#

Load iFixit repair guides, device wikis and answers.

iFixit is the largest, open repair community on the web. The site contains nearly 100k repair manuals, 200k Questions & Answers on 42k devices, and all the data is licensed under CC-BY.

This loader will allow you to download the text of a repair guide, text of Q&A’s and wikis from devices on iFixit using their open APIs and web scraping.

load() → List[langchain.schema.Document][source]#: Load data into document objects.

load_device(url_override: Optional[str] = None, include_guides: bool = True) → List[langchain.schema.Document][source]#

load_guide(url_override: Optional[str] = None) → List[langchain.schema.Document][source]#

load_questions_and_answers(url_override: Optional[str] = None) → List[langchain.schema.Document][source]#

static load_suggestions(query: str = '', doc_type: str = 'all') → List[langchain.schema.Document][source]#

class langchain.document_loaders.IMSDbLoader(web_path: Union[str, List[str]], header_template: Optional[dict] = None)[source]#

Loader that loads IMSDb webpages.

load() → List[langchain.schema.Document][source]#: Load webpage.

web_paths: List[str]#

class langchain.document_loaders.ImageCaptionLoader(path_images: Union[str, List[str]], blip_processor: str = 'Salesforce/blip-image-captioning-base', blip_model: str = 'Salesforce/blip-image-captioning-base')[source]#

Loader that loads the captions of an image

load() → List[langchain.schema.Document][source]#: Load from a list of image files

class langchain.document_loaders.NotebookLoader(path: str, include_outputs: bool = False, max_output_length: int = 10, remove_newline: bool = False, traceback: bool = False)[source]#

Loader that loads .ipynb notebook files.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.NotionDBLoader(integration_token: str, database_id: str)[source]#

Notion DB Loader. Reads content from pages within a Noton Database. :param integration_token: Notion integration token. :type integration_token: str :param database_id: Notion database id. :type database_id: str

load() → List[langchain.schema.Document][source]#: Load documents from the Notion database. :returns: List of documents. :rtype: List[Document]

load_page(page_id: str) → langchain.schema.Document[source]#: Read a page.

class langchain.document_loaders.NotionDirectoryLoader(path: str)[source]#

Loader that loads Notion directory dump.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.ObsidianLoader(path: str, encoding: str = 'UTF-8', collect_metadata: bool = True)[source]#

Loader that loads Obsidian files from disk.

FRONT_MATTER_REGEX = re.compile('^---\\n(.*?)\\n---\\n', re.MULTILINE|re.DOTALL)#

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.OnlinePDFLoader(file_path: str)[source]#

Loader that loads online PDFs.

file_path: str#

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.OutlookMessageLoader(file_path: str)[source]#

Loader that loads Outlook Message files using extract_msg. TeamMsgExtractor/msg-extractor

load() → List[langchain.schema.Document][source]#: Load data into document objects.

class langchain.document_loaders.PDFMinerLoader(file_path: str)[source]#

Loader that uses PDFMiner to load PDF files.

file_path: str#

load() → List[langchain.schema.Document][source]#: Load file.

class langchain.document_loaders.PDFMinerPDFasHTMLLoader(file_path: str)[source]#

Loader that uses PDFMiner to load PDF files as HTML content.

file_path: str#

load() → List[langchain.schema.Document][source]#: Load file.

langchain.document_loaders.PagedPDFSplitter#: alias of langchain.document_loaders.pdf.PyPDFLoader

class langchain.document_loaders.PlaywrightURLLoader(urls: List[str], continue_on_failure: bool = True, headless: bool = True, remove_selectors: Optional[List[str]] = None)[source]#

Loader that uses Playwright and to load a page and unstructured to load the html. This is useful for loading pages that require javascript to render.

urls#

List of URLs to load.

Type: List[str]

continue_on_failure#

If True, continue loading other URLs on failure.

Type: bool

headless#

If True, the browser will run in headless mode.

Type: bool

load() → List[langchain.schema.Document][source]#

Load the specified URLs using Playwright and create Document instances.

Returns: A list of Document instances with loaded content.
Return type: List[Document]

class langchain.document_loaders.PyMuPDFLoader(file_path: str)[source]#

Loader that uses PyMuPDF to load PDF files.

file_path: str#

load(**kwargs: Optional[Any]) → List[langchain.schema.Document][source]#: Load file.

class langchain.document_loaders.PyPDFLoader(file_path: str)[source]#

Loads a PDF with pypdf and chunks at character level.

Loader also stores page numbers in metadatas.

file_path: str#

load() → List[langchain.schema.Document][source]#: Load given path as pages.

class langchain.document_loaders.PythonLoader(file_path: str)[source]#: Load Python files, respecting any non-default encoding if specified.

class langchain.document_loaders.ReadTheDocsLoader(path: str, encoding: Optional[str] = None, errors: Optional[str] = None, **kwargs: Optional[Any])[source]#

Loader that loads ReadTheDocs documentation directory dump.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.RoamLoader(path: str)[source]#

Loader that loads Roam files from disk.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.S3DirectoryLoader(bucket: str, prefix: str = '')[source]#

Loading logic for loading documents from s3.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.S3FileLoader(bucket: str, key: str)[source]#

Loading logic for loading documents from s3.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.SRTLoader(file_path: str)[source]#

Loader for .srt (subtitle) files.

load() → List[langchain.schema.Document][source]#: Load using pysrt file.

class langchain.document_loaders.SeleniumURLLoader(urls: List[str], continue_on_failure: bool = True, browser: Literal['chrome', 'firefox'] = 'chrome', executable_path: Optional[str] = None, headless: bool = True)[source]#

Loader that uses Selenium and to load a page and unstructured to load the html. This is useful for loading pages that require javascript to render.

urls#

List of URLs to load.

Type: List[str]

continue_on_failure#

If True, continue loading other URLs on failure.

Type: bool

browser#

The browser to use, either ‘chrome’ or ‘firefox’.

Type: str

executable_path#

The path to the browser executable.

Type: Optional[str]

headless#

If True, the browser will run in headless mode.

Type: bool

load() → List[langchain.schema.Document][source]#

Load the specified URLs using Selenium and create Document instances.

Returns: A list of Document instances with loaded content.
Return type: List[Document]

class langchain.document_loaders.SitemapLoader(web_path: str, filter_urls: Optional[List[str]] = None, parsing_function: Optional[Callable] = None)[source]#

Loader that fetches a sitemap and loads those URLs.

load() → List[langchain.schema.Document][source]#: Load sitemap.

parse_sitemap(soup: Any) → List[dict][source]#: Parse sitemap xml and load into a list of dicts.

web_paths: List[str]#

class langchain.document_loaders.SlackDirectoryLoader(zip_path: str, workspace_url: Optional[str] = None)[source]#

Loader for loading documents from a Slack directory dump.

load() → List[langchain.schema.Document][source]#: Load and return documents from the Slack directory dump.

class langchain.document_loaders.TelegramChatLoader(path: str)[source]#

Loader that loads Telegram chat json directory dump.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.TextLoader(file_path: str, encoding: Optional[str] = None)[source]#

Load text files.

load() → List[langchain.schema.Document][source]#: Load from file path.

class langchain.document_loaders.TwitterTweetLoader(auth_handler: Union[OAuthHandler, OAuth2BearerHandler], twitter_users: Sequence[str], number_tweets: Optional[int] = 100)[source]#

Twitter tweets loader. Read tweets of user twitter handle.

First you need to go to https://developer.twitter.com/en/docs/twitter-api /getting-started/getting-access-to-the-twitter-api to get your token. And create a v2 version of the app.

classmethod from_bearer_token(oauth2_bearer_token: str, twitter_users: Sequence[str], number_tweets: Optional[int] = 100) → langchain.document_loaders.twitter.TwitterTweetLoader[source]#: Create a TwitterTweetLoader from OAuth2 bearer token.

classmethod from_secrets(access_token: str, access_token_secret: str, consumer_key: str, consumer_secret: str, twitter_users: Sequence[str], number_tweets: Optional[int] = 100) → langchain.document_loaders.twitter.TwitterTweetLoader[source]#: Create a TwitterTweetLoader from access tokens and secrets.

load() → List[langchain.schema.Document][source]#: Load tweets.

class langchain.document_loaders.UnstructuredEPubLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load epub files.

class langchain.document_loaders.UnstructuredEmailLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load email files.

class langchain.document_loaders.UnstructuredFileIOLoader(file: IO, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load file IO objects.

class langchain.document_loaders.UnstructuredFileLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load files.

class langchain.document_loaders.UnstructuredHTMLLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load HTML files.

class langchain.document_loaders.UnstructuredImageLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load image files, such as PNGs and JPGs.

class langchain.document_loaders.UnstructuredMarkdownLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load markdown files.

class langchain.document_loaders.UnstructuredPDFLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load PDF files.

class langchain.document_loaders.UnstructuredPowerPointLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load powerpoint files.

class langchain.document_loaders.UnstructuredRTFLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load rtf files.

class langchain.document_loaders.UnstructuredURLLoader(urls: List[str], continue_on_failure: bool = True, **unstructured_kwargs: Any)[source]#

Loader that uses unstructured to load HTML files.

load() → List[langchain.schema.Document][source]#: Load file.

class langchain.document_loaders.UnstructuredWordDocumentLoader(file_path: str, mode: str = 'single', **unstructured_kwargs: Any)[source]#: Loader that uses unstructured to load word documents.

class langchain.document_loaders.WebBaseLoader(web_path: Union[str, List[str]], header_template: Optional[dict] = None)[source]#

Loader that uses urllib and beautiful soup to load webpages.

aload() → List[langchain.schema.Document][source]#: Load text from the urls in web_path async into Documents.

default_parser: str = 'html.parser'#: Default parser to use for BeautifulSoup.

async fetch_all(urls: List[str]) → Any[source]#: Fetch all urls concurrently with rate limiting.

load() → List[langchain.schema.Document][source]#: Load text from the url(s) in web_path.

requests_per_second: int = 2#: Max number of concurrent requests to make.

scrape(parser: Optional[str] = None) → Any[source]#: Scrape data from webpage and return it in BeautifulSoup format.

scrape_all(urls: List[str], parser: Optional[str] = None) → List[Any][source]#: Fetch all urls, then return soups for all results.

property web_path: str#

web_paths: List[str]#

class langchain.document_loaders.WhatsAppChatLoader(path: str)[source]#

Loader that loads WhatsApp messages text file.

load() → List[langchain.schema.Document][source]#: Load documents.

class langchain.document_loaders.YoutubeLoader(video_id: str, add_video_info: bool = False, language: str = 'en', continue_on_failure: bool = False)[source]#

Loader that loads Youtube transcripts.

classmethod from_youtube_url(youtube_url: str, **kwargs: Any) → langchain.document_loaders.youtube.YoutubeLoader[source]#: Given youtube URL, load video.

load() → List[langchain.schema.Document][source]#: Load documents.