File manager - Edit - /home/newsbmcs.com/public_html/static/img/logo/builder.tar
Back
__pycache__/_lxml.cpython-310.pyc 0000644 00000023712 15030265652 0012601 0 ustar 00 o �hd: � @ s� d Z ddgZzddlmZ W n ey& Z z ddlmZ W Y dZ[ndZ[ww ddlmZ ddlm Z ddl m Z dd l m Z mZmZmZmZ dd lmZmZmZmZmZmZmZmZ dd lm Z d Z d d� Z G dd� de�Z G dd� dee �Z dS )�MIT�LXMLTreeBuilderForXML�LXMLTreeBuilder� )�CallableN)�BytesIO)�StringIO)�etree)�Comment�Doctype�NamespacedAttribute�ProcessingInstruction�XMLProcessingInstruction)�DetectsXMLParsedAsHTML�FAST�HTML�HTMLTreeBuilder� PERMISSIVE�ParserRejectedMarkup� TreeBuilder�XML)�EncodingDetector�lxmlc C s t dd� t| �� �D ��S )zInvert a dictionary.c s s � | ] \}}||fV qd S �N� )�.0�k�vr r �G/usr/local/CyberPanel/lib/python3.10/site-packages/bs4/builder/_lxml.py� <genexpr>' s � z_invert.<locals>.<genexpr>)�dict�list�items)�dr r r �_invert% s r# c s� e Zd ZejZdZeZdZ dgZ e e e e egZdZedd�Zee�Z� fdd�Zd d � Zd d � Zd d� Zd*� fdd� Zdd� Z d+dd�Zdd� Z dd� Z i fdd�Z d d � Z d d � Z d d!� Z!d"d#� Z"d$d%� Z#d&d'� Z$d(d)� Z%� Z&S ),r Tzlxml-xml�xmli z$http://www.w3.org/XML/1998/namespace)r$ c s t t| ��|� | �| j� dS )z�Let the BeautifulSoup object know about the standard namespace mapping. :param soup: A `BeautifulSoup`. N)�superr �initialize_soup�_register_namespaces�DEFAULT_NSMAPS)�self�soup�� __class__r r r& D s z%LXMLTreeBuilderForXML.initialize_soupc C s6 t |�� �D ]\}}|r|| jjvr|| jj|< qdS )a� Let the BeautifulSoup object know about namespaces encountered while parsing the document. This might be useful later on when creating CSS selectors. This will track (almost) all namespaces, even ones that were only in scope for part of the document. If two namespaces have the same prefix, only the first one encountered will be tracked. Un-prefixed namespaces are not tracked. :param mapping: A dictionary mapping namespace prefixes to URIs. N)r r! r* � _namespaces)r) �mapping�key�valuer r r r' M s ��z*LXMLTreeBuilderForXML._register_namespacesc C s" | j dur| j S tj| dd|d�S )z�Find the default parser for the given encoding. :param encoding: A string. :return: Either a parser object or a class, which will be instantiated with default arguments. NFT��target� strip_cdata�recover�encoding)�_default_parserr � XMLParser�r) r5 r r r �default_parsere s �z$LXMLTreeBuilderForXML.default_parserc C s( | � |�}t|t�r|| dd|d�}|S )z�Instantiate an appropriate parser for the given encoding. :param encoding: A string. :return: A parser object such as an `etree.XMLParser`. FTr1 )r9 � isinstancer )r) r5 �parserr r r � parser_forq s �z LXMLTreeBuilderForXML.parser_forNc sP || _ |d ur t|�| _d | _| jg| _t| j�g| _t t | �j di |�� d S )Nr ) r6 �set�empty_element_tagsr* �DEFAULT_NSMAPS_INVERTED�nsmapsr r( �active_namespace_prefixesr% r �__init__)r) r; r> �kwargsr+ r r rB � s z LXMLTreeBuilderForXML.__init__c C s, |d dkrt |dd � �dd��S d |fS )Nr �{� �})�tuple�split)r) �tagr r r � _getNsTag� s z LXMLTreeBuilderForXML._getNsTagc c s� � | j }|rt| _tj|dd� nt| _t|t�r3t|�dkr,|d dkr,|dd� }|d|dfV t|t�rB|� d�d|dfV |g}|g}t |||||d �}|j D ] } |j | |dfV qTdS ) aA Run any preliminary steps necessary to make incoming markup acceptable to the parser. lxml really wants to get a bytestring and convert it to Unicode itself. So instead of using UnicodeDammit to convert the bytestring to Unicode using different encodings, this implementation uses EncodingDetector to iterate over the encodings, and tell lxml to try to parse the document as each one in turn. :param markup: Some markup -- hopefully a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. � )� stacklevelr u rE NF�utf8)�known_definite_encodings�user_encodings�is_html�exclude_encodings) �is_xmlr � processing_instruction_classr � warn_if_markup_looks_like_xmlr r: �str�len�encoder � encodings�markup) r) rY �user_specified_encodingrQ �document_declared_encodingrP rN rO �detectorr5 r r r �prepare_markup� s4 � � �� �z$LXMLTreeBuilderForXML.prepare_markupc C s� t |t�r t|�}n t |t�rt|�}|�| j�}z4| �| jj �| _ | j � |� t |�dkrF|�| j�}t |�dkr@| j � |� t |�dks.| j � � W d S tttjfya } zt|��d }~ww )Nr )r: �bytesr rU r �read� CHUNK_SIZEr< r* �original_encodingr; �feedrV �close�UnicodeDecodeError� LookupErrorr � ParserErrorr )r) rY �data�er r r rb � s$ ���zLXMLTreeBuilderForXML.feedc C s | j g| _d S r )r? r@ )r) r r r rc � s zLXMLTreeBuilderForXML.closec C sR t |�}d }t|�dkrt| j�dkr| j�d � nEt|�dkr_| �|� | j�t|�� t | jd �}|�|� d|v r@|d= | j�|� |�� }t |� � �D ]\}}t d|d�}|||< qPi } t |� � �D ]#\} } | � | �\}} |d u r{| | | < qg| � |�}t || |�} | | | < qg| }| � |�\}}| � |�}| jj||||| jd d� d S )Nr rE ���� �xmlnsz http://www.w3.org/2000/xmlns/)� namespaces)r rV r@ �appendr' r# rA �update�copyr r! r rJ �_prefix_for_namespacer* �handle_starttag) r) �name�attrs�nsmap�nsprefix�current_mapping�prefix� namespace� attribute� new_attrs�attrr0 r r r �start� sB � �zLXMLTreeBuilderForXML.startc C s< |du rdS t | j�D ]}|dur||v r|| S q dS )z9Find the currently active prefix for the given namespace.N)�reversedr@ )r) rx �inverted_nsmapr r r rp 4 s �z+LXMLTreeBuilderForXML._prefix_for_namespacec C s� | j �� | j jd }| �|�\}}d }|d ur.t| j�D ]}|d ur-||v r-|| } nq | j �||� t| j�dkrL| j�� }|d urN| j �� d S d S d S )Nri rE ) r* �endData�tagStackrJ r} r@ � handle_endtagrV �poprA )r) rr � completed_tagrx ru r~ �out_of_scope_nsmapr r r �end= s" � �zLXMLTreeBuilderForXML.endc C s4 | j �� |d | }| j �|� | j �| j� d S )N� )r* r � handle_datarS )r) r2 rg r r r �piS s zLXMLTreeBuilderForXML.pic C s | j �|� d S r )r* r� �r) �contentr r r rg Y s zLXMLTreeBuilderForXML.datac C s( | j �� t�|||�}| j �|� d S r )r* r r �for_name_and_ids�object_was_parsed)r) rr �pubid�system�doctyper r r r� \ s z LXMLTreeBuilderForXML.doctypec C s& | j �� | j �|� | j �t� dS )z#Handle comments as Comment objects.N)r* r r� r r� r r r �commenta s z LXMLTreeBuilderForXML.commentc C � d| S )�See `TreeBuilder`.z)<?xml version="1.0" encoding="utf-8"?> %sr �r) �fragmentr r r �test_fragment_to_documentg � z/LXMLTreeBuilderForXML.test_fragment_to_document)NN)NNN)'�__name__� __module__� __qualname__r r7 �DEFAULT_PARSER_CLASSrR r rS �NAME�ALTERNATE_NAMES�LXMLr r r �featuresr` r r( r# r? r& r' r9 r< rB rJ r] rb rc r| rp r� r� rg r� r� r� � __classcell__r r r+ r r ) s< �H ? c @ sF e Zd ZeZdgZeeeeeg Z dZ e Z dd� Z dd� Zdd� Zd S ) r z lxml-htmlFc C s t jS r )r � HTMLParserr8 r r r r9 u s z LXMLTreeBuilder.default_parserc C sZ | j j}z| �|�| _| j�|� | j�� W d S tttj fy, } zt |��d }~ww r ) r* ra r< r; rb rc rd re r rf r )r) rY r5 rh r r r rb x s ��zLXMLTreeBuilder.feedc C r� )r� z <html><body>%s</body></html>r r� r r r r� � r� z)LXMLTreeBuilder.test_fragment_to_documentN)r� r� r� r� r� r� r r r r� rR r rS r9 rb r� r r r r r l s )!� __license__�__all__�collections.abcr � ImportErrorrh � collections�ior r r r � bs4.elementr r r r r � bs4.builderr r r r r r r r � bs4.dammitr r� r# r r r r r r �<module> s, ��� ( E __pycache__/_htmlparser.cpython-310.pyc 0000644 00000023137 15030265652 0014007 0 ustar 00 o �hK: � @ s� d Z dZdgZddlmZ ddlZddlZddlmZm Z m Z m Z m Z ddl mZmZ ddlmZmZmZmZmZ d ZG d d � d ee�ZG d d� de�ZdS ) zCUse the HTMLParser library to parse HTML files that aren't too bad.�MIT�HTMLParserTreeBuilder� )� HTMLParserN)�CData�Comment� Declaration�Doctype�ProcessingInstruction)�EntitySubstitution� UnicodeDammit)�DetectsXMLParsedAsHTML�ParserRejectedMarkup�HTML�HTMLTreeBuilder�STRICTz html.parserc @ s| e Zd ZdZdZdZdd� Zdd� Zdd � Zd d d �Z d d d�Z dd� Z dd� Z dd� Z dd� Zdd� Zdd� Zdd � Zd S ) �BeautifulSoupHTMLParserz�A subclass of the Python standard library's HTMLParser class, which listens for HTMLParser events and translates them into calls to Beautiful Soup's tree construction API. �ignore�replacec O s: |� d| j�| _tj| g|�R i |�� g | _| �� dS )a Constructor. :param on_duplicate_attribute: A strategy for what to do if a tag includes the same attribute more than once. Accepted values are: REPLACE (replace earlier values with later ones, the default), IGNORE (keep the earliest value encountered), or a callable. A callable must take three arguments: the dictionary of attributes already processed, the name of the duplicate attribute, and the most recent value encountered. �on_duplicate_attributeN)�pop�REPLACEr r �__init__� already_closed_empty_element�_initialize_xml_detector)�self�args�kwargs� r �M/usr/local/CyberPanel/lib/python3.10/site-packages/bs4/builder/_htmlparser.pyr . s � z BeautifulSoupHTMLParser.__init__c C s t |��)N)r )r �messager r r �errorJ s z BeautifulSoupHTMLParser.errorc C s | j ||dd�}| �|� dS )z�Handle an incoming empty-element tag. This is only called when the markup looks like <tag/>. :param name: Name of the tag. :param attrs: Dictionary of the tag's attributes. F)�handle_empty_elementN)�handle_starttag� handle_endtag)r �name�attrs�tagr r r �handle_startendtagZ s z*BeautifulSoupHTMLParser.handle_startendtagTc C s� i }|D ]0\}}|du rd}||v r.| j }|| jkrn|d| jfv r'|||< n ||||� n|||< d}q| �� \} } | jj|dd|| | d�} | r[| jr[|r[| j|dd� | j� |� | j du rg| � |� dS dS )a3 Handle an opening tag, e.g. '<tag>' :param name: Name of the tag. :param attrs: Dictionary of the tag's attributes. :param handle_empty_element: True if this tag is known to be an empty-element tag (i.e. there is not expected to be any closing tag). N� z"")� sourceline� sourceposF)�check_already_closed) r �IGNOREr �getpos�soupr" �is_empty_elementr# r �append� _root_tag�_root_tag_encountered) r r$ r% r! � attr_dict�key�value�on_dupe� attrvaluer) r* r& r r r r" i s0 � �z'BeautifulSoupHTMLParser.handle_starttagc C s. |r|| j v r| j �|� dS | j�|� dS )z�Handle a closing tag, e.g. '</tag>' :param name: A tag name. :param check_already_closed: True if this tag is expected to be the closing portion of an empty-element tag, e.g. '<tag></tag>'. N)r �remover. r# )r r$ r+ r r r r# � s z%BeautifulSoupHTMLParser.handle_endtagc C s | j �|� dS )z4Handle some textual data that shows up between tags.N)r. � handle_data�r �datar r r r9 � s z#BeautifulSoupHTMLParser.handle_datac C s� |� d�rt|�d�d�}n|� d�r t|�d�d�}nt|�}d}|dk rN| jjdfD ]!}|s1q,z t|g��|�}W q, tyM } zW Y d}~q,d}~ww |skzt|�}W n t t fyj } zW Y d}~nd}~ww |pnd}| � |� dS )z�Handle a numeric character reference by converting it to the corresponding Unicode character and treating it as textual data. :param name: Character number, possibly in hexadecimal. �x� �XN� z windows-1252u �) � startswith�int�lstripr. �original_encoding� bytearray�decode�UnicodeDecodeError�chr� ValueError� OverflowErrorr9 )r r$ � real_namer; �encoding�er r r �handle_charref� s2 �� ��z&BeautifulSoupHTMLParser.handle_charrefc C s0 t j�|�}|dur |}nd| }| �|� dS )z�Handle a named entity reference by converting it to the corresponding Unicode character(s) and treating it as textual data. :param name: Name of the entity reference. Nz&%s)r �HTML_ENTITY_TO_CHARACTER�getr9 )r r$ � characterr; r r r �handle_entityref� s z(BeautifulSoupHTMLParser.handle_entityrefc C s& | j �� | j �|� | j �t� dS )zOHandle an HTML comment. :param data: The text of the comment. N)r. �endDatar9 r r: r r r �handle_comment� s z&BeautifulSoupHTMLParser.handle_commentc C s6 | j �� |td�d� }| j �|� | j �t� dS )zYHandle a DOCTYPE declaration. :param data: The text of the declaration. zDOCTYPE N)r. rR �lenr9 r r: r r r � handle_decl� s z#BeautifulSoupHTMLParser.handle_declc C sN |� � �d�rt}|td�d� }nt}| j�� | j�|� | j�|� dS )z{Handle a declaration of unknown type -- probably a CDATA block. :param data: The text of the declaration. zCDATA[N)�upperr@ r rT r r. rR r9 )r r; �clsr r r � unknown_decl s z$BeautifulSoupHTMLParser.unknown_declc C s0 | j �� | j �|� | �|� | j �t� dS )z\Handle a processing instruction. :param data: The text of the instruction. N)r. rR r9 �_document_might_be_xmlr r: r r r � handle_pi s z!BeautifulSoupHTMLParser.handle_piN)T)�__name__� __module__� __qualname__�__doc__r, r r r r' r" r# r9 rM rQ rS rU rX rZ r r r r r $ s 7( r c sR e Zd ZdZdZdZeZeee gZ dZ d � fdd� Z d dd�Z d d � Z� ZS ) r zpA Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser, found in the Python standard library. FTNc sp t � }dD ]}||v r|�|�}|||< qtt| �jdi |�� |p#g }|p'i }|�|� d|d< ||f| _dS )a� Constructor. :param parser_args: Positional arguments to pass into the BeautifulSoupHTMLParser constructor, once it's invoked. :param parser_kwargs: Keyword arguments to pass into the BeautifulSoupHTMLParser constructor, once it's invoked. :param kwargs: Keyword arguments for the superclass constructor. )r F�convert_charrefsNr )�dictr �superr r �update� parser_args)r rc � parser_kwargsr �extra_parser_kwargs�argr5 �� __class__r r r * s � z HTMLParserTreeBuilder.__init__c c s^ � t |t�r|dddfV dS |g}|g}||g}t|||d|d�}|j|j|j|jfV dS )a� Run any preliminary steps necessary to make incoming markup acceptable to the parser. :param markup: Some markup -- probably a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. NFT)�known_definite_encodings�user_encodings�is_html�exclude_encodings)� isinstance�strr �markuprC �declared_html_encoding� contains_replacement_characters) r ro �user_specified_encoding�document_declared_encodingrl ri rj � try_encodings�dammitr r r �prepare_markupC s$ � � �z$HTMLParserTreeBuilder.prepare_markupc C sb | j \}}t|i |��}| j|_z |�|� |�� W n ty+ } zt|��d}~ww g |_dS )z{Run some incoming markup through some parsing process, populating the `BeautifulSoup` object in self.soup. N)rc r r. �feed�close�AssertionErrorr r )r ro r r �parserrL r r r rw t s �� zHTMLParserTreeBuilder.feed)NN)NNN)r[ r\ r] r^ �is_xml� picklable� HTMLPARSER�NAMEr r �features�TRACKS_LINE_NUMBERSr rv rw � __classcell__r r rg r r s �1)r^ � __license__�__all__� html.parserr �sys�warnings� bs4.elementr r r r r � bs4.dammitr r � bs4.builderr r r r r r} r r r r r r �<module> s � z __pycache__/__init__.cpython-310.pyc 0000644 00000045466 15030265652 0013237 0 ustar 00 o �h�_ � @ sf d Z ddlmZ ddlZddlZddlZddlZddlmZm Z m Z m Z m Z m Z mZmZ g d�ZdZdZdZd Zd Zd ZG d d � d e�ZG dd� de�Ze� ZG dd� de�Z G dd� de �Z G dd� de �Z G dd� de�Z dd� Z G dd� de!�Z"d d l#m$Z$ e e$� z d d l#m%Z% e e%� W n e&y� Y nw z d d l#m'Z' e e'� W dS e&y� Y dS w ) �MIT� )� defaultdictN)�CharsetMetaAttributeValue�ContentMetaAttributeValue�RubyParenthesisString�RubyTextString� Stylesheet�Script�TemplateString�nonwhitespace_re)�HTMLTreeBuilder�SAXTreeBuilder� TreeBuilder�TreeBuilderRegistry�fast� permissive�strict�xml�html�html5c @ s e Zd ZdZdZdS )�XMLParsedAsHTMLWarningzWThe warning issued when an HTML parser is used to parse XML that is not XHTML. a� It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor.N)�__name__� __module__� __qualname__�__doc__�MESSAGE� r r �J/usr/local/CyberPanel/lib/python3.10/site-packages/bs4/builder/__init__.pyr # s r c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) r zYA way of looking up TreeBuilder subclasses by their name or by desired features. c C s t t�| _g | _d S �N)r �list�builders_for_feature�builders��selfr r r �__init__/ s z TreeBuilderRegistry.__init__c C s0 |j D ] }| j| �d|� q| j�d|� dS )z�Register a treebuilder based on its advertised features. :param treebuilder_class: A subclass of Treebuilder. its .features attribute should list its features. r N)�featuresr �insertr! )r# �treebuilder_class�featurer r r �register3 s z TreeBuilderRegistry.registerc G s� t | j�dkr dS t |�dkr| jd S t|�}|�� d}d}t |�dkrO|�� }| j�|g �}t |�dkrI|du rB|}t|�}n|�t|��}t |�dks&|du rUdS |D ] }||v ra| S qWdS )aj Look up a TreeBuilder subclass with the desired features. :param features: A list of features to look for. If none are provided, the most recently registered TreeBuilder subclass will be used. :return: A TreeBuilder subclass, or None if there's no registered subclass with all the requested features. r N) �lenr! r �reverse�popr �get�set� intersection)r# r% � candidates� candidate_setr( �we_have_the_feature� candidater r r �lookup= s4 � ��zTreeBuilderRegistry.lookupN)r r r r r$ r) r4 r r r r r * s r c @ s� e Zd ZdZdZg Zg ZdZdZdZ e e �Z e � Zi Ze� ZdZeeeefdd�Zdd� Zd d � Zd d � Zd d� Z ddd�Zdd� Zdd� Zdd� ZdS )r z:Turn a textual document into a Beautiful Soup object tree.z[Unknown tree builder]FNc C sb d| _ || ju r | j}|| _|| ju r| j}|| _|| jkr!| j}|| _|| jkr,| j}|| _ dS )a� Constructor. :param multi_valued_attributes: If this is set to None, the TreeBuilder will not turn any values for attributes like 'class' into lists. Setting this to a dictionary will customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES for an example. Internally, these are called "CDATA list attributes", but that probably doesn't make sense to an end-user, so the argument name is `multi_valued_attributes`. :param preserve_whitespace_tags: A list of tags to treat the way <pre> tags are treated in HTML. Tags in this list are immune from pretty-printing; their contents will always be output as-is. :param string_containers: A dictionary mapping tag names to the classes that should be instantiated to contain the textual contents of those tags. The default is to use NavigableString for every tag, no matter what the name. You can override the default by changing DEFAULT_STRING_CONTAINERS. :param store_line_numbers: If the parser keeps track of the line numbers and positions of the original markup, that information will, by default, be stored in each corresponding `Tag` object. You can turn this off by passing store_line_numbers=False. If the parser you're using doesn't keep track of this information, then setting store_line_numbers=True will do nothing. N) �soup� USE_DEFAULT� DEFAULT_CDATA_LIST_ATTRIBUTES�cdata_list_attributes� DEFAULT_PRESERVE_WHITESPACE_TAGS�preserve_whitespace_tags�TRACKS_LINE_NUMBERS�store_line_numbers�DEFAULT_STRING_CONTAINERS�string_containers)r# �multi_valued_attributesr: r< r> r r r r$ � s $ zTreeBuilder.__init__c C s || _ dS )z�The BeautifulSoup object has been initialized and is now being associated with the TreeBuilder. :param soup: A BeautifulSoup object. N)r5 )r# r5 r r r �initialize_soup� s zTreeBuilder.initialize_soupc C � dS )z�Do any work necessary to reset the underlying parser for a new document. By default, this does nothing. Nr r" r r r �reset� s zTreeBuilder.resetc C s | j du rdS || j v S )a� Might a tag with this name be an empty-element tag? The final markup may or may not actually present this tag as self-closing. For instance: an HTMLBuilder does not consider a <p> tag to be an empty-element tag (it's not in HTMLBuilder.empty_element_tags). This means an empty <p> tag will be presented as "<p></p>", not "<p/>" or "<p>". The default implementation has no opinion about which tags are empty-element tags, so a tag will be presented as an empty-element tag if and only if it has no children. "<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will be left alone. :param tag_name: The name of a markup tag. NT)�empty_element_tags)r# �tag_namer r r �can_be_empty_element� s z TreeBuilder.can_be_empty_elementc C � t � �)z�Run some incoming markup through some parsing process, populating the `BeautifulSoup` object in self.soup. This method is not implemented in TreeBuilder; it must be implemented in subclasses. :return: None. ��NotImplementedError�r# �markupr r r �feed� s zTreeBuilder.feedc c s � |dddfV dS )a Run any preliminary steps necessary to make incoming markup acceptable to the parser. :param markup: Some markup -- probably a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. NOTE: This argument is not used by the calling code and can probably be removed. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. By default, the only strategy is to parse the markup as-is. See `LXMLTreeBuilderForXML` and `HTMLParserTreeBuilder` for implementations that take into account the quirks of particular parsers. NFr )r# rJ �user_specified_encoding�document_declared_encoding�exclude_encodingsr r r �prepare_markup� s �zTreeBuilder.prepare_markupc C s |S )a Wrap an HTML fragment to make it look like a document. Different parsers do this differently. For instance, lxml introduces an empty <head> tag, and html5lib doesn't. Abstracting this away lets us write simple tests which run HTML fragments through the parser and compare the results against other HTML fragments. This method should not be used outside of tests. :param fragment: A string -- fragment of HTML. :return: A string -- a full HTML document. r )r# �fragmentr r r �test_fragment_to_document s z%TreeBuilder.test_fragment_to_documentc C rA )a2 Set up any substitutions that will need to be performed on a `Tag` when it's output as a string. By default, this does nothing. See `HTMLTreeBuilder` for a case where this is used. :param tag: A `Tag` :return: Whether or not a substitution was performed. Fr )r# �tagr r r �set_up_substitutions s z TreeBuilder.set_up_substitutionsc C s� |s|S | j r?| j �dg �}| j �|�� d�}t|�� �D ]!}||v s)|r>||v r>|| }t|t�r8t�|�}n|}|||< q |S )a� When an attribute value is associated with a tag that can have multiple values for that attribute, convert the string value to a list of strings. Basically, replaces class="foo bar" with class=["foo", "bar"] NOTE: This method modifies its input in place. :param tag_name: The name of a tag. :param attrs: A dictionary containing the tag's attributes. Any appropriate attribute values will be modified in place. �*N) r8 r- �lowerr �keys� isinstance�strr �findall)r# rD �attrs� universal� tag_specific�attr�value�valuesr r r �$_replace_cdata_list_attribute_values' s � �z0TreeBuilder._replace_cdata_list_attribute_values)NNN) r r r r �NAME�ALTERNATE_NAMESr% �is_xml� picklablerC r r r7 r. r9 r= �objectr6 r; r$ r@ rB rE rK rO rQ rS r` r r r r r o s8 �2 � r c @ sh e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z dd� ZdS )r z�A Beautiful Soup treebuilder that listens for SAX events. This is not currently used for anything, but it demonstrates how a simple TreeBuilder would work. c C rF r rG rI r r r rK T s zSAXTreeBuilder.feedc C � d S r r r" r r r �closeW � zSAXTreeBuilder.closec C s, t dd� t|�� �D ��}| j�||� d S )Nc s s � | ] \}}|d |fV qdS )� Nr )�.0�keyr^ r r r � <genexpr>[ s � z.SAXTreeBuilder.startElement.<locals>.<genexpr>)�dictr �itemsr5 �handle_starttag)r# �namerZ r r r � startElementZ s zSAXTreeBuilder.startElementc C � | j �|� d S r )r5 � handle_endtag�r# rp r r r � endElement_ � zSAXTreeBuilder.endElementc C s | � ||� d S r )rq )r# �nsTuple�nodeNamerZ r r r �startElementNSc rv z SAXTreeBuilder.startElementNSc C s | � |� d S r )ru )r# rw rx r r r � endElementNSg s zSAXTreeBuilder.endElementNSc C rf r r )r# �prefix� nodeValuer r r �startPrefixMappingl s z!SAXTreeBuilder.startPrefixMappingc C rf r r )r# r{ r r r �endPrefixMappingp s z SAXTreeBuilder.endPrefixMappingc C rr r )r5 � handle_data)r# �contentr r r � charactersu s zSAXTreeBuilder.charactersc C rf r r r" r r r � startDocumentx rh z SAXTreeBuilder.startDocumentc C rf r r r" r r r � endDocument{ rh zSAXTreeBuilder.endDocumentN)r r r r rK rg rq ru ry rz r} r~ r� r� r� r r r r r M s r c @ s� e Zd ZdZeg d��Zeg d��Zeee e e d�Z g d�ddgddgdgdgdgd gd gdgd gd gd gd� Z eddg�Zdd� ZdS )r z]This TreeBuilder knows facts about HTML. Such as which tags are empty-element tags. )�area�base�br�col�embed�hr�img�input�keygen�link�menuitem�meta�param�source�track�wbr�basefont�bgsound�command�frame�image�isindex�nextid�spacer)#�address�article�aside� blockquote�canvas�dd�div�dl�dt�fieldset� figcaption�figure�footer�form�h1�h2�h3�h4�h5�h6�headerr� �li�main�nav�noscript�ol�output�p�pre�section�table�tfoot�ul�video)�rt�rp�style�script�template)�class� accesskey�dropzone�rel�rev�headerszaccept-charset�archive�sizes�sandbox�for) rT �ar� �td�thr� r� re r� �icon�iframer� r� �textareac C s� |j dkrdS |�d�}|�d�}|�d�}d}|dur(|}t|�|d< |duS |dur<|dur<|�� dkr<t|�|d< |duS )a� Replace the declared encoding in a <meta> tag with a placeholder, to be substituted when the tag is output to a string. An HTML document may come in to Beautiful Soup as one encoding, but exit in a different encoding, and the <meta> tag needs to be changed to reflect this. :param tag: A `Tag` :return: Whether or not a substitution was performed. r� Fz http-equivr� �charsetNz content-type)rp r- r rU r )r# rR � http_equivr� r� � meta_encodingr r r rS � s � z$HTMLTreeBuilder.set_up_substitutionsN)r r r r r. rC �block_elementsr r r r r r= r7 r9 rS r r r r r s2 �� r c @ sh e Zd ZdZe�dej�Ze�dej�ZdZ dZ e ddd��Z e dd d ��Z d d � Zdd� Zdd� ZdS )�DetectsXMLParsedAsHTMLa A mixin class for any class (a TreeBuilder, or some class used by a TreeBuilder) that's in a position to detect whether an XML document is being incorrectly parsed as HTML, and issue an appropriate warning. This requires being able to observe an incoming processing instruction that might be an XML declaration, and also able to observe tags as they're opened. If you can't do that for a given TreeBuilder, there's a less reliable implementation based on examining the raw markup. z <[^ +]htmls <[^ +]htmlz<?xmls <?xml� c C s` t |t�r | j}| j}n| j}| j}|dur.|�|�r.|�|dd� �s.| j|d d� dS dS )a� Perform a check on some markup to see if it looks like XML that's not XHTML. If so, issue a warning. This is much less reliable than doing the check while parsing, but some of the tree builders can't do that. :param stacklevel: The stacklevel of the code calling this function. :return: True if the markup looks like non-XHTML XML, False otherwise. Ni� � �� stacklevelTF) rW �bytes� XML_PREFIX_B�LOOKS_LIKE_HTML_B� XML_PREFIX�LOOKS_LIKE_HTML� startswith�search�_warn)�clsrJ r� r{ �looks_like_htmlr r r � warn_if_markup_looks_like_xml s ��z4DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml� c C s t jtjt|d� dS )z/Issue a warning about XML being parsed as HTML.r� N)�warnings�warnr r )r� r� r r r r� " s �z DetectsXMLParsedAsHTML._warnc C s d| _ d| _dS )z+Call this method before parsing a document.N�� _first_processing_instruction� _root_tagr" r r r �_initialize_xml_detector* s z/DetectsXMLParsedAsHTML._initialize_xml_detectorc C s" | j dus | jdur dS || _ dS )z�Call this method when encountering an XML declaration, or a "processing instruction" that might be an XML declaration. Nr� )r# �processing_instructionr r r �_document_might_be_xml/ s z-DetectsXMLParsedAsHTML._document_might_be_xmlc C sN | j durdS || _ |dkr!| jdur#| j�� �d�r%| �� dS dS dS dS )z�Call this when you encounter the document's root tag. This is where we actually check whether an XML document is being incorrectly parsed as HTML, and issue the warning. Nr zxml )r� r� rU r� r� rt r r r �_root_tag_encountered> s � �z,DetectsXMLParsedAsHTML._root_tag_encounteredN)r� )r� )r r r r �re�compile�Ir� r� r� r� � classmethodr� r� r� r� r� r r r r r� � s r� c C sR t jt }| jD ] }t| |�}t|t�r&t|||� |j�|� |j � |� qdS )z9Copy TreeBuilders from the given module into this module.N) �sys�modulesr �__all__�getattr� issubclassr �setattr�append�builder_registryr) )�module� this_modulerp �objr r r �register_treebuilders_fromR s ��r c s e Zd ZdZ� fdd�Z� ZS )�ParserRejectedMarkupzgAn Exception to be raised when the underlying parser simply refuses to parse the given markup. c s6 t |t�r|}d|jjt|�f }tt| ��|� dS )zzExplain why the parser rejected the given markup, either with a textual explanation or another exception. z%s: %sN)rW � Exception� __class__r rX �superr r$ )r# �message_or_exception�e�r r r r$ b s z ParserRejectedMarkup.__init__)r r r r r$ � __classcell__r r r r r ^ s r ri )� _htmlparser)� _html5lib)�_lxml)(� __license__� collectionsr � itertoolsr� r� r� � bs4.elementr r r r r r r r r� �FAST� PERMISSIVE�STRICT�XML�HTML�HTML_5� UserWarningr re r r� r r r r� r r r � r r � ImportErrorr r r r r �<module> sL ( C _2qb � � __pycache__/_html5lib.cpython-310.pyc 0000644 00000030542 15030265652 0013344 0 ustar 00 o �h�J � @ s d Z dgZddlZddlZddlmZmZmZmZm Z ddl m Z m Z ddl Z ddlmZmZ ddl mZmZmZmZ z ddlmZ d ZW n ey_ Z zdd lmZ d ZW Y dZ[ndZ[ww G d d� de �Z G d d� dej �Z G dd� de �Z G dd� dej!�Z"G dd� de"�Z#dS )�MIT�HTML5TreeBuilder� N)�DetectsXMLParsedAsHTML� PERMISSIVE�HTML�HTML_5�HTMLTreeBuilder)�NamespacedAttribute�nonwhitespace_re)� namespaces�prefixes)�Comment�Doctype�NavigableString�Tag)�_baseF)�baseTc @ sH e Zd ZdZdZeeeegZdZ d dd�Z dd� Z d d � Z d d � Z dS )r a Use html5lib to build a tree. Note that this TreeBuilder does not support some features common to HTML TreeBuilders. Some of these features could theoretically be implemented, but at the very least it's quite difficult, because html5lib moves the parse tree around as it's being built. * This TreeBuilder doesn't use different subclasses of NavigableString based on the name of the tag in which the string was found. * You can't use a SoupStrainer to parse only part of a document. �html5libTNc c s: � || _ |r tjddd� tj|dd� |d d dfV d S )NzjYou provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.� �� stacklevelF)�user_specified_encoding�warnings�warnr � warn_if_markup_looks_like_xml)�self�markupr �document_declared_encoding�exclude_encodings� r �K/usr/local/CyberPanel/lib/python3.10/site-packages/bs4/builder/_html5lib.py�prepare_markup@ s ���z HTML5TreeBuilder.prepare_markupc C s� | j jd ur tjddd� tj| jd�}|| j_t � }t |t �s-t r(| j |d< n| j |d< |j|fi |��}t |t �r?d |_n|jjjd }t |t �sN|j}||_d | j_d S )Nz�You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.� r )�tree�override_encoding�encodingr )�soup� parse_onlyr r r � HTMLParser�create_treebuilder�underlying_builder�parser�dict� isinstance�str� new_html5libr �parse�original_encoding� tokenizer�stream� charEncoding�name)r r r+ � extra_kwargs�docr1 r r r �feedW s( � zHTML5TreeBuilder.feedc C s t || j| jd�| _| jS )N)�store_line_numbers)�TreeBuilderForHtml5libr& r9 r* )r �namespaceHTMLElementsr r r r) v s �z#HTML5TreeBuilder.create_treebuilderc C s d| S )zSee `TreeBuilder`.z)<html><head></head><body>%s</body></html>r )r �fragmentr r r �test_fragment_to_document} s z*HTML5TreeBuilder.test_fragment_to_document)NN)�__name__� __module__� __qualname__�__doc__�NAMEr r r �features�TRACKS_LINE_NUMBERSr! r8 r) r= r r r r r * s � c sj e Zd Z d� fdd� Zdd� Zdd� Zd d � Zd d � Zd d� Zdd� Z dd� Z dd� Z dd� Z � Z S )r: NTc sN |r|| _ nddlm} | dd|i|��| _ tt| ��|� d | _|| _d S )Nr �� BeautifulSoup� � html.parserr9 )rG rH )r& �bs4rF �superr: �__init__r+ r9 )r r; r& r9 �kwargsrF �� __class__r r rK � s ��� z TreeBuilderForHtml5lib.__init__c C s | j �� t| j | j d �S �N)r& �reset�Element�r r r r � documentClass� s z$TreeBuilderForHtml5lib.documentClassc C s6 |d }|d }|d }t �|||�}| j�|� d S )Nr5 �publicId�systemId)r �for_name_and_idsr& �object_was_parsed)r �tokenr5 rT rU �doctyper r r � insertDoctype� s z$TreeBuilderForHtml5lib.insertDoctypec C sZ i }| j r| jr| j jj�� \}}||d< |d |d< | jj||fi |��}t|| j|�S )N� sourceline� � sourcepos)r+ r9 r2 r3 �positionr& �new_tagrQ )r r5 � namespacerL r[ r] �tagr r r � elementClass� s z#TreeBuilderForHtml5lib.elementClassc C s t t|�| j�S rO )�TextNoder r& )r �datar r r � commentClass� s z#TreeBuilderForHtml5lib.commentClassc C s0 ddl m} |dd�| _d| j_t| j| jd �S )Nr rE rG rH z[document_fragment])rI rF r& r5 rQ )r rF r r r � fragmentClass� s z$TreeBuilderForHtml5lib.fragmentClassc C s | j �|j� d S rO )r& �append�element�r �noder r r � appendChild� s z"TreeBuilderForHtml5lib.appendChildc C s | j S rO )r& rR r r r � getDocument� s z"TreeBuilderForHtml5lib.getDocumentc C s t j�| �jS rO )�treebuilder_base� TreeBuilder� getFragmentrh rR r r r ro � � z"TreeBuilderForHtml5lib.getFragmentc sB ddl m� g �t�d��d� ���fdd� ��|d� d���S )Nr rE z8^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$c s� t | � �r t | t�rW��| �}|rK|�d�}|jdkr>|�d�p"d}|�d�p.|�d�p.d}��dd| |||f � d S ��dd| |f � d S ��d d| f � d S t | t�ri��d d| | f � d S t | t�r{��d d| | f � d S | jr�d t | j | j f }n| j }��d d| |f � | j r�g }t | j � � �D ]$\}}t |t�r�d t |j |j f }t |t �r�d�|�}|�||f� q�t|�D ]\}}��dd|d ||f � q�|d7 }| jD ]}�||� q�d S )Nr\ � rG r r" z|%s<!DOCTYPE %s "%s" "%s">� z|%s<!DOCTYPE %s>z|%s<!DOCTYPE >z|%s<!-- %s -->z|%s"%s"z%s %sz|%s<%s>z |%s%s="%s")r- r �match�group� lastindexrg r r r` r r5 �attrs�list�itemsr �join�sorted�children) rh �indent�mr5 rT rU � attributes�value�child�rF � doctype_re�rv�serializeElementr r r� � sL � � �z?TreeBuilderForHtml5lib.testSerializer.<locals>.serializeElement� )r )rI rF �re�compilery �r rh r r� r �testSerializer� s ) z%TreeBuilderForHtml5lib.testSerializer)NT)r> r? r@ rK rS rZ rb re rf rk rl ro r� � __classcell__r r rM r r: � s � r: c @ sL e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�AttrListc C s || _ t| j j�| _d S rO )rh r, rv r� r r r rK � s zAttrList.__init__c C s t | j�� ��� S rO )rw rv rx �__iter__rR r r r r� � � zAttrList.__iter__c C s^ | j jpi }||�dg �v s | j j|v r(||�| j jg �v r(t|t�s(t�|�}|| j |< d S )N�*)rh �cdata_list_attributes�getr5 r- rw r �findall)r r5 r � list_attrr r r � __setitem__ s zAttrList.__setitem__c C � t | j�� �S rO )rw rv rx rR r r r rx rp zAttrList.itemsc C r� rO �rw rv �keysrR r r r r� rp z AttrList.keysc C s t | j�S rO )�lenrv rR r r r �__len__ � zAttrList.__len__c C s | j | S rO )rv �r r5 r r r � __getitem__ r� zAttrList.__getitem__c C s |t | j�� �v S rO r� r� r r r � __contains__ r� zAttrList.__contains__N) r> r? r@ rK r� r� rx r� r� r� r� r r r r r� � s r� c @ sx e Zd Zdd� Zdd� Zdd� Zdd� Zeee�Zdd d �Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� Zee�Zd S )rQ c C s& t j�| |j� || _|| _|| _d S rO )rm �NoderK r5 rh r& r` )r rh r& r` r r r rK s zElement.__init__c C s* d }}t |t�r| }}n t |t�r|}n|jjtkr%|j }}| |_n|j}| |_t |t�s:|jd ur:|j�� |d urd| jjrd| jjd jtkrd| jjd }| j � || �}|� |� || j _ d S t |t�ro| j � |�}| jjrz| j� d�}n| jjd ur�| j � � }n| j}| j j|| j|d� d S )N���F)�parent�most_recent_element)r- r. r rh rN r r� �extract�contentsr& � new_string� replace_with�_most_recent_element�_last_descendant� next_elementrW )r rj � string_childr� � old_element� new_elementr� r r r rk s: �zElement.appendChildc C s t | jt�ri S t| j�S rO )r- rh r r� rR r r r � getAttributesU s zElement.getAttributesc C s� |d urJt |�dkrLg }t|�� �D ]\}}t|t�r&t|� }||= |||< q| jj�| j |� t|�� �D ] \}}|| j |< q6| jj� | j � d S d S d S )Nr ) r� rw rx r- �tupler r& �builder�$_replace_cdata_list_attribute_valuesr5 rh �set_up_substitutions)r r~ �converted_attributesr5 r �new_namer r r � setAttributesZ s �� �zElement.setAttributesNc C s6 t | j�|�| j�}|r| �||� d S | �|� d S rO )rc r& r� � insertBeforerk )r rd r� �textr r r � insertTextp s zElement.insertTextc C s� | j �|j �}|j jtkr4| j jr4| j j|d jtkr4| j j|d }| j�||j �}|�|� d S | j �||j � | |_ d S )Nr\ ) rh �indexrN r r� r& r� r� �insertr� )r rj �refNoder� �old_node�new_strr r r r� w s zElement.insertBeforec C s |j �� d S rO )rh r� ri r r r � removeChild� rp zElement.removeChildc C s� | j }|j }|j}|�dd�}t|j�dkr |jd }|j}nd}|j}|j}t|�dkrf|d } |dur9|| _n|| _|| _|durG| |_n| |_|durQ| |_|d �dd�} || _|durc| |_d| _|D ] } || _|j� | � qhg |_||_dS )z1Move all of this tag's children into another tag.Fr r� NT) rh � next_siblingr� r� r� r� �previous_element�previous_siblingr� rg ) r � new_parentrh �new_parent_element�final_next_element�new_parents_last_descendant�new_parents_last_child�(new_parents_last_descendant_next_element� to_append� first_child�last_childs_last_descendantr� r r r �reparentChildren� s> zElement.reparentChildrenc C sB | j �| jj| j�}t|| j | j�}| jD ] \}}||j|< q|S rO )r& r_ rh r5 r` rQ r~ )r ra rj �keyr r r r � cloneNode� s zElement.cloneNodec C s | j jS rO )rh r� rR r r r � hasContent� s zElement.hasContentc C s$ | j d kr td | jfS | j | jfS )N�html)r` r r5 rR r r r � getNameTuple� s zElement.getNameTuplerO )r> r? r@ rK rk r� r� �propertyr~ r� r� r� r� r� r� r� � nameTupler r r r rQ s 6 B rQ c @ s e Zd Zdd� Zdd� ZdS )rc c C s t j�| d � || _|| _d S rO )rm r� rK rh r& )r rh r& r r r rK � s zTextNode.__init__c C s t �rO )�NotImplementedErrorrR r r r r� � s zTextNode.cloneNodeN)r> r? r@ rK r� r r r r rc � s rc )$� __license__�__all__r r� � bs4.builderr r r r r � bs4.elementr r r �html5lib.constantsr r r r r r �html5lib.treebuildersr rm r/ � ImportError�er r rn r: �objectr� r� rQ rc r r r r �<module> s0 � ��Xx C _htmlparser.py 0000644 00000035113 15030265652 0007445 0 ustar 00 # encoding: utf-8 """Use the HTMLParser library to parse HTML files that aren't too bad.""" # Use of this source code is governed by the MIT license. __license__ = "MIT" __all__ = [ 'HTMLParserTreeBuilder', ] from html.parser import HTMLParser import sys import warnings from bs4.element import ( CData, Comment, Declaration, Doctype, ProcessingInstruction, ) from bs4.dammit import EntitySubstitution, UnicodeDammit from bs4.builder import ( DetectsXMLParsedAsHTML, ParserRejectedMarkup, HTML, HTMLTreeBuilder, STRICT, ) HTMLPARSER = 'html.parser' class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML): """A subclass of the Python standard library's HTMLParser class, which listens for HTMLParser events and translates them into calls to Beautiful Soup's tree construction API. """ # Strategies for handling duplicate attributes IGNORE = 'ignore' REPLACE = 'replace' def __init__(self, *args, **kwargs): """Constructor. :param on_duplicate_attribute: A strategy for what to do if a tag includes the same attribute more than once. Accepted values are: REPLACE (replace earlier values with later ones, the default), IGNORE (keep the earliest value encountered), or a callable. A callable must take three arguments: the dictionary of attributes already processed, the name of the duplicate attribute, and the most recent value encountered. """ self.on_duplicate_attribute = kwargs.pop( 'on_duplicate_attribute', self.REPLACE ) HTMLParser.__init__(self, *args, **kwargs) # Keep a list of empty-element tags that were encountered # without an explicit closing tag. If we encounter a closing tag # of this type, we'll associate it with one of those entries. # # This isn't a stack because we don't care about the # order. It's a list of closing tags we've already handled and # will ignore, assuming they ever show up. self.already_closed_empty_element = [] self._initialize_xml_detector() def error(self, message): # NOTE: This method is required so long as Python 3.9 is # supported. The corresponding code is removed from HTMLParser # in 3.5, but not removed from ParserBase until 3.10. # https://github.com/python/cpython/issues/76025 # # The original implementation turned the error into a warning, # but in every case I discovered, this made HTMLParser # immediately crash with an error message that was less # helpful than the warning. The new implementation makes it # more clear that html.parser just can't parse this # markup. The 3.10 implementation does the same, though it # raises AssertionError rather than calling a method. (We # catch this error and wrap it in a ParserRejectedMarkup.) raise ParserRejectedMarkup(message) def handle_startendtag(self, name, attrs): """Handle an incoming empty-element tag. This is only called when the markup looks like <tag/>. :param name: Name of the tag. :param attrs: Dictionary of the tag's attributes. """ # is_startend() tells handle_starttag not to close the tag # just because its name matches a known empty-element tag. We # know that this is an empty-element tag and we want to call # handle_endtag ourselves. tag = self.handle_starttag(name, attrs, handle_empty_element=False) self.handle_endtag(name) def handle_starttag(self, name, attrs, handle_empty_element=True): """Handle an opening tag, e.g. '<tag>' :param name: Name of the tag. :param attrs: Dictionary of the tag's attributes. :param handle_empty_element: True if this tag is known to be an empty-element tag (i.e. there is not expected to be any closing tag). """ # XXX namespace attr_dict = {} for key, value in attrs: # Change None attribute values to the empty string # for consistency with the other tree builders. if value is None: value = '' if key in attr_dict: # A single attribute shows up multiple times in this # tag. How to handle it depends on the # on_duplicate_attribute setting. on_dupe = self.on_duplicate_attribute if on_dupe == self.IGNORE: pass elif on_dupe in (None, self.REPLACE): attr_dict[key] = value else: on_dupe(attr_dict, key, value) else: attr_dict[key] = value attrvalue = '""' #print("START", name) sourceline, sourcepos = self.getpos() tag = self.soup.handle_starttag( name, None, None, attr_dict, sourceline=sourceline, sourcepos=sourcepos ) if tag and tag.is_empty_element and handle_empty_element: # Unlike other parsers, html.parser doesn't send separate end tag # events for empty-element tags. (It's handled in # handle_startendtag, but only if the original markup looked like # <tag/>.) # # So we need to call handle_endtag() ourselves. Since we # know the start event is identical to the end event, we # don't want handle_endtag() to cross off any previous end # events for tags of this name. self.handle_endtag(name, check_already_closed=False) # But we might encounter an explicit closing tag for this tag # later on. If so, we want to ignore it. self.already_closed_empty_element.append(name) if self._root_tag is None: self._root_tag_encountered(name) def handle_endtag(self, name, check_already_closed=True): """Handle a closing tag, e.g. '</tag>' :param name: A tag name. :param check_already_closed: True if this tag is expected to be the closing portion of an empty-element tag, e.g. '<tag></tag>'. """ #print("END", name) if check_already_closed and name in self.already_closed_empty_element: # This is a redundant end tag for an empty-element tag. # We've already called handle_endtag() for it, so just # check it off the list. #print("ALREADY CLOSED", name) self.already_closed_empty_element.remove(name) else: self.soup.handle_endtag(name) def handle_data(self, data): """Handle some textual data that shows up between tags.""" self.soup.handle_data(data) def handle_charref(self, name): """Handle a numeric character reference by converting it to the corresponding Unicode character and treating it as textual data. :param name: Character number, possibly in hexadecimal. """ # TODO: This was originally a workaround for a bug in # HTMLParser. (http://bugs.python.org/issue13633) The bug has # been fixed, but removing this code still makes some # Beautiful Soup tests fail. This needs investigation. if name.startswith('x'): real_name = int(name.lstrip('x'), 16) elif name.startswith('X'): real_name = int(name.lstrip('X'), 16) else: real_name = int(name) data = None if real_name < 256: # HTML numeric entities are supposed to reference Unicode # code points, but sometimes they reference code points in # some other encoding (ahem, Windows-1252). E.g. “ # instead of É for LEFT DOUBLE QUOTATION MARK. This # code tries to detect this situation and compensate. for encoding in (self.soup.original_encoding, 'windows-1252'): if not encoding: continue try: data = bytearray([real_name]).decode(encoding) except UnicodeDecodeError as e: pass if not data: try: data = chr(real_name) except (ValueError, OverflowError) as e: pass data = data or "\N{REPLACEMENT CHARACTER}" self.handle_data(data) def handle_entityref(self, name): """Handle a named entity reference by converting it to the corresponding Unicode character(s) and treating it as textual data. :param name: Name of the entity reference. """ character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name) if character is not None: data = character else: # If this were XML, it would be ambiguous whether "&foo" # was an character entity reference with a missing # semicolon or the literal string "&foo". Since this is # HTML, we have a complete list of all character entity references, # and this one wasn't found, so assume it's the literal string "&foo". data = "&%s" % name self.handle_data(data) def handle_comment(self, data): """Handle an HTML comment. :param data: The text of the comment. """ self.soup.endData() self.soup.handle_data(data) self.soup.endData(Comment) def handle_decl(self, data): """Handle a DOCTYPE declaration. :param data: The text of the declaration. """ self.soup.endData() data = data[len("DOCTYPE "):] self.soup.handle_data(data) self.soup.endData(Doctype) def unknown_decl(self, data): """Handle a declaration of unknown type -- probably a CDATA block. :param data: The text of the declaration. """ if data.upper().startswith('CDATA['): cls = CData data = data[len('CDATA['):] else: cls = Declaration self.soup.endData() self.soup.handle_data(data) self.soup.endData(cls) def handle_pi(self, data): """Handle a processing instruction. :param data: The text of the instruction. """ self.soup.endData() self.soup.handle_data(data) self._document_might_be_xml(data) self.soup.endData(ProcessingInstruction) class HTMLParserTreeBuilder(HTMLTreeBuilder): """A Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser, found in the Python standard library. """ is_xml = False picklable = True NAME = HTMLPARSER features = [NAME, HTML, STRICT] # The html.parser knows which line number and position in the # original file is the source of an element. TRACKS_LINE_NUMBERS = True def __init__(self, parser_args=None, parser_kwargs=None, **kwargs): """Constructor. :param parser_args: Positional arguments to pass into the BeautifulSoupHTMLParser constructor, once it's invoked. :param parser_kwargs: Keyword arguments to pass into the BeautifulSoupHTMLParser constructor, once it's invoked. :param kwargs: Keyword arguments for the superclass constructor. """ # Some keyword arguments will be pulled out of kwargs and placed # into parser_kwargs. extra_parser_kwargs = dict() for arg in ('on_duplicate_attribute',): if arg in kwargs: value = kwargs.pop(arg) extra_parser_kwargs[arg] = value super(HTMLParserTreeBuilder, self).__init__(**kwargs) parser_args = parser_args or [] parser_kwargs = parser_kwargs or {} parser_kwargs.update(extra_parser_kwargs) parser_kwargs['convert_charrefs'] = False self.parser_args = (parser_args, parser_kwargs) def prepare_markup(self, markup, user_specified_encoding=None, document_declared_encoding=None, exclude_encodings=None): """Run any preliminary steps necessary to make incoming markup acceptable to the parser. :param markup: Some markup -- probably a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. """ if isinstance(markup, str): # Parse Unicode as-is. yield (markup, None, None, False) return # Ask UnicodeDammit to sniff the most likely encoding. # This was provided by the end-user; treat it as a known # definite encoding per the algorithm laid out in the HTML5 # spec. (See the EncodingDetector class for details.) known_definite_encodings = [user_specified_encoding] # This was found in the document; treat it as a slightly lower-priority # user encoding. user_encodings = [document_declared_encoding] try_encodings = [user_specified_encoding, document_declared_encoding] dammit = UnicodeDammit( markup, known_definite_encodings=known_definite_encodings, user_encodings=user_encodings, is_html=True, exclude_encodings=exclude_encodings ) yield (dammit.markup, dammit.original_encoding, dammit.declared_html_encoding, dammit.contains_replacement_characters) def feed(self, markup): """Run some incoming markup through some parsing process, populating the `BeautifulSoup` object in self.soup. """ args, kwargs = self.parser_args parser = BeautifulSoupHTMLParser(*args, **kwargs) parser.soup = self.soup try: parser.feed(markup) parser.close() except AssertionError as e: # html.parser raises AssertionError in rare cases to # indicate a fatal problem with the markup, especially # when there's an error in the doctype declaration. raise ParserRejectedMarkup(e) parser.already_closed_empty_element = [] _html5lib.py 0000644 00000045252 15030265652 0007011 0 ustar 00 # Use of this source code is governed by the MIT license. __license__ = "MIT" __all__ = [ 'HTML5TreeBuilder', ] import warnings import re from bs4.builder import ( DetectsXMLParsedAsHTML, PERMISSIVE, HTML, HTML_5, HTMLTreeBuilder, ) from bs4.element import ( NamespacedAttribute, nonwhitespace_re, ) import html5lib from html5lib.constants import ( namespaces, prefixes, ) from bs4.element import ( Comment, Doctype, NavigableString, Tag, ) try: # Pre-0.99999999 from html5lib.treebuilders import _base as treebuilder_base new_html5lib = False except ImportError as e: # 0.99999999 and up from html5lib.treebuilders import base as treebuilder_base new_html5lib = True class HTML5TreeBuilder(HTMLTreeBuilder): """Use html5lib to build a tree. Note that this TreeBuilder does not support some features common to HTML TreeBuilders. Some of these features could theoretically be implemented, but at the very least it's quite difficult, because html5lib moves the parse tree around as it's being built. * This TreeBuilder doesn't use different subclasses of NavigableString based on the name of the tag in which the string was found. * You can't use a SoupStrainer to parse only part of a document. """ NAME = "html5lib" features = [NAME, PERMISSIVE, HTML_5, HTML] # html5lib can tell us which line number and position in the # original file is the source of an element. TRACKS_LINE_NUMBERS = True def prepare_markup(self, markup, user_specified_encoding, document_declared_encoding=None, exclude_encodings=None): # Store the user-specified encoding for use later on. self.user_specified_encoding = user_specified_encoding # document_declared_encoding and exclude_encodings aren't used # ATM because the html5lib TreeBuilder doesn't use # UnicodeDammit. if exclude_encodings: warnings.warn( "You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.", stacklevel=3 ) # html5lib only parses HTML, so if it's given XML that's worth # noting. DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml( markup, stacklevel=3 ) yield (markup, None, None, False) # These methods are defined by Beautiful Soup. def feed(self, markup): if self.soup.parse_only is not None: warnings.warn( "You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.", stacklevel=4 ) parser = html5lib.HTMLParser(tree=self.create_treebuilder) self.underlying_builder.parser = parser extra_kwargs = dict() if not isinstance(markup, str): if new_html5lib: extra_kwargs['override_encoding'] = self.user_specified_encoding else: extra_kwargs['encoding'] = self.user_specified_encoding doc = parser.parse(markup, **extra_kwargs) # Set the character encoding detected by the tokenizer. if isinstance(markup, str): # We need to special-case this because html5lib sets # charEncoding to UTF-8 if it gets Unicode input. doc.original_encoding = None else: original_encoding = parser.tokenizer.stream.charEncoding[0] if not isinstance(original_encoding, str): # In 0.99999999 and up, the encoding is an html5lib # Encoding object. We want to use a string for compatibility # with other tree builders. original_encoding = original_encoding.name doc.original_encoding = original_encoding self.underlying_builder.parser = None def create_treebuilder(self, namespaceHTMLElements): self.underlying_builder = TreeBuilderForHtml5lib( namespaceHTMLElements, self.soup, store_line_numbers=self.store_line_numbers ) return self.underlying_builder def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<html><head></head><body>%s</body></html>' % fragment class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder): def __init__(self, namespaceHTMLElements, soup=None, store_line_numbers=True, **kwargs): if soup: self.soup = soup else: from bs4 import BeautifulSoup # TODO: Why is the parser 'html.parser' here? To avoid an # infinite loop? self.soup = BeautifulSoup( "", "html.parser", store_line_numbers=store_line_numbers, **kwargs ) # TODO: What are **kwargs exactly? Should they be passed in # here in addition to/instead of being passed to the BeautifulSoup # constructor? super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements) # This will be set later to an html5lib.html5parser.HTMLParser # object, which we can use to track the current line number. self.parser = None self.store_line_numbers = store_line_numbers def documentClass(self): self.soup.reset() return Element(self.soup, self.soup, None) def insertDoctype(self, token): name = token["name"] publicId = token["publicId"] systemId = token["systemId"] doctype = Doctype.for_name_and_ids(name, publicId, systemId) self.soup.object_was_parsed(doctype) def elementClass(self, name, namespace): kwargs = {} if self.parser and self.store_line_numbers: # This represents the point immediately after the end of the # tag. We don't know when the tag started, but we do know # where it ended -- the character just before this one. sourceline, sourcepos = self.parser.tokenizer.stream.position() kwargs['sourceline'] = sourceline kwargs['sourcepos'] = sourcepos-1 tag = self.soup.new_tag(name, namespace, **kwargs) return Element(tag, self.soup, namespace) def commentClass(self, data): return TextNode(Comment(data), self.soup) def fragmentClass(self): from bs4 import BeautifulSoup # TODO: Why is the parser 'html.parser' here? To avoid an # infinite loop? self.soup = BeautifulSoup("", "html.parser") self.soup.name = "[document_fragment]" return Element(self.soup, self.soup, None) def appendChild(self, node): # XXX This code is not covered by the BS4 tests. self.soup.append(node.element) def getDocument(self): return self.soup def getFragment(self): return treebuilder_base.TreeBuilder.getFragment(self).element def testSerializer(self, element): from bs4 import BeautifulSoup rv = [] doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$') def serializeElement(element, indent=0): if isinstance(element, BeautifulSoup): pass if isinstance(element, Doctype): m = doctype_re.match(element) if m: name = m.group(1) if m.lastindex > 1: publicId = m.group(2) or "" systemId = m.group(3) or m.group(4) or "" rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % (' ' * indent, name, publicId, systemId)) else: rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name)) else: rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) elif isinstance(element, Comment): rv.append("|%s<!-- %s -->" % (' ' * indent, element)) elif isinstance(element, NavigableString): rv.append("|%s\"%s\"" % (' ' * indent, element)) else: if element.namespace: name = "%s %s" % (prefixes[element.namespace], element.name) else: name = element.name rv.append("|%s<%s>" % (' ' * indent, name)) if element.attrs: attributes = [] for name, value in list(element.attrs.items()): if isinstance(name, NamespacedAttribute): name = "%s %s" % (prefixes[name.namespace], name.name) if isinstance(value, list): value = " ".join(value) attributes.append((name, value)) for name, value in sorted(attributes): rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) indent += 2 for child in element.children: serializeElement(child, indent) serializeElement(element, 0) return "\n".join(rv) class AttrList(object): def __init__(self, element): self.element = element self.attrs = dict(self.element.attrs) def __iter__(self): return list(self.attrs.items()).__iter__() def __setitem__(self, name, value): # If this attribute is a multi-valued attribute for this element, # turn its value into a list. list_attr = self.element.cdata_list_attributes or {} if (name in list_attr.get('*', []) or (self.element.name in list_attr and name in list_attr.get(self.element.name, []))): # A node that is being cloned may have already undergone # this procedure. if not isinstance(value, list): value = nonwhitespace_re.findall(value) self.element[name] = value def items(self): return list(self.attrs.items()) def keys(self): return list(self.attrs.keys()) def __len__(self): return len(self.attrs) def __getitem__(self, name): return self.attrs[name] def __contains__(self, name): return name in list(self.attrs.keys()) class Element(treebuilder_base.Node): def __init__(self, element, soup, namespace): treebuilder_base.Node.__init__(self, element.name) self.element = element self.soup = soup self.namespace = namespace def appendChild(self, node): string_child = child = None if isinstance(node, str): # Some other piece of code decided to pass in a string # instead of creating a TextElement object to contain the # string. string_child = child = node elif isinstance(node, Tag): # Some other piece of code decided to pass in a Tag # instead of creating an Element object to contain the # Tag. child = node elif node.element.__class__ == NavigableString: string_child = child = node.element node.parent = self else: child = node.element node.parent = self if not isinstance(child, str) and child.parent is not None: node.element.extract() if (string_child is not None and self.element.contents and self.element.contents[-1].__class__ == NavigableString): # We are appending a string onto another string. # TODO This has O(n^2) performance, for input like # "a</a>a</a>a</a>..." old_element = self.element.contents[-1] new_element = self.soup.new_string(old_element + string_child) old_element.replace_with(new_element) self.soup._most_recent_element = new_element else: if isinstance(node, str): # Create a brand new NavigableString from this string. child = self.soup.new_string(node) # Tell Beautiful Soup to act as if it parsed this element # immediately after the parent's last descendant. (Or # immediately after the parent, if it has no children.) if self.element.contents: most_recent_element = self.element._last_descendant(False) elif self.element.next_element is not None: # Something from further ahead in the parse tree is # being inserted into this earlier element. This is # very annoying because it means an expensive search # for the last element in the tree. most_recent_element = self.soup._last_descendant() else: most_recent_element = self.element self.soup.object_was_parsed( child, parent=self.element, most_recent_element=most_recent_element) def getAttributes(self): if isinstance(self.element, Comment): return {} return AttrList(self.element) def setAttributes(self, attributes): if attributes is not None and len(attributes) > 0: converted_attributes = [] for name, value in list(attributes.items()): if isinstance(name, tuple): new_name = NamespacedAttribute(*name) del attributes[name] attributes[new_name] = value self.soup.builder._replace_cdata_list_attribute_values( self.name, attributes) for name, value in list(attributes.items()): self.element[name] = value # The attributes may contain variables that need substitution. # Call set_up_substitutions manually. # # The Tag constructor called this method when the Tag was created, # but we just set/changed the attributes, so call it again. self.soup.builder.set_up_substitutions(self.element) attributes = property(getAttributes, setAttributes) def insertText(self, data, insertBefore=None): text = TextNode(self.soup.new_string(data), self.soup) if insertBefore: self.insertBefore(text, insertBefore) else: self.appendChild(text) def insertBefore(self, node, refNode): index = self.element.index(refNode.element) if (node.element.__class__ == NavigableString and self.element.contents and self.element.contents[index-1].__class__ == NavigableString): # (See comments in appendChild) old_node = self.element.contents[index-1] new_str = self.soup.new_string(old_node + node.element) old_node.replace_with(new_str) else: self.element.insert(index, node.element) node.parent = self def removeChild(self, node): node.element.extract() def reparentChildren(self, new_parent): """Move all of this tag's children into another tag.""" # print("MOVE", self.element.contents) # print("FROM", self.element) # print("TO", new_parent.element) element = self.element new_parent_element = new_parent.element # Determine what this tag's next_element will be once all the children # are removed. final_next_element = element.next_sibling new_parents_last_descendant = new_parent_element._last_descendant(False, False) if len(new_parent_element.contents) > 0: # The new parent already contains children. We will be # appending this tag's children to the end. new_parents_last_child = new_parent_element.contents[-1] new_parents_last_descendant_next_element = new_parents_last_descendant.next_element else: # The new parent contains no children. new_parents_last_child = None new_parents_last_descendant_next_element = new_parent_element.next_element to_append = element.contents if len(to_append) > 0: # Set the first child's previous_element and previous_sibling # to elements within the new parent first_child = to_append[0] if new_parents_last_descendant is not None: first_child.previous_element = new_parents_last_descendant else: first_child.previous_element = new_parent_element first_child.previous_sibling = new_parents_last_child if new_parents_last_descendant is not None: new_parents_last_descendant.next_element = first_child else: new_parent_element.next_element = first_child if new_parents_last_child is not None: new_parents_last_child.next_sibling = first_child # Find the very last element being moved. It is now the # parent's last descendant. It has no .next_sibling and # its .next_element is whatever the previous last # descendant had. last_childs_last_descendant = to_append[-1]._last_descendant(False, True) last_childs_last_descendant.next_element = new_parents_last_descendant_next_element if new_parents_last_descendant_next_element is not None: # TODO: This code has no test coverage and I'm not sure # how to get html5lib to go through this path, but it's # just the other side of the previous line. new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant last_childs_last_descendant.next_sibling = None for child in to_append: child.parent = new_parent_element new_parent_element.contents.append(child) # Now that this element has no children, change its .next_element. element.contents = [] element.next_element = final_next_element # print("DONE WITH MOVE") # print("FROM", self.element) # print("TO", new_parent_element) def cloneNode(self): tag = self.soup.new_tag(self.element.name, self.namespace) node = Element(tag, self.soup, self.namespace) for key,value in self.attributes: node.attributes[key] = value return node def hasContent(self): return self.element.contents def getNameTuple(self): if self.namespace == None: return namespaces["html"], self.name else: return self.namespace, self.name nameTuple = property(getNameTuple) class TextNode(Element): def __init__(self, element, soup): treebuilder_base.Node.__init__(self, None) self.element = element self.soup = soup def cloneNode(self): raise NotImplementedError _lxml.py 0000644 00000035144 15030265652 0006244 0 ustar 00 # Use of this source code is governed by the MIT license. __license__ = "MIT" __all__ = [ 'LXMLTreeBuilderForXML', 'LXMLTreeBuilder', ] try: from collections.abc import Callable # Python 3.6 except ImportError as e: from collections import Callable from io import BytesIO from io import StringIO from lxml import etree from bs4.element import ( Comment, Doctype, NamespacedAttribute, ProcessingInstruction, XMLProcessingInstruction, ) from bs4.builder import ( DetectsXMLParsedAsHTML, FAST, HTML, HTMLTreeBuilder, PERMISSIVE, ParserRejectedMarkup, TreeBuilder, XML) from bs4.dammit import EncodingDetector LXML = 'lxml' def _invert(d): "Invert a dictionary." return dict((v,k) for k, v in list(d.items())) class LXMLTreeBuilderForXML(TreeBuilder): DEFAULT_PARSER_CLASS = etree.XMLParser is_xml = True processing_instruction_class = XMLProcessingInstruction NAME = "lxml-xml" ALTERNATE_NAMES = ["xml"] # Well, it's permissive by XML parser standards. features = [NAME, LXML, XML, FAST, PERMISSIVE] CHUNK_SIZE = 512 # This namespace mapping is specified in the XML Namespace # standard. DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace') DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS) # NOTE: If we parsed Element objects and looked at .sourceline, # we'd be able to see the line numbers from the original document. # But instead we build an XMLParser or HTMLParser object to serve # as the target of parse messages, and those messages don't include # line numbers. # See: https://bugs.launchpad.net/lxml/+bug/1846906 def initialize_soup(self, soup): """Let the BeautifulSoup object know about the standard namespace mapping. :param soup: A `BeautifulSoup`. """ super(LXMLTreeBuilderForXML, self).initialize_soup(soup) self._register_namespaces(self.DEFAULT_NSMAPS) def _register_namespaces(self, mapping): """Let the BeautifulSoup object know about namespaces encountered while parsing the document. This might be useful later on when creating CSS selectors. This will track (almost) all namespaces, even ones that were only in scope for part of the document. If two namespaces have the same prefix, only the first one encountered will be tracked. Un-prefixed namespaces are not tracked. :param mapping: A dictionary mapping namespace prefixes to URIs. """ for key, value in list(mapping.items()): # This is 'if key' and not 'if key is not None' because we # don't track un-prefixed namespaces. Soupselect will # treat an un-prefixed namespace as the default, which # causes confusion in some cases. if key and key not in self.soup._namespaces: # Let the BeautifulSoup object know about a new namespace. # If there are multiple namespaces defined with the same # prefix, the first one in the document takes precedence. self.soup._namespaces[key] = value def default_parser(self, encoding): """Find the default parser for the given encoding. :param encoding: A string. :return: Either a parser object or a class, which will be instantiated with default arguments. """ if self._default_parser is not None: return self._default_parser return etree.XMLParser( target=self, strip_cdata=False, recover=True, encoding=encoding) def parser_for(self, encoding): """Instantiate an appropriate parser for the given encoding. :param encoding: A string. :return: A parser object such as an `etree.XMLParser`. """ # Use the default parser. parser = self.default_parser(encoding) if isinstance(parser, Callable): # Instantiate the parser with default arguments parser = parser( target=self, strip_cdata=False, recover=True, encoding=encoding ) return parser def __init__(self, parser=None, empty_element_tags=None, **kwargs): # TODO: Issue a warning if parser is present but not a # callable, since that means there's no way to create new # parsers for different encodings. self._default_parser = parser if empty_element_tags is not None: self.empty_element_tags = set(empty_element_tags) self.soup = None self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED] self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)] super(LXMLTreeBuilderForXML, self).__init__(**kwargs) def _getNsTag(self, tag): # Split the namespace URL out of a fully-qualified lxml tag # name. Copied from lxml's src/lxml/sax.py. if tag[0] == '{': return tuple(tag[1:].split('}', 1)) else: return (None, tag) def prepare_markup(self, markup, user_specified_encoding=None, exclude_encodings=None, document_declared_encoding=None): """Run any preliminary steps necessary to make incoming markup acceptable to the parser. lxml really wants to get a bytestring and convert it to Unicode itself. So instead of using UnicodeDammit to convert the bytestring to Unicode using different encodings, this implementation uses EncodingDetector to iterate over the encodings, and tell lxml to try to parse the document as each one in turn. :param markup: Some markup -- hopefully a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. """ is_html = not self.is_xml if is_html: self.processing_instruction_class = ProcessingInstruction # We're in HTML mode, so if we're given XML, that's worth # noting. DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml( markup, stacklevel=3 ) else: self.processing_instruction_class = XMLProcessingInstruction if isinstance(markup, str): # We were given Unicode. Maybe lxml can parse Unicode on # this system? # TODO: This is a workaround for # https://bugs.launchpad.net/lxml/+bug/1948551. # We can remove it once the upstream issue is fixed. if len(markup) > 0 and markup[0] == u'\N{BYTE ORDER MARK}': markup = markup[1:] yield markup, None, document_declared_encoding, False if isinstance(markup, str): # No, apparently not. Convert the Unicode to UTF-8 and # tell lxml to parse it as UTF-8. yield (markup.encode("utf8"), "utf8", document_declared_encoding, False) # This was provided by the end-user; treat it as a known # definite encoding per the algorithm laid out in the HTML5 # spec. (See the EncodingDetector class for details.) known_definite_encodings = [user_specified_encoding] # This was found in the document; treat it as a slightly lower-priority # user encoding. user_encodings = [document_declared_encoding] detector = EncodingDetector( markup, known_definite_encodings=known_definite_encodings, user_encodings=user_encodings, is_html=is_html, exclude_encodings=exclude_encodings ) for encoding in detector.encodings: yield (detector.markup, encoding, document_declared_encoding, False) def feed(self, markup): if isinstance(markup, bytes): markup = BytesIO(markup) elif isinstance(markup, str): markup = StringIO(markup) # Call feed() at least once, even if the markup is empty, # or the parser won't be initialized. data = markup.read(self.CHUNK_SIZE) try: self.parser = self.parser_for(self.soup.original_encoding) self.parser.feed(data) while len(data) != 0: # Now call feed() on the rest of the data, chunk by chunk. data = markup.read(self.CHUNK_SIZE) if len(data) != 0: self.parser.feed(data) self.parser.close() except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(e) def close(self): self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED] def start(self, name, attrs, nsmap={}): # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy. attrs = dict(attrs) nsprefix = None # Invert each namespace map as it comes in. if len(nsmap) == 0 and len(self.nsmaps) > 1: # There are no new namespaces for this tag, but # non-default namespaces are in play, so we need a # separate tag stack to know when they end. self.nsmaps.append(None) elif len(nsmap) > 0: # A new namespace mapping has come into play. # First, Let the BeautifulSoup object know about it. self._register_namespaces(nsmap) # Then, add it to our running list of inverted namespace # mappings. self.nsmaps.append(_invert(nsmap)) # The currently active namespace prefixes have # changed. Calculate the new mapping so it can be stored # with all Tag objects created while these prefixes are in # scope. current_mapping = dict(self.active_namespace_prefixes[-1]) current_mapping.update(nsmap) # We should not track un-prefixed namespaces as we can only hold one # and it will be recognized as the default namespace by soupsieve, # which may be confusing in some situations. if '' in current_mapping: del current_mapping[''] self.active_namespace_prefixes.append(current_mapping) # Also treat the namespace mapping as a set of attributes on the # tag, so we can recreate it later. attrs = attrs.copy() for prefix, namespace in list(nsmap.items()): attribute = NamespacedAttribute( "xmlns", prefix, "http://www.w3.org/2000/xmlns/") attrs[attribute] = namespace # Namespaces are in play. Find any attributes that came in # from lxml with namespaces attached to their names, and # turn then into NamespacedAttribute objects. new_attrs = {} for attr, value in list(attrs.items()): namespace, attr = self._getNsTag(attr) if namespace is None: new_attrs[attr] = value else: nsprefix = self._prefix_for_namespace(namespace) attr = NamespacedAttribute(nsprefix, attr, namespace) new_attrs[attr] = value attrs = new_attrs namespace, name = self._getNsTag(name) nsprefix = self._prefix_for_namespace(namespace) self.soup.handle_starttag( name, namespace, nsprefix, attrs, namespaces=self.active_namespace_prefixes[-1] ) def _prefix_for_namespace(self, namespace): """Find the currently active prefix for the given namespace.""" if namespace is None: return None for inverted_nsmap in reversed(self.nsmaps): if inverted_nsmap is not None and namespace in inverted_nsmap: return inverted_nsmap[namespace] return None def end(self, name): self.soup.endData() completed_tag = self.soup.tagStack[-1] namespace, name = self._getNsTag(name) nsprefix = None if namespace is not None: for inverted_nsmap in reversed(self.nsmaps): if inverted_nsmap is not None and namespace in inverted_nsmap: nsprefix = inverted_nsmap[namespace] break self.soup.handle_endtag(name, nsprefix) if len(self.nsmaps) > 1: # This tag, or one of its parents, introduced a namespace # mapping, so pop it off the stack. out_of_scope_nsmap = self.nsmaps.pop() if out_of_scope_nsmap is not None: # This tag introduced a namespace mapping which is no # longer in scope. Recalculate the currently active # namespace prefixes. self.active_namespace_prefixes.pop() def pi(self, target, data): self.soup.endData() data = target + ' ' + data self.soup.handle_data(data) self.soup.endData(self.processing_instruction_class) def data(self, content): self.soup.handle_data(content) def doctype(self, name, pubid, system): self.soup.endData() doctype = Doctype.for_name_and_ids(name, pubid, system) self.soup.object_was_parsed(doctype) def comment(self, content): "Handle comments as Comment objects." self.soup.endData() self.soup.handle_data(content) self.soup.endData(Comment) def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML): NAME = LXML ALTERNATE_NAMES = ["lxml-html"] features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE] is_xml = False processing_instruction_class = ProcessingInstruction def default_parser(self, encoding): return etree.HTMLParser def feed(self, markup): encoding = self.soup.original_encoding try: self.parser = self.parser_for(encoding) self.parser.feed(markup) self.parser.close() except (UnicodeDecodeError, LookupError, etree.ParserError) as e: raise ParserRejectedMarkup(e) def test_fragment_to_document(self, fragment): """See `TreeBuilder`.""" return '<html><body>%s</body></html>' % fragment __init__.py 0000644 00000057766 15030265652 0006706 0 ustar 00 # Use of this source code is governed by the MIT license. __license__ = "MIT" from collections import defaultdict import itertools import re import warnings import sys from bs4.element import ( CharsetMetaAttributeValue, ContentMetaAttributeValue, RubyParenthesisString, RubyTextString, Stylesheet, Script, TemplateString, nonwhitespace_re ) __all__ = [ 'HTMLTreeBuilder', 'SAXTreeBuilder', 'TreeBuilder', 'TreeBuilderRegistry', ] # Some useful features for a TreeBuilder to have. FAST = 'fast' PERMISSIVE = 'permissive' STRICT = 'strict' XML = 'xml' HTML = 'html' HTML_5 = 'html5' class XMLParsedAsHTMLWarning(UserWarning): """The warning issued when an HTML parser is used to parse XML that is not XHTML. """ MESSAGE = """It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor.""" class TreeBuilderRegistry(object): """A way of looking up TreeBuilder subclasses by their name or by desired features. """ def __init__(self): self.builders_for_feature = defaultdict(list) self.builders = [] def register(self, treebuilder_class): """Register a treebuilder based on its advertised features. :param treebuilder_class: A subclass of Treebuilder. its .features attribute should list its features. """ for feature in treebuilder_class.features: self.builders_for_feature[feature].insert(0, treebuilder_class) self.builders.insert(0, treebuilder_class) def lookup(self, *features): """Look up a TreeBuilder subclass with the desired features. :param features: A list of features to look for. If none are provided, the most recently registered TreeBuilder subclass will be used. :return: A TreeBuilder subclass, or None if there's no registered subclass with all the requested features. """ if len(self.builders) == 0: # There are no builders at all. return None if len(features) == 0: # They didn't ask for any features. Give them the most # recently registered builder. return self.builders[0] # Go down the list of features in order, and eliminate any builders # that don't match every feature. features = list(features) features.reverse() candidates = None candidate_set = None while len(features) > 0: feature = features.pop() we_have_the_feature = self.builders_for_feature.get(feature, []) if len(we_have_the_feature) > 0: if candidates is None: candidates = we_have_the_feature candidate_set = set(candidates) else: # Eliminate any candidates that don't have this feature. candidate_set = candidate_set.intersection( set(we_have_the_feature)) # The only valid candidates are the ones in candidate_set. # Go through the original list of candidates and pick the first one # that's in candidate_set. if candidate_set is None: return None for candidate in candidates: if candidate in candidate_set: return candidate return None # The BeautifulSoup class will take feature lists from developers and use them # to look up builders in this registry. builder_registry = TreeBuilderRegistry() class TreeBuilder(object): """Turn a textual document into a Beautiful Soup object tree.""" NAME = "[Unknown tree builder]" ALTERNATE_NAMES = [] features = [] is_xml = False picklable = False empty_element_tags = None # A tag will be considered an empty-element # tag when and only when it has no contents. # A value for these tag/attribute combinations is a space- or # comma-separated list of CDATA, rather than a single CDATA. DEFAULT_CDATA_LIST_ATTRIBUTES = defaultdict(list) # Whitespace should be preserved inside these tags. DEFAULT_PRESERVE_WHITESPACE_TAGS = set() # The textual contents of tags with these names should be # instantiated with some class other than NavigableString. DEFAULT_STRING_CONTAINERS = {} USE_DEFAULT = object() # Most parsers don't keep track of line numbers. TRACKS_LINE_NUMBERS = False def __init__(self, multi_valued_attributes=USE_DEFAULT, preserve_whitespace_tags=USE_DEFAULT, store_line_numbers=USE_DEFAULT, string_containers=USE_DEFAULT, ): """Constructor. :param multi_valued_attributes: If this is set to None, the TreeBuilder will not turn any values for attributes like 'class' into lists. Setting this to a dictionary will customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES for an example. Internally, these are called "CDATA list attributes", but that probably doesn't make sense to an end-user, so the argument name is `multi_valued_attributes`. :param preserve_whitespace_tags: A list of tags to treat the way <pre> tags are treated in HTML. Tags in this list are immune from pretty-printing; their contents will always be output as-is. :param string_containers: A dictionary mapping tag names to the classes that should be instantiated to contain the textual contents of those tags. The default is to use NavigableString for every tag, no matter what the name. You can override the default by changing DEFAULT_STRING_CONTAINERS. :param store_line_numbers: If the parser keeps track of the line numbers and positions of the original markup, that information will, by default, be stored in each corresponding `Tag` object. You can turn this off by passing store_line_numbers=False. If the parser you're using doesn't keep track of this information, then setting store_line_numbers=True will do nothing. """ self.soup = None if multi_valued_attributes is self.USE_DEFAULT: multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES self.cdata_list_attributes = multi_valued_attributes if preserve_whitespace_tags is self.USE_DEFAULT: preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS self.preserve_whitespace_tags = preserve_whitespace_tags if store_line_numbers == self.USE_DEFAULT: store_line_numbers = self.TRACKS_LINE_NUMBERS self.store_line_numbers = store_line_numbers if string_containers == self.USE_DEFAULT: string_containers = self.DEFAULT_STRING_CONTAINERS self.string_containers = string_containers def initialize_soup(self, soup): """The BeautifulSoup object has been initialized and is now being associated with the TreeBuilder. :param soup: A BeautifulSoup object. """ self.soup = soup def reset(self): """Do any work necessary to reset the underlying parser for a new document. By default, this does nothing. """ pass def can_be_empty_element(self, tag_name): """Might a tag with this name be an empty-element tag? The final markup may or may not actually present this tag as self-closing. For instance: an HTMLBuilder does not consider a <p> tag to be an empty-element tag (it's not in HTMLBuilder.empty_element_tags). This means an empty <p> tag will be presented as "<p></p>", not "<p/>" or "<p>". The default implementation has no opinion about which tags are empty-element tags, so a tag will be presented as an empty-element tag if and only if it has no children. "<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will be left alone. :param tag_name: The name of a markup tag. """ if self.empty_element_tags is None: return True return tag_name in self.empty_element_tags def feed(self, markup): """Run some incoming markup through some parsing process, populating the `BeautifulSoup` object in self.soup. This method is not implemented in TreeBuilder; it must be implemented in subclasses. :return: None. """ raise NotImplementedError() def prepare_markup(self, markup, user_specified_encoding=None, document_declared_encoding=None, exclude_encodings=None): """Run any preliminary steps necessary to make incoming markup acceptable to the parser. :param markup: Some markup -- probably a bytestring. :param user_specified_encoding: The user asked to try this encoding. :param document_declared_encoding: The markup itself claims to be in this encoding. NOTE: This argument is not used by the calling code and can probably be removed. :param exclude_encodings: The user asked _not_ to try any of these encodings. :yield: A series of 4-tuples: (markup, encoding, declared encoding, has undergone character replacement) Each 4-tuple represents a strategy for converting the document to Unicode and parsing it. Each strategy will be tried in turn. By default, the only strategy is to parse the markup as-is. See `LXMLTreeBuilderForXML` and `HTMLParserTreeBuilder` for implementations that take into account the quirks of particular parsers. """ yield markup, None, None, False def test_fragment_to_document(self, fragment): """Wrap an HTML fragment to make it look like a document. Different parsers do this differently. For instance, lxml introduces an empty <head> tag, and html5lib doesn't. Abstracting this away lets us write simple tests which run HTML fragments through the parser and compare the results against other HTML fragments. This method should not be used outside of tests. :param fragment: A string -- fragment of HTML. :return: A string -- a full HTML document. """ return fragment def set_up_substitutions(self, tag): """Set up any substitutions that will need to be performed on a `Tag` when it's output as a string. By default, this does nothing. See `HTMLTreeBuilder` for a case where this is used. :param tag: A `Tag` :return: Whether or not a substitution was performed. """ return False def _replace_cdata_list_attribute_values(self, tag_name, attrs): """When an attribute value is associated with a tag that can have multiple values for that attribute, convert the string value to a list of strings. Basically, replaces class="foo bar" with class=["foo", "bar"] NOTE: This method modifies its input in place. :param tag_name: The name of a tag. :param attrs: A dictionary containing the tag's attributes. Any appropriate attribute values will be modified in place. """ if not attrs: return attrs if self.cdata_list_attributes: universal = self.cdata_list_attributes.get('*', []) tag_specific = self.cdata_list_attributes.get( tag_name.lower(), None) for attr in list(attrs.keys()): if attr in universal or (tag_specific and attr in tag_specific): # We have a "class"-type attribute whose string # value is a whitespace-separated list of # values. Split it into a list. value = attrs[attr] if isinstance(value, str): values = nonwhitespace_re.findall(value) else: # html5lib sometimes calls setAttributes twice # for the same tag when rearranging the parse # tree. On the second call the attribute value # here is already a list. If this happens, # leave the value alone rather than trying to # split it again. values = value attrs[attr] = values return attrs class SAXTreeBuilder(TreeBuilder): """A Beautiful Soup treebuilder that listens for SAX events. This is not currently used for anything, but it demonstrates how a simple TreeBuilder would work. """ def feed(self, markup): raise NotImplementedError() def close(self): pass def startElement(self, name, attrs): attrs = dict((key[1], value) for key, value in list(attrs.items())) #print("Start %s, %r" % (name, attrs)) self.soup.handle_starttag(name, attrs) def endElement(self, name): #print("End %s" % name) self.soup.handle_endtag(name) def startElementNS(self, nsTuple, nodeName, attrs): # Throw away (ns, nodeName) for now. self.startElement(nodeName, attrs) def endElementNS(self, nsTuple, nodeName): # Throw away (ns, nodeName) for now. self.endElement(nodeName) #handler.endElementNS((ns, node.nodeName), node.nodeName) def startPrefixMapping(self, prefix, nodeValue): # Ignore the prefix for now. pass def endPrefixMapping(self, prefix): # Ignore the prefix for now. # handler.endPrefixMapping(prefix) pass def characters(self, content): self.soup.handle_data(content) def startDocument(self): pass def endDocument(self): pass class HTMLTreeBuilder(TreeBuilder): """This TreeBuilder knows facts about HTML. Such as which tags are empty-element tags. """ empty_element_tags = set([ # These are from HTML5. 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr', # These are from earlier versions of HTML and are removed in HTML5. 'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer' ]) # The HTML standard defines these as block-level elements. Beautiful # Soup does not treat these elements differently from other elements, # but it may do so eventually, and this information is available if # you need to use it. block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"]) # These HTML tags need special treatment so they can be # represented by a string class other than NavigableString. # # For some of these tags, it's because the HTML standard defines # an unusual content model for them. I made this list by going # through the HTML spec # (https://html.spec.whatwg.org/#metadata-content) and looking for # "metadata content" elements that can contain strings. # # The Ruby tags (<rt> and <rp>) are here despite being normal # "phrasing content" tags, because the content they contain is # qualitatively different from other text in the document, and it # can be useful to be able to distinguish it. # # TODO: Arguably <noscript> could go here but it seems # qualitatively different from the other tags. DEFAULT_STRING_CONTAINERS = { 'rt' : RubyTextString, 'rp' : RubyParenthesisString, 'style': Stylesheet, 'script': Script, 'template': TemplateString, } # The HTML standard defines these attributes as containing a # space-separated list of values, not a single value. That is, # class="foo bar" means that the 'class' attribute has two values, # 'foo' and 'bar', not the single value 'foo bar'. When we # encounter one of these attributes, we will parse its value into # a list of values if possible. Upon output, the list will be # converted back into a string. DEFAULT_CDATA_LIST_ATTRIBUTES = { "*" : ['class', 'accesskey', 'dropzone'], "a" : ['rel', 'rev'], "link" : ['rel', 'rev'], "td" : ["headers"], "th" : ["headers"], "td" : ["headers"], "form" : ["accept-charset"], "object" : ["archive"], # These are HTML5 specific, as are *.accesskey and *.dropzone above. "area" : ["rel"], "icon" : ["sizes"], "iframe" : ["sandbox"], "output" : ["for"], } DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) def set_up_substitutions(self, tag): """Replace the declared encoding in a <meta> tag with a placeholder, to be substituted when the tag is output to a string. An HTML document may come in to Beautiful Soup as one encoding, but exit in a different encoding, and the <meta> tag needs to be changed to reflect this. :param tag: A `Tag` :return: Whether or not a substitution was performed. """ # We are only interested in <meta> tags if tag.name != 'meta': return False http_equiv = tag.get('http-equiv') content = tag.get('content') charset = tag.get('charset') # We are interested in <meta> tags that say what encoding the # document was originally in. This means HTML 5-style <meta> # tags that provide the "charset" attribute. It also means # HTML 4-style <meta> tags that provide the "content" # attribute and have "http-equiv" set to "content-type". # # In both cases we will replace the value of the appropriate # attribute with a standin object that can take on any # encoding. meta_encoding = None if charset is not None: # HTML 5 style: # <meta charset="utf8"> meta_encoding = charset tag['charset'] = CharsetMetaAttributeValue(charset) elif (content is not None and http_equiv is not None and http_equiv.lower() == 'content-type'): # HTML 4 style: # <meta http-equiv="content-type" content="text/html; charset=utf8"> tag['content'] = ContentMetaAttributeValue(content) return (meta_encoding is not None) class DetectsXMLParsedAsHTML(object): """A mixin class for any class (a TreeBuilder, or some class used by a TreeBuilder) that's in a position to detect whether an XML document is being incorrectly parsed as HTML, and issue an appropriate warning. This requires being able to observe an incoming processing instruction that might be an XML declaration, and also able to observe tags as they're opened. If you can't do that for a given TreeBuilder, there's a less reliable implementation based on examining the raw markup. """ # Regular expression for seeing if markup has an <html> tag. LOOKS_LIKE_HTML = re.compile("<[^ +]html", re.I) LOOKS_LIKE_HTML_B = re.compile(b"<[^ +]html", re.I) XML_PREFIX = '<?xml' XML_PREFIX_B = b'<?xml' @classmethod def warn_if_markup_looks_like_xml(cls, markup, stacklevel=3): """Perform a check on some markup to see if it looks like XML that's not XHTML. If so, issue a warning. This is much less reliable than doing the check while parsing, but some of the tree builders can't do that. :param stacklevel: The stacklevel of the code calling this function. :return: True if the markup looks like non-XHTML XML, False otherwise. """ if isinstance(markup, bytes): prefix = cls.XML_PREFIX_B looks_like_html = cls.LOOKS_LIKE_HTML_B else: prefix = cls.XML_PREFIX looks_like_html = cls.LOOKS_LIKE_HTML if (markup is not None and markup.startswith(prefix) and not looks_like_html.search(markup[:500]) ): cls._warn(stacklevel=stacklevel+2) return True return False @classmethod def _warn(cls, stacklevel=5): """Issue a warning about XML being parsed as HTML.""" warnings.warn( XMLParsedAsHTMLWarning.MESSAGE, XMLParsedAsHTMLWarning, stacklevel=stacklevel ) def _initialize_xml_detector(self): """Call this method before parsing a document.""" self._first_processing_instruction = None self._root_tag = None def _document_might_be_xml(self, processing_instruction): """Call this method when encountering an XML declaration, or a "processing instruction" that might be an XML declaration. """ if (self._first_processing_instruction is not None or self._root_tag is not None): # The document has already started. Don't bother checking # anymore. return self._first_processing_instruction = processing_instruction # We won't know until we encounter the first tag whether or # not this is actually a problem. def _root_tag_encountered(self, name): """Call this when you encounter the document's root tag. This is where we actually check whether an XML document is being incorrectly parsed as HTML, and issue the warning. """ if self._root_tag is not None: # This method was incorrectly called multiple times. Do # nothing. return self._root_tag = name if (name != 'html' and self._first_processing_instruction is not None and self._first_processing_instruction.lower().startswith('xml ')): # We encountered an XML declaration and then a tag other # than 'html'. This is a reliable indicator that a # non-XHTML document is being parsed as XML. self._warn() def register_treebuilders_from(module): """Copy TreeBuilders from the given module into this module.""" this_module = sys.modules[__name__] for name in module.__all__: obj = getattr(module, name) if issubclass(obj, TreeBuilder): setattr(this_module, name, obj) this_module.__all__.append(name) # Register the builder while we're at it. this_module.builder_registry.register(obj) class ParserRejectedMarkup(Exception): """An Exception to be raised when the underlying parser simply refuses to parse the given markup. """ def __init__(self, message_or_exception): """Explain why the parser rejected the given markup, either with a textual explanation or another exception. """ if isinstance(message_or_exception, Exception): e = message_or_exception message_or_exception = "%s: %s" % (e.__class__.__name__, str(e)) super(ParserRejectedMarkup, self).__init__(message_or_exception) # Builders are registered in reverse order of priority, so that custom # builder registrations will take precedence. In general, we want lxml # to take precedence over html5lib, because it's faster. And we only # want to use HTMLParser as a last resort. from . import _htmlparser register_treebuilders_from(_htmlparser) try: from . import _html5lib register_treebuilders_from(_html5lib) except ImportError: # They don't have html5lib installed. pass try: from . import _lxml register_treebuilders_from(_lxml) except ImportError: # They don't have lxml installed. pass
| ver. 1.4 |
Github
|
.
| PHP 8.2.28 | Generation time: 0.02 |
proxy
|
phpinfo
|
Settings