3
] @ s d Z ddlZddlZddlZddljZddlm Z ddl
m Z ddl m
Z
mZmZmZmZmZmZmZmZmZmZmZmZ dZG d d
d
ejj Z dd d
Z
dS )z>html2text: Turn HTML into equivalent Markdown-structured text. N)wrap)config)
dumb_css_parser
element_style escape_mdescape_md_sectiongoogle_fixed_width_fontgoogle_has_heightgoogle_list_stylegoogle_text_emphasishnlist_numbering_startpad_tables_in_textskipwrap unifiable_n c s e Zd Zddejf fdd Z fddZdd Zd d
Z fd d Z d
d Z
dd Z dd Z dd Z
dd Zdd Zdd Zdd Zd d Zd d Zd.d"d#Zd/d$d%Zd&d' Zd(d) Zd*d+ Zd,d- Z ZS )0 HTML2TextN c s
t jdd d| _d| _d| _tj| _tj| _ tj
| _ || _ tj
| _tj| _tj| _tj| _tj| _tj| _tj| _tj| _ tj | _ tj | _ tj!| _"tj#| _$d| _%d| _&d| _'d| _(tj)| _*tj+| _,d| _-tj.| _/tj0| _1tj2| _3tj4| _5tj6| _7d| _8tj9| _:tj;| _<|dkr | j=| _>n|| _>g | _?d| _@d| _Ad| _Bd| _Cd| _Dg | _Eg | _Fd| _Gd| _HtIjJd | _Kd| _Lg | _Md| _Nd| _Od| _Pd| _Qd| _Rd
| _Sd| _Td| _Ud| _Vi | _Wg | _Xd| _Yd| _Zd| _[d| _\d| _]i | _^|| __d| _`d| _ad| _bd| _cd tjdd < dS )
z
Input parameters:
out: possible custom replacement for self.outtextf (which
appends lines of text).
baseurl: base URL of the document we process
F)Zconvert_charrefsr *_z**NTz^[a-zA-Z+]+://r z _place_holder;nbsp)esuper__init__
split_next_tdtd_count table_startr Z UNICODE_SNOB unicode_snobZ ESCAPE_SNOB escape_snobZLINKS_EACH_PARAGRAPHlinks_each_paragraph
body_widthZSKIP_INTERNAL_LINKSskip_internal_linksZ INLINE_LINKS inline_linksZ
PROTECT_LINKS
protect_linksZGOOGLE_LIST_INDENTgoogle_list_indentZIGNORE_ANCHORS ignore_linksZ
IGNORE_IMAGES
ignore_imagesZIMAGES_AS_HTMLimages_as_htmlZ
IMAGES_TO_ALT
images_to_altZIMAGES_WITH_SIZEimages_with_sizeZIGNORE_EMPHASISignore_emphasisZ
BYPASS_TABLES
bypass_tablesZ
IGNORE_TABLES
ignore_tables
google_doc ul_item_mark
emphasis_mark strong_markZSINGLE_LINE_BREAKsingle_line_breakZUSE_AUTOMATIC_LINKSuse_automatic_linkshide_strikethroughZ MARK_CODE mark_codeZWRAP_LIST_ITEMSwrap_list_itemsZ
WRAP_LINKS
wrap_linksZ
PAD_TABLES
pad_tablesZDEFAULT_IMAGE_ALTdefault_image_alt tag_callbackZ
OPEN_QUOTE
open_quoteZ CLOSE_QUOTE close_quoteouttextfout outtextlistquietp_poutcountstartspaceaastackmaybe_automatic_link
empty_linkrecompileabsolute_url_matcheracountlist
blockquoteprestartprecodequote br_toggle lastWasNL lastWasListstyle style_def tag_stackemphasisdrop_white_spaceinheader
abbr_title abbr_data abbr_listbaseurlstressedpreceding_stressedpreceding_data current_tag UNIFIABLE)selfr> r_ bodywidth) __class__ /usr/lib/python3.6/__init__.pyr " s
zHTML2Text.__init__c s |j dd}t j| d S )Nzz )replacer feed)re data)rg rh ri rk | s zHTML2Text.feedc C s8 | j | | j d | j| j }| jr0t|S |S d S )Nr )rk optwrapcloser8 r )re rl Zmarkdownrh rh ri handle s
zHTML2Text.handlec C s" | j j| |r |d dk| _d S )N
)r? appendrT )re srh rh ri r= s zHTML2Text.outtextfc sZ t j | j | jddd dj| j}| jr@tjj d }nd}|j
d|}g | _|S )Nr end)forceznbsp; z _place_holder;) r rn pbrojoinr? r htmlentitieshtml5rj )re Zouttextr )rg rh ri rn s
zHTML2Text.closec C s | j | j|d d S )NT) handle_datacharref)re crh rh ri handle_charref s zHTML2Text.handle_charrefc C s | j |}|r| j|d d S )NT) entityrefr~ )re r refrh rh ri handle_entityref s
zHTML2Text.handle_entityrefc C s | j ||dd d S )NT)rC )
handle_tag)re tagattrsrh rh ri handle_starttag s zHTML2Text.handle_starttagc C s | j |d dd d S )NF)rC )r )re r rh rh ri
handle_endtag s zHTML2Text.handle_endtagc C s d|kr dS d}xpt | jD ]b\}}d|krv|d |d krvd|ksLd|krrd|krvd|krv|d |d krvd}nd}|r |S q W dS )z
:type attrs: dict
:returns: The index of certain set of attributes (of a link) in the
self.a list. If the set of attributes is not found, returns None
:rtype: int
hrefNFtitleT) enumeraterE )re r matchirE rh rh ri
previousIndex s zHTML2Text.previousIndexc C s t |}t |}d|ko | j}d}x$tjD ]}||ko<||k}|r*P q*W d|koVd|k} t|opt| op| j }
|r|s| s|
r| jd7 _|r| jd7 _| r| j| j | j
d7 _
|r| j| j | j
d7 _
|
r| jd | j
d7 _
d| _ n|s| s|
r,| jd8 _d| _
|
rZ| j
rJ| j
d8 _
n
| jd d| _ |r| j
rx| j
d8 _
n | j| j | r| j
r| j
d8 _
n | j| j |s| r| j r| jd |r| jd8 _dS ) z/
Handles various text emphases
z line-throughFitalicrp `Trw N)r r4 r ZBOLD_TEXT_STYLE_VALUESr rO rY r@ ry r0 rZ r1 rQ rD ) re rC tag_style parent_styleZ tag_emphasisZparent_emphasisZ
strikethroughZboldZ bold_markerr Zfixedrh rh ri handle_emphasis s^
zHTML2Text.handle_emphasisc C s || _ |d kri }nt|}| jd k r>| j| |||dkr>d S |rx| jd k rx|d^krx|dksb| jrx| jd d | _d | _| jri }|r| jr| jd_ d }t || j
|}| jj |||f n4| jr| jj nd i i f\}}}| jr| jd` d }t
|r0| j |r&d| _| jt
|d d
n
d | _d S |dakr| jrf|r\t|r\| j n| j n | jrz|dkrzn| j |dkr|r| jdkr| jd n
| jd |dkr|r| j | jd | j |dbkr |r| jd
7 _n| jd
8 _|dkr:|r,| jd
7 _n| jd
8 _|dckrJd| _|dkr|r| j | jddd d| _| jd
7 _n| jd
8 _| j dd }|ddkr| j r|r|| rd
| j }n| j}| j| |rd| _|dekr<| j r<|r || r d
| j } n| j} | j| |r|d8 |d1< |j1d9pN| j2}| j3st| j4rd:|kstd;|kr| jd<|d8 d= d:|kr| jd>|d: d= d;|kr| jd?|d; d= |r| jd@| d= | jdA d S | jd k r\| j}| j0rFt)||krF| j5j6|rF| jd2t)| d3 d | _d S | jd d | _d | _| j0rt| jt)| n| jdBt)| d7 | j(r|j1d1pd+}| jdCt)t7j8| j9| dD nh| j+|}|d k r| j,| }n.| j-d
7 _-| j-|d4< | j.|d5< | j,j | | jdt/|d4 d7 |dkrD|rD| j |dkr^| r^| j: |dEkrx|rx| jdF |dEkr| r| j: |dik r(| j; r| j< r| j |r| jrt=|}n|}t>|}| j;j ||dI n0| j; r | j;j | j r | j; r | jdJ d| _ z
Zhrz* * *headscriptbodyrN z> )rv c S s | j otjd| j d S )Nz[^\s]rp rr )rb rI r )re rh rh ri no_preceding_space| s z0HTML2Text.handle_tag..no_preceding_spaceemr ustrongbdelstrikert z ~~z~~kbdrQ ttr abbrr r qc S s@ t j| j|}|j r dj|nd}| jdjt||d d S )Nz "{}"r z]({url}{title}))urlr )urlparseurljoinr_ stripformatry r )re linkr r rh rh ri link_url s z&HTML2Text.handle_tag..link_urlrE r <>countrB z][]srcaltwidthZheightz
![z/>z![()Zddz olul)namenumrq liz r r z. tabletrtdthz<{}>
z
</{}>z<{}>z</{}>z</z| |z---rO z
[/code])r r rV r r rr rr )r r )r rV r )r )r r r )r r )r r rt )r rQ r )r )r r rr )r r r r )r r )r r )r r )Orc dictr: rG r](z)
)re r r rC r r Zdummyr rY r r r rE r r r r Z
list_styleZnumbering_startr
nest_countrh rh ri r s$
zHTML2Text.handle_tagc C s | j dkrd| _ dS )z Pretty print has a line breakr rp N)rA )re rh rh ri rx s
z
HTML2Text.pbrc C s | j r
dnd| _dS )z Set pretty print to 1 or 2 linesrp r N)r2 rA )re rh rh ri r s z HTML2Text.pc C s | j d| _dS )z Soft breaksz N)rx rS )re rh rh ri r s zHTML2Text.soft_brFc C sF | j dk r| j |7 _ | jsB| jrT|j }| jrF| jp>| j rF|}|dkrTd| _|r| j rtjdd|}|r|d dkrd| _ |dd }| r| rdS | j
r|j d r|j d rd| }| j r| j
d
d| _d | j }|o|o|d d k r| jr|d7 }| jrX| js&|d 7 }x tt| jD ]}|d 7 }q6W |jdd| }| j
rxd
| _
| jrx|jd}| jrd
| _ d| _d
| _|dkrd| _| j
d d
| _ | jr| j
| jd | | j d
| _ d| _| j r | js| j
d d
| _ | jr| jdkr | js&|dkr|dkr:| j
d g }x| jD ]x}| j|d kr| j
dt|d d tj | j |d d|kr| j
d|d d | j
d n
|j | qFW | j|kr| j
d || _| j r$|dkr$x2| j j D ]$\} }
| j
d| d |
d qW d| _| j
| | jd7 _dS )z6
Deal with indentation and whitespace
Nr r z\s+rw Trp rq z
z
[code]r z Fru r rB z [r z]: r r z (r z *[)!r] r@ r. lstriprZ rO rQ rI subrD rP r r5 r> rA rN rM ranger rj rC rS rT rE r rB r r r r_ rs r^ items) re rl puredatarv Zlstripped_dataZbqr Znewar r Z
definitionrh rh ri ry s
(
z HTML2Text.oc C s |sd S | j r$|j }d| _ d| _n:| jr^tjd|d rXt| j rX| jdkrXd| }d| _| jrt| jj t
| | j d k r| j }||kr| j j|r| j
r| jd | d
d| _d S | jd d | _ d| _| j r| j r| rt|| jd }|| _| j|dd
d S )NFTz[^\s.!?]r rE rQ rO rw r r r )Zsnob)r )rE rQ rO )r` r ra rI r r rc rV rW updater rG rK r3 ry rH rQ rO r r rb )re rl Z entity_charr rh rh ri r~ s:
zHTML2Text.handle_datac C sb |d dkr t |dd d}nt |}| j r@|tkr@t| S yt|S tk
r\ dS X d S )Nr xXrp r )r r )intr r chr
ValueError)re r r rh rh ri r = s zHTML2Text.charrefc C sd | j r |tjkr tj| S ytjj|d }W n tk
rL d| d S X |dkr`tj| S |S )N;&r )r r rd r{ r| r} r )re r Zchrh rh ri r K s
zHTML2Text.entityrefc C s* d}d|kr&t |d dd | j }|S )zq
Calculate the nesting count of google doc lists
:type style: dict
:rtype: int
r z margin-leftNr )r r% )re rV r rh rh ri r T s zHTML2Text.google_nest_countc C s | j s
|S d}d}| js d| _x|jdD ]}t|dkrt|| j| jsd}|jd| j rdd}n|jdrrd}t || j d|d}|dj
|7 }|j dr|d 7 }d
}q|r|d7 }d
}q|d 7 }d }n t j
j|s||d 7 }d
}q*|d k r*|d7 }|d
7 }q*W |S )
zi
Wrap all paragraphs in the provided text.
:type text: str
:rtype: str
r r Frq z z z> )Zbreak_long_wordsZsubsequent_indentz
rp z
r )r! r7 r# splitr r r6 r r/ r rz endswithr ZRE_SPACEr )re textresultnewlinesZparaindentwrappedrh rh ri rm b sF
zHTML2Text.optwrap)FF)F)__name__
__module__ __qualname__r
BODY_WIDTHr rk ro r= rn r r r r r r r rx r r ry r~ r r r rm
__classcell__rh rh )rg ri r ! s. Z G }
o
+ r r c C s$ |d krt j}t||d}|j| S )N)r_ rf )r r r ro )r{ r_ rf hrh rh ri html2text s r )r r r )r N) __doc__Z
html.entitiesr{ Z html.parserrI Z urllib.parseparser textwrapr r r Zhtml2text.utilsr r r r r r r
r r r
r r r __version__parserZ
HTMLParserr rh rh rh ri s"
<