Current File : //usr/lib/python3.6/site-packages/pyzor/__pycache__/digest.cpython-36.pyc
3
�2�^� � @ s� d Z ddlmZ ddlZddlZddlZddlZy ddlZW n ek
rZ ddl j
ZY nX ddgZ ej Z
ee
d�j� �Zejd dkr�ejd �ej�e_G d
d � d ej�ZG d d
� d
e�ZG dd� de�ZdS )z Handle digesting the messages.� )�print_functionN� � �< � � �utf8c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
S ) � HTMLStripperz Strip all tags from the HTML.c C s$ t j j| � | j� || _d| _d S )NT)�
HTMLParser�__init__�reset� collector�collect)�selfr
� r � /usr/lib/python3.6/digest.pyr s zHTMLStripper.__init__c C s" |j � }|r | jr | jj|� dS )zKeep track of the data.N)�stripr r
�append)r �datar r r � handle_data" s
zHTMLStripper.handle_datac C s& t j j| ||� |j� dkr"d| _d S )N�script�styleF)r r )r
�handle_starttag�lowerr )r �tagZattrsr r r r ( s z HTMLStripper.handle_starttagc C s$ t j j| |� |j� dkr d| _d S )Nr r T)r r )r
�
handle_endtagr r )r r r r r r - s zHTMLStripper.handle_endtagN)�__name__�
__module__� __qualname__�__doc__r r r r r r r r r s
r c @ s� e Zd ZdZddgZdZdZejd�Z ejdej
�Z ejd�Z ejd �Z
d
Zd d d
�Zdd� Zdd� Zdd� Zedd� �Zedd� �Zedd� �Zedd� �Zd S ) � DataDigesterzThe major workhouse class.�value�digest� � z\S+@\S+z
[a-z]+:\S+z\S{10,}z\s� Nc
C s� |d kr t }d | _t� | _g }xf| j|�D ]X}xR|j� D ]F}| j|�}| j|�r8y|j|j dd�� W q8 t
k
r| w8Y q8X q8W q*W t |�| j kr�| j
|� n | j||� | jj� | _t | j�tks�t�d S )Nr �ignore)� digest_specr! �HASHr" �digest_payloads�
splitlines� normalize�should_handle_liner �encode� UnicodeError�len�atomic_num_lines�
handle_atomic�
handle_pieced� hexdigest� HASH_SIZE�AssertionError)r �msg�spec�lines�payload�lineZnormr r r r P s$
zDataDigester.__init__c C s x|D ]}| j |� qW dS )zWe digest everything.N)� handle_line)r r8 r: r r r r1 j s
zDataDigester.handle_atomicc
C sh xb|D ]Z\}}xPt |�D ]D}y |t|t|� d �| }W n tk
rP Y qX | j|� qW qW dS )z#Digest stuff according to the spec.�d N)�range�intr/ �
IndexErrorr; )r r8 r7 �offsetZlength�ir: r r r r2 o s zDataDigester.handle_piecedc C s | j j|j� � d S )N)r" �update�rstrip)r r: r r r r; z s zDataDigester.handle_linec C sN |j dd�}| j}| jj||�}| jj||�}| jj||�}| jjd|�j� S )N� r% )�replace�unwanted_txt_repl� longstr_ptrn�sub�
email_ptrn�url_ptrn�ws_ptrnr )�cls�s�replr r r r+ } s zDataDigester.normalizec C s@ g }t |�}y|j| � W n ttjfk
r4 Y nX dj|�S )N� )r Zfeed�UnicodeDecodeErrorr
ZHTMLParseError�join)rM r Zstripperr r r �normalize_html_part� s z DataDigester.normalize_html_partc C s t |�o| jt |�kS )N)r/ �min_line_length)rL rM r r r r, � s z DataDigester.should_handle_linec c s� x�|j � D ]�}|j� dkr�|jdd�}|j� }|r>|jdd�}d}|sLd}n|j� jdd �dkrdd}y|j||�}W nB ttt fk
r� y|jdd�}W n tk
r� w
Y nX Y nX |j
� dkr�| j |�V q�|V q
|j � r�q
|j� V q
W d S )N�textT)�decoderD r% r& �ascii�_�-� quopri-codec�quopri�quoted-printable�quotedprintable�strictZhtml)rY rZ r[ r\ )
�walkZget_content_maintypeZ get_payloadZget_content_charsetrE r rU � LookupErrorr. r5 Zget_content_subtyperR Z is_multipart)rL r6 �partr9 �charset�errorsr r r r) � s2 z DataDigester.digest_payloads)N)r r r r � __slots__rS r0 �re�compilerI �
IGNORECASErJ rG rK rF r r1 r2 r; � classmethodr+ � staticmethodrR r, r) r r r r r 3 s"
r c s e Zd ZdZ� fdd�Z� ZS )�PrintingDataDigesterz6Extends DataDigester: prints out what we're digesting.c s" t |jd�� tt| �j|� d S )Nr )�printrU �superri r; )r r: )� __class__r r r; � s z PrintingDataDigester.handle_line)r r r r r; �
__classcell__r r )rl r ri � s ri )r r )r r )r Z
__future__r rd �sys�codecsZhashlibr
� ImportErrorZ html.parser�parserr' Zsha1r( r/ r3 r4 � version_info� getwriter�stdoutr �objectr ri r r r r �<module> s$
Mini Shell