Current File : //opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyc
�
5}�ac @ s" d d l Z d d l Z d d l Z d d l m Z d d l m Z m Z d d l Z y d d l Z Wn e
k
r d Z n Xd d# d � � YZ d e e j
f d � � YZ d e e j
f d � � YZ d
e e j
f d � � YZ d e f d
� � YZ d e e j
f d � � YZ d e e j
f d � � YZ d e e j
f d � � YZ d e e j
f d � � YZ d e e j
f d � � YZ d e e j
f d � � YZ d e f d � � YZ e j e d � d e j
f d � � Y� Z d e j
f d � � YZ d! � Z e d" k r e � n d S($ i����N( t support( t BaseHTTPRequestHandlert
HTTPServert
BaseRobotTestc B sD e Z d Z d Z g Z g Z d � Z d � Z d � Z d � Z RS( t t test_robotparserc C s; t j | j � j � } t j � | _ | j j | � d S( N( t StringIOt
robots_txtt readlinest robotparsert RobotFileParsert parsert parse( t selft lines( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt setUp s c C s2 t | t � r% | \ } } | | f S| j | f S( N( t
isinstancet tuplet agent( R
t urlR ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt get_agent_and_url s
c C sI xB | j D]7 } | j | � \ } } | j | j j | | � � q
Wd S( N( t goodR t
assertTrueR t can_fetch( R
R R ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt test_good_urls s c C sI xB | j D]7 } | j | � \ } } | j | j j | | � � q
Wd S( N( t badR t assertFalseR R ( R
R R ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt
test_bad_urls# s (
t __name__t
__module__R R R R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR
s t UserAgentWildcardTestc B s) e Z d Z d d g Z d d d g Z RS( s� User-agent: *
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
Disallow: /tmp/ # these will soon disappear
Disallow: /foo.html
t /s
/test.htmls /cyberworld/map/index.htmls /tmp/xxxs /foo.html( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR ) s t RejectAllRobotsTestc B s# e Z d Z g Z d d d g Z RS( s( # go away
User-agent: *
Disallow: /
s /cyberworld/map/index.htmlR s /tmp/( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR 4 s t UserAgentOrderingTestc B s e Z d Z d Z d g Z RS( sM User-agent: Googlebot
Disallow: /
User-agent: Googlebot-Mobile
Allow: /
t Googlebots /something.jpg( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR! > s
t UserAgentGoogleMobileTestc B s e Z d Z RS( s Googlebot-Mobile( R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR# M s t GoogleURLOrderingTestc B s& e Z d Z d Z d g Z d g Z RS( sJ User-agent: Googlebot
Allow: /folder1/myfile.html
Disallow: /folder1/
t googlebots /folder1/myfile.htmls /folder1/anotherfile.html( R R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR$ Q s t DisallowQueryStringTestc B s e Z d Z d g Z d g Z RS( s2 User-agent: *
Disallow: /some/path?name=value
s
/some/paths /some/path?name=value( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR& ^ s t UseFirstUserAgentWildcardTestc B s e Z d Z d g Z d g Z RS( sN User-agent: *
Disallow: /some/path
User-agent: *
Disallow: /another/path
s
/another/paths
/some/path( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR' h s t EmptyQueryStringTestc B s e Z d Z d g Z d g Z RS( s> User-agent: *
Allow: /some/path?
Disallow: /another/path?
s /some/path?s /another/path?( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR( u s t DefaultEntryTestc B s# e Z d Z d d g Z d g Z RS( sO User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/
R s
/test.htmls /cyberworld/map/index.html( R R R R R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR) � s t StringFormattingTestc B s e Z d Z d Z d � Z RS( s� User-agent: *
Crawl-delay: 1
Request-rate: 3/15
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
# Cybermapper knows where to go.
User-agent: cybermapper
Disallow: /some/path
sX User-agent: cybermapper
Disallow: /some/path
User-agent: *
Disallow: /cyberworld/map/
c C s | j t | j � | j � d S( N( t assertEqualt strR t expected_output( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt test_string_formatting� s ( R R R R- R. ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR* � s
t RobotHandlerc B s e Z d � Z d � Z RS( c C s | j d d � d S( Ni� s Forbidden access( t
send_error( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt do_GET� s c G s d S( N( ( R
t formatt args( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt log_message� s ( R R R1 R4 ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR/ � s s threading required for this testt PasswordProtectedSiteTestCasec B s, e Z d � Z d � Z e j d � � Z RS( c C sf t t j d f t � | _ t j d d d | j j d i d d 6� | _ t | j _
| j j � d S( Ni t names HTTPServer servingt targett kwargsg{�G�z�?t
poll_interval( R R t HOSTR/ t servert threadingt Threadt
serve_forevert tt Truet daemont start( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR � s c C s+ | j j � | j j � | j j � d S( N( R; t shutdownR? t joint server_close( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt tearDown� s
c C su | j j } d t j d t | d � } | d } t j � } | j | � | j � | j | j
d | � � d S( Ns http://t :i s /robots.txtt *( R; t server_addressR R: R, R R
t set_urlt readR R ( R
t addrR t
robots_urlR ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt testPasswordProtectedSite� s
( R R R RF R t reap_threadsRN ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR5 � s
t NetworkTestCasec B sP e Z d Z d j e � Z e d � � Z d � Z d � Z d � Z d � Z
RS( s http://www.pythontest.net/s {}elsewhere/robots.txtc C sL t j d � t j | j � �' t j | j � | _ | j j � Wd QXd S( Nt network( R t requirest transient_internett base_urlR R
R R RK ( t cls( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt
setUpClass� s
c C s2 d j | j | t j j | � d s+ d n d � S( Ns {}{}{}i R R ( R2 RT t ost patht splitext( R
RX ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyR � s c C sC | j | j j � | j | j j � | j | j j � d � d S( Ni ( R R t disallow_allt allow_allt
assertGreatert mtime( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt
test_basic� s c C s� | j | j j d | j d � � � | j | j j d | j � � | j | j j d | j d � � � | j | j j d | j d � � � | j | j j d | j d � � � | j | j j d | j � � d S( NRH t elsewheret Nutcht briant webstats( R R R R R RT ( R
( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt test_can_fetch� s % %%%c C s\ t j | j d � � } | j � | j | j � | j | j � | j | j � d � d S( Ns i-robot.txti (
R R
R RK R R[ R RZ R+ R] ( R
R ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt
test_read_404� s
( R R RT R2 R t classmethodRV R R^ Rc Rd ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyRP � s c
C s2 t j t t t t t t t t t
t t t
� d S( N( R t run_unittestR R R! R# R$ R& R' R( R) R* R5 RP ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt test_main� s t __main__( ( RW R t unittestt testR t BaseHTTPServerR R R R< t ImportErrort NoneR t TestCaseR R R! R# R$ R& R' R( R) R* R/ t
skipUnlessR5 RP Rg R ( ( ( s> /opt/plesk/python/2.7/lib64/python2.7/test/test_robotparser.pyt <module> s6
&
Mini Shell