from urllib import robotparser
robot_parser = robotparser.RobotFileParser()
def prepare(robots_txt_url):
robot_parser.set_url(robots_txt_url)
robot_parser.read()
def is_allowed(target_url, user_agent='*'):
return robot_parser.can_fetch(user_agent, target_url)
if __name__ == '__main__':
prepare('http://www.apress.com/robots.txt')
print(is_allowed('http://www.apress.com/covers/'))
print(is_allowed('http://www.apress.com/gp/python'))
Print website robots.txt
Be the first to comment
You can use [html][/html], [css][/css], [php][/php] and more to embed the code. Urls are automatically hyperlinked. Line breaks and paragraphs are automatically generated.