robots.py

from urllib import robotparser robot_parser = robotparser.RobotFileParser() def prepare(robots_txt_url): robot_parser.set_url(robots_txt_url) robot_parser.read() def is_allowed(target_url, user_agent='*'): return robot_parser.can_fetch(user_agent, target_url) if __name__ == '__main__': prepare('http://www.apress.com/robots.txt') print(is_allowed('http://www.apress.com/covers/')) print(is_allowed('http://www.apress.com/gp/python'))
Print website robots.txt

Be the first to comment

You can use [html][/html], [css][/css], [php][/php] and more to embed the code. Urls are automatically hyperlinked. Line breaks and paragraphs are automatically generated.