深圳全飞鸿

标题: python抓取网页资源的方法验证 [打印本页]
作者: zhgc    时间: 2019-7-15 14:14
标题: python抓取网页资源的方法验证
1、最简单

  1. import urllib.request
  2. response = urllib.request.urlopen('http://python.org/')
  3. html = response.read()
复制代码



2、使用 Request

  1. import urllib.request

  3. req = urllib.request.Request('http://python.org/')
  4. response = urllib.request.urlopen(req)
  5. the_page = response.read()
复制代码


3、发送数据

  2. #! /usr/bin/env python3

  4. import urllib.parse
  5. import urllib.request

  7. url = 'http://localhost/login.php'
  8. user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
  9. values = {
  10. 'act' : 'login',
  11. 'login[email]' : 'yzhang@i9i8.com',
  12. 'login[password]' : '123456'
  13.          }

  15. data = urllib.parse.urlencode(values)
  16. req = urllib.request.Request(url, data)
  17. req.add_header('Referer', 'http://www.python.org/')
  18. response = urllib.request.urlopen(req)
  19. the_page = response.read()

  21. print(the_page.decode("utf8"))
复制代码


4、发送数据和header

  1. #! /usr/bin/env python3

  3. import urllib.parse
  4. import urllib.request

  6. url = 'http://localhost/login.php'
  7. user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
  8. values = {
  9. 'act' : 'login',
  10. 'login[email]' : 'yzhang@i9i8.com',
  11. 'login[password]' : '123456'
  12.          }
  13. headers = { 'User-Agent' : user_agent }

  15. data = urllib.parse.urlencode(values)
  16. req = urllib.request.Request(url, data, headers)
  17. response = urllib.request.urlopen(req)
  18. the_page = response.read()

  20. print(the_page.decode("utf8"))
复制代码


5、http 错误

  1. #! /usr/bin/env python3

  3. import urllib.request

  5. req = urllib.request.Request('http://www.python.org/fish.html')
  6. try:
  7.     urllib.request.urlopen(req)
  8. except urllib.error.HTTPError as e:
  9. print(e.code)
  10. print(e.read().decode("utf8"))
复制代码


6、异常处理1

  1. #! /usr/bin/env python3

  3. from urllib.request import Request, urlopen
  4. from urllib.error import URLError, HTTPError
  5. req = Request("http://twitter.com/")
  6. try:
  7.     response = urlopen(req)
  8. except HTTPError as e:
  9. print('The server couldn\'t fulfill the request.')
  10. print('Error code: ', e.code)
  11. except URLError as e:
  12. print('We failed to reach a server.')
  13. print('Reason: ', e.reason)
  14. else:
  15. print("good!")
  16. print(response.read().decode("utf8"))
复制代码


7、异常处理2

  1. #! /usr/bin/env python3

  3. from urllib.request import Request, urlopen
  4. from urllib.error import  URLError
  5. req = Request("http://twitter.com/")
  6. try:
  7.     response = urlopen(req)
  8. except URLError as e:
  9. if hasattr(e, 'reason'):
  10. print('We failed to reach a server.')
  11. print('Reason: ', e.reason)
  12. elif hasattr(e, 'code'):
  13. print('The server couldn\'t fulfill the request.')
  14. print('Error code: ', e.code)
  15. else:
  16. print("good!")
  17. print(response.read().decode("utf8"))
复制代码


8、HTTP 认证

  1. #! /usr/bin/env python3

  3. import urllib.request

  5. # create a password manager
  6. password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()

  8. # Add the username and password.
  9. # If we knew the realm, we could use it instead of None.
  10. top_level_url = "https://cms.tetx.com/"
  11. password_mgr.add_password(None, top_level_url, 'yzhang', 'cccddd')

  13. handler = urllib.request.HTTPBasicAuthHandler(password_mgr)

  15. # create "opener" (OpenerDirector instance)
  16. opener = urllib.request.build_opener(handler)

  18. # use the opener to fetch a URL
  19. a_url = "https://cms.tetx.com/"
  20. x = opener.open(a_url)
  21. print(x.read())

  23. # Install the opener.
  24. # Now all calls to urllib.request.urlopen use our opener.
  25. urllib.request.install_opener(opener)

  27. a = urllib.request.urlopen(a_url).read().decode('utf8')
  28. print(a)
复制代码



9、使用代理

  1. #! /usr/bin/env python3

  3. import urllib.request

  5. proxy_support = urllib.request.ProxyHandler({'sock5': 'localhost:1080'})
  6. opener = urllib.request.build_opener(proxy_support)
  7. urllib.request.install_opener(opener)


  10. a = urllib.request.urlopen("http://g.cn").read().decode("utf8")
  11. print(a)
复制代码


10、超时

  1. #! /usr/bin/env python3

  3. import socket
  4. import urllib.request

  6. # timeout in seconds
  7. timeout = 2
  8. socket.setdefaulttimeout(timeout)

  10. # this call to urllib.request.urlopen now uses the default timeout
  11. # we have set in the socket module
  12. req = urllib.request.Request('http://twitter.com/')
  13. a = urllib.request.urlopen(req).read()
  14. print(a)
复制代码





欢迎光临 深圳全飞鸿 (http://www.nagomes.com/disc/) Powered by Discuz! X3.2