python抓取网页资源的方法验证

zhgc

1、最简单

1. import urllib.request
   
2. response = urllib.request.urlopen('http://python.org/')
   
3. html = response.read()
   

复制代码

2、使用 Request

1. import urllib.request
   
2. 
   
3. req = urllib.request.Request('http://python.org/')
   
4. response = urllib.request.urlopen(req)
   
5. the_page = response.read()

复制代码

3、发送数据

1. 
   
2. #! /usr/bin/env python3
   
3. 
   
4. import urllib.parse
   
5. import urllib.request
   
6. 
   
7. url = 'http://localhost/login.php'
   
8. user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
   
9. values = {
   
10. 'act' : 'login',
    
11. 'login[email]' : 'yzhang@i9i8.com',
    
12. 'login[password]' : '123456'
    
13.          }
    
14. 
    
15. data = urllib.parse.urlencode(values)
    
16. req = urllib.request.Request(url, data)
    
17. req.add_header('Referer', 'http://www.python.org/')
    
18. response = urllib.request.urlopen(req)
    
19. the_page = response.read()
    
20. 
    
21. print(the_page.decode("utf8"))
    

复制代码

4、发送数据和header

1. #! /usr/bin/env python3
   
2. 
   
3. import urllib.parse
   
4. import urllib.request
   
5. 
   
6. url = 'http://localhost/login.php'
   
7. user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
   
8. values = {
   
9. 'act' : 'login',
   
10. 'login[email]' : 'yzhang@i9i8.com',
    
11. 'login[password]' : '123456'
    
12.          }
    
13. headers = { 'User-Agent' : user_agent }
    
14. 
    
15. data = urllib.parse.urlencode(values)
    
16. req = urllib.request.Request(url, data, headers)
    
17. response = urllib.request.urlopen(req)
    
18. the_page = response.read()
    
19. 
    
20. print(the_page.decode("utf8"))
    

复制代码

5、http 错误

1. #! /usr/bin/env python3
   
2. 
   
3. import urllib.request
   
4. 
   
5. req = urllib.request.Request('http://www.python.org/fish.html')
   
6. try:
   
7.     urllib.request.urlopen(req)
   
8. except urllib.error.HTTPError as e:
   
9. print(e.code)
   
10. print(e.read().decode("utf8"))
    

复制代码

6、异常处理1

1. #! /usr/bin/env python3
   
2. 
   
3. from urllib.request import Request, urlopen
   
4. from urllib.error import URLError, HTTPError
   
5. req = Request("http://twitter.com/")
   
6. try:
   
7.     response = urlopen(req)
   
8. except HTTPError as e:
   
9. print('The server couldn\'t fulfill the request.')
   
10. print('Error code: ', e.code)
    
11. except URLError as e:
    
12. print('We failed to reach a server.')
    
13. print('Reason: ', e.reason)
    
14. else:
    
15. print("good!")
    
16. print(response.read().decode("utf8"))
    

复制代码

7、异常处理2

1. #! /usr/bin/env python3
   
2. 
   
3. from urllib.request import Request, urlopen
   
4. from urllib.error import  URLError
   
5. req = Request("http://twitter.com/")
   
6. try:
   
7.     response = urlopen(req)
   
8. except URLError as e:
   
9. if hasattr(e, 'reason'):
   
10. print('We failed to reach a server.')
    
11. print('Reason: ', e.reason)
    
12. elif hasattr(e, 'code'):
    
13. print('The server couldn\'t fulfill the request.')
    
14. print('Error code: ', e.code)
    
15. else:
    
16. print("good!")
    
17. print(response.read().decode("utf8"))
    

复制代码

8、HTTP 认证

1. #! /usr/bin/env python3
   
2. 
   
3. import urllib.request
   
4. 
   
5. # create a password manager
   
6. password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
   
7. 
   
8. # Add the username and password.
   
9. # If we knew the realm, we could use it instead of None.
   
10. top_level_url = "https://cms.tetx.com/"
    
11. password_mgr.add_password(None, top_level_url, 'yzhang', 'cccddd')
    
12. 
    
13. handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
    
14. 
    
15. # create "opener" (OpenerDirector instance)
    
16. opener = urllib.request.build_opener(handler)
    
17. 
    
18. # use the opener to fetch a URL
    
19. a_url = "https://cms.tetx.com/"
    
20. x = opener.open(a_url)
    
21. print(x.read())
    
22. 
    
23. # Install the opener.
    
24. # Now all calls to urllib.request.urlopen use our opener.
    
25. urllib.request.install_opener(opener)
    
26. 
    
27. a = urllib.request.urlopen(a_url).read().decode('utf8')
    
28. print(a)

复制代码

9、使用代理

1. #! /usr/bin/env python3
   
2. 
   
3. import urllib.request
   
4. 
   
5. proxy_support = urllib.request.ProxyHandler({'sock5': 'localhost:1080'})
   
6. opener = urllib.request.build_opener(proxy_support)
   
7. urllib.request.install_opener(opener)
   
8. 
   
9. 
   
10. a = urllib.request.urlopen("http://g.cn").read().decode("utf8")
    
11. print(a)
    

复制代码

10、超时

1. #! /usr/bin/env python3
   
2. 
   
3. import socket
   
4. import urllib.request
   
5. 
   
6. # timeout in seconds
   
7. timeout = 2
   
8. socket.setdefaulttimeout(timeout)
   
9. 
   
10. # this call to urllib.request.urlopen now uses the default timeout
    
11. # we have set in the socket module
    
12. req = urllib.request.Request('http://twitter.com/')
    
13. a = urllib.request.urlopen(req).read()
    
14. print(a)
    

复制代码