初學(xué)Python爬蟲時(shí)都會(huì)從最簡單的方法開始，以下為幾種常見的基礎(chǔ)做法。
"""
簡單的循環(huán)處理
"""
import
requests
url_list = [
"https://www.baidu.com"
,
"https://www.cnblogs.com/"
]
for
url
in
url_list:
result = requests.get(url)
print
(result.text)
"""
線程池處理
"""
import
requests
from
concurrent.futures
import
ThreadPoolExecutor
def
fetch_request
(
url
):
result = requests.get(
url
)
print
(result.text)
url_list = [
"https://www.baidu.com/"
,
"https://www.cnblogs.com/"
]
pool = ThreadPoolExecutor(
10
)
for
url
in
url_list:
# 線程池中獲取線程，執(zhí)行fetch_request方法
pool.submit(fetch_request
,
url)
# 關(guān)閉線程池
pool.shutdown()
"""
線程池+回調(diào)函數(shù)
"""
import
requests
from
concurrent.futures
import
ThreadPoolExecutor
def
fetch_async
(
url
):
response = requests.get(
url
)
return
response
def
callback
(
future
):
print
(
future
.result().text)
url_list = [
"https://www.baidu.com/"
,
"https://www.cnblogs.com/"
]
pool = ThreadPoolExecutor(
10
)
for
url
in
url_list:
v = pool.submit(fetch_async
,
url)
# 調(diào)用回調(diào)函數(shù)
v.add_done_callback(callback)
pool.shutdown()
"""
進(jìn)程池處理
"""
import
requests
from
concurrent.futures
import
ProcessPoolExecutor
def
fetch_requst
(
url
):
result = requests.get(
url
)
print
(result.text)
url_list = [
"https://www.baidu.com/"
,
"https://www.cnblogs.com/"
]
if
__name__ ==
'__main__'
:
pool = ProcessPoolExecutor(
max_workers
=
10
)
for
url
in
url_list:
pool.submit(fetch_requst
,
url)
pool.shutdown()
"""
進(jìn)程池+回調(diào)函數(shù)
"""
import
requests
from
concurrent.futures
import
ProcessPoolExecutor
def
fetch_async
(
url
):
response = requests.get(
url
)
return
response
def
callback
(
future
):
print
(
future
.result().text)
url_list = [
"https://www.baidu.com/"
,
"https://www.cnblogs.com/"
]
if
__name__ ==
'__main__'
:
pool = ProcessPoolExecutor(
10
)
for
url
in
url_list:
v = pool.submit(fetch_async
,
url)
v.add_done_callback(callback)
pool.shutdown()
更多文章、技術(shù)交流、商務(wù)合作、聯(lián)系博主
微信掃碼或搜索：z360901061
微信掃一掃加我為好友
QQ號聯(lián)系： 360901061
您的支持是博主寫作最大的動(dòng)力，如果您喜歡我的文章，感覺我的文章對您有幫助，請用微信掃描下面二維碼支持博主2元、5元、10元、20元等您想捐的金額吧，狠狠點(diǎn)擊下面給點(diǎn)支持吧，站長非常感激您！手機(jī)微信長按不能支付解決辦法：請將微信支付二維碼保存到相冊，切換到微信，然后點(diǎn)擊微信右上角掃一掃功能，選擇支付二維碼完成支付。
【本文對您有幫助就好】元
