例子：爬取笔趣阁的小说圣墟

1.爬取小说章节的URL

from bs4 import BeautifulSoup
from selenium import webdriver
import re

def book_url():
    chromeOptions = webdriver.ChromeOptions()
# 设置代理
    chromeOptions.add_argument("--proxy-server=http://202.20.16.82:10152")
# 一定要注意，=两边不能有空格
    driver = webdriver.Firefox()
    driver.implicitly_wait(30)  # 隐性等待，最长等30秒
    driver.get(r\'http://www.xbiquge.la/13/13959/\')
    txt = driver.page_source
    soup = BeautifulSoup(txt, \'html.parser\')
    url = re.findall(\'<a href="(.*)">\', str(soup.find_all(\'div\', id=\'list\')))
    word = re.findall(\'<a.*>(.*)</a>\', str(soup.find_all(\'div\', id=\'list\')))
    word_dict = dict(zip(list(word), list(url)))
    driver.quit()
return word_dict

2.爬取小说前200章的内容并写入txt文本中

from bs4 import BeautifulSoup
from selenium import webdriver
import re
import codecs
import crawling.pro_2.py1 as py1


def url():
    word_dict = py1.book_url()
    word = []
for i in word_dict.values():
        word.append(i)
return word


def book(url):
    chromeOptions = webdriver.ChromeOptions()
# 设置代理
    chromeOptions.add_argument("--proxy-server=http://202.20.16.82:10152")
# 一定要注意，=两边不能有空格，不能是这样--proxy-server = http://202.20.16.82:10152
    driver = webdriver.Firefox()
    driver.implicitly_wait(30)  # 隐性等待，最长等30秒
    driver.get(\'http://www.xbiquge.la/\'+url)
    txt = driver.page_source
    soup = BeautifulSoup(txt, \'html.parser\')
    a = soup.find_all(\'div\', id=\'content\')
    a = re.sub(r\'<div id="content">\', \'\', str(a))
    a = re.sub(r\'</p></div>\', \'\', str(a))
    a = re.sub(r\'\xa0\', \'\', str(a))
    a = re.sub(r\'<p><a href=\', \'\', str(a))
    a = re.sub(r\'target="_blank">\', \'\', str(a))
    a = re.sub(r\'</a>\', \'\', str(a))
    a = str(a)
    line = list(a.split("<br/>"))
    name = re.findall(\'<h1>(.*)</h1\', str(soup.find_all(\'div\', class_=\'bookname\')))
    name = re.sub("\'", \'\', str(name))
    f = codecs.open(\'小说圣墟.txt\', \'a\', \'utf-8\')
    kong_list = []
for j in line:
if j == \'\n\':
            kong_list.append(j)
for k in kong_list:
        line.remove(k)
print(name, end=\'\n\', file=f)
for i in line:
        text = re.sub(r\'\n\', \'\', i)
print(text, file=f)
    driver.quit()


if __name__ == \'__main__\':
    url_list = url()
del url_list[200:]
for time in url_list:
        book(time)