python爬虫之爬取小说（二）

jsxxd 2021-08-30 原文

爬取“全书网”《斗罗大陆》小说

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by Fzy on 2018/12/27 17:14
import requests
import re
headers = {
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\'
}


def get_html(first_url):
    try:
        r = requests.get(first_url, headers=headers)
        r.encoding = \'gbk\'
        html = r.text
        return html
    except Exception as e:
        peat = str(e) + \'\n访问失败！\'
        print(peat)


def get_info(html):
    txt_info = {}
    txt_info[\'title\'] = re.findall(r\'<div class="chapName">.*?<strong>(.*?)</strong>\', html)[0]
    txt_info[\'author\'] = re.findall(r\'<div class="chapName"><span class="r">(.*?)</span>\', html)[0]
    return txt_info


def get_urls(html):
    li_tags = re.findall(r\'<DIV class="clearfix dirconone">(.*?)</div>\', html, re.S|re.I)[0]
    urls = re.findall(r\'<a href="(.*?)"\', li_tags)
    return urls


def save_text(urls, txt_info):
    with open(txt_info[\'title\'] + \'.txt\', \'a+\') as file:
        file.write(txt_info[\'title\']+\'\n\n\')
        file.write(txt_info[\'author\']+\'\n\')
    print(\'正在下载《{}》全本小说（共六百八十七章），时间稍长，请稍等......\'.format(txt_info[\'title\']))
    for url in urls:
        html = get_html(url)
        text = re.findall(r\'&nbsp;&nbsp;&nbsp;&nbsp;(.*?)<br />\', html)
        text1 = re.findall(r\'&nbsp;&nbsp;&nbsp;&nbsp;(.*?)<script type="text/javascript">\', html)[0]
        text.append(text1)
        for i in text:
            with open(txt_info[\'title\']+\'.txt\', \'a+\') as file:
                file.write(\'    \'+i+\'\n\')
        print(text[0])


def main():
    first_url = \'http://www.quanshuwang.com/book/44/44683\'
    html = get_html(first_url)
    txt_info = get_info(html)
    urls = get_urls(html)
    save_text(urls, txt_info)


if __name__ == \'__main__\':
    main()