python 爬取微信公众号文章二
from selenium import webdriver import time import json import random import requests import re import pymysql from urllib import parse gzlist = [] def getListWxName(): db = pymysql.connect("xxx.xx.xx.xx", "root", "xxx", "xxx") cursor = db.cursor() sql = "SELECT w.wxName FROM `wx_account` w;" try: cursor.execute(sql) results = cursor.fetchall() for row in results: gzlist.append(row[0]) print(gzlist) except Exception as inst: print(inst) db.rollback() # 登录微信公众号，获取登录之后的cookies信息，并保存到本地文本中 def wechat_login(): # 用webdriver启动谷歌浏览器 print("启动浏览器，打开微信公众号登录界面") driver = webdriver.Chrome(r"E:\chromedriver.exe") driver.get("https://mp.weixin.qq.com/") time.sleep(2) print("正在输入微信公众号登录账号和密码......") time.sleep(20) print("登录成功") # 获取cookies cookie_items = driver.get_cookies() post = {} # 获取到的cookies是列表形式，将cookies转成json形式并存入本地名为cookie的文本中 for cookie_item in cookie_items: post[cookie_item[\'name\']] = cookie_item[\'value\'] cookie_str = json.dumps(post) with open(\'cookie.txt\', \'w+\', encoding=\'utf-8\') as f: f.write(cookie_str) print("cookies信息已保存到本地") driver.quit() # 爬取微信公众号文章，并存在本地文本中 def get_content(query): # query为要爬取的公众号名称 # 公众号主页 url = \'https://mp.weixin.qq.com\' # 设置headers header = { "HOST": "mp.weixin.qq.com", "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36" } from requests.packages import urllib3 urllib3.disable_warnings() # 关闭警告 # 读取上一步获取到的cookies with open(\'cookie.txt\', \'r\', encoding=\'utf-8\') as f: cookie = f.read() cookies = json.loads(cookie) # 增加重试连接次数 session = requests.Session() session.keep_alive = False # 增加重试连接次数 session.adapters.DEFAULT_RETRIES = 511 time.sleep(5) # 登录之后的微信公众号首页url变化为：https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token=1849751598，从这里获取token信息 response = session.get(url=url, cookies=cookies, verify=False) token = re.findall(r\'token=(\d+)\', str(response.url))[0] time.sleep(2) # 搜索微信公众号的接口地址 search_url = \'https://mp.weixin.qq.com/cgi-bin/searchbiz?\' # 搜索微信公众号接口需要传入的参数，有三个变量：微信公众号token、随机数random、搜索的微信公众号名字 query_id = { \'action\': \'search_biz\', \'token\': token, \'lang\': \'zh_CN\', \'f\': \'json\', \'ajax\': \'1\', \'random\': random.random(), \'query\': query, \'begin\': \'0\', \'count\': \'5\' } # 打开搜索微信公众号接口地址，需要传入相关参数信息如：cookies、params、headers search_response = session.get( search_url, cookies=cookies, headers=header, params=query_id) # 取搜索结果中的第一个公众号 lists = search_response.json().get(\'list\')[0] print(lists) # 获取这个公众号的fakeid，后面爬取公众号文章需要此字段 fakeid = lists.get(\'fakeid\') # 微信公众号文章接口地址 appmsg_url = \'https://mp.weixin.qq.com/cgi-bin/appmsg?\' # 搜索文章需要传入几个参数：登录的公众号token、要爬取文章的公众号fakeid、随机数random query_id_data = { \'token\': token, \'lang\': \'zh_CN\', \'f\': \'json\', \'ajax\': \'1\', \'random\': random.random(), \'action\': \'list_ex\', \'begin\': \'0\', # 不同页，此参数变化，变化规则为每页加5 \'count\': \'5\', \'query\': \'\', \'fakeid\': fakeid, \'type\': \'9\' } # 打开搜索的微信公众号文章列表页 appmsg_response = session.get( appmsg_url, cookies=cookies, headers=header, params=query_id_data) # 获取文章总数 max_num = appmsg_response.json().get(\'app_msg_cnt\') # 每页至少有5条，获取文章总的页数，爬取时需要分页爬 num = int(int(max_num) / 5) # 起始页begin参数，往后每页加5 begin = 0 seq = 0 while num + 1 > 0: query_id_data = { \'token\': token, \'lang\': \'zh_CN\', \'f\': \'json\', \'ajax\': \'1\', \'random\': random.random(), \'action\': \'list_ex\', \'begin\': \'{}\'.format(str(begin)), \'count\': \'5\', \'query\': \'\', \'fakeid\': fakeid, \'type\': \'9\' } print(\'正在翻页：--------------\', begin) time.sleep(5) # 获取每一页文章的标题和链接地址，并写入本地文本中 query_fakeid_response = requests.get( appmsg_url, cookies=cookies, headers=header, params=query_id_data) fakeid_list = query_fakeid_response.json().get(\'app_msg_list\') if fakeid_list: for item in fakeid_list: content_link = item.get(\'link\') content_title = item.get(\'title\') fileName = query + \'.txt\' qs = parse.parse_qs(content_link) articleId = qs[\'mid\'][0] seq += 1 with open(fileName, \'a\', encoding=\'utf-8\') as fh: fh.write(str(seq) + "|" + articleId + "|" + content_title + "|" + content_link + "\n") num -= 1 begin = int(begin) begin += 5 if __name__ == \'__main__\': getListWxName() # 登录微信公众号，获取登录之后的cookies信息，并保存到本地文本中 wechat_login() query = "XXX" print("开始爬取公众号：" + query) get_content(query) print("爬取完成") # 登录之后，通过微信公众号后台提供的微信公众号文章接口爬取文章 for query in gzlist: # 爬取微信公众号文章，并存在本地文本中 print("开始爬取公众号：" + query) get_content(query) print("爬取完成")
SET FOREIGN_KEY_CHECKS=0; -- ---------------------------- -- Table structure for wx_article -- ---------------------------- DROP TABLE IF EXISTS `wx_article`; CREATE TABLE `wx_article` ( `id` int(11) NOT NULL AUTO_INCREMENT, `wxCode` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `wxBiz` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `articleId` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `articleTitle` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `articleUrl` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `articleDigest` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `coverUrl` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `articleSouUrl` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `readNum` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `goodNum` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `companyName` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `creditCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `regisCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `cityCode` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `publishTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `createTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, `updateTime` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) USING BTREE, UNIQUE KEY `articleUrl` (`articleUrl`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=15459 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
https://www.cnblogs.com/xiao-apple36/p/9447877.html
https://cuiqingcai.com/4652.html
版权声明：本文为luweiweicode原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。