python爬虫笔记（九）实例4：前程无忧招聘信息爬虫

douzujun 2021-08-27 原文

1. 前程无忧招聘信息爬虫

（设置选项后）分析链接得：https://search.51job.com/jobsearch/search_result.php

# -*- coding: utf-8 -*-

import requests
import re
from parsel import Selector

key = "java"
url = "https://search.51job.com/jobsearch/search_result.php"

data = {"fromJs" : "1", 
        "jobarea" : "020000",
        "keyword" : key, 
        "keywordtype" : "2", 
        "curr_page" : "1",
        }
hd = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36"}
response = requests.get(url, params=data, headers=hd)
print(response.request.url)
# 解决编码问题
data = bytes(response.text, response.encoding).decode("gbk", "ignore")
pat_page = "共(.*?)条职位"
allline = re.compile(pat_page, re.S).findall(data)[0]
print(allline)                     # allline条数据
allpage = int(allline) // 50 + 1   # 每页50条数据
for i in range(0, allpage):
    print("----正在爬" + str(i + 1) + "页-------")
    getdata = {"fromJs" : "1", 
                "jobarea" : "020000",
                "keyword" : key, 
                "keywordtype" : "2", 
                "curr_page" : str(i+1),
                }
    response = requests.get(url, params=getdata, headers=hd)
    # 解决编码问题
    thisdata = bytes(response.text, response.encoding).decode("gbk", "ignore")
#    print(response.request.url)
    sel = Selector(thisdata)
    job_url_all = sel.xpath("//p[@class=\'t1 \']/span/a/@href").getall()
    # 遍历当前页上所有工作链接
    for job_url in job_url_all:
#        print(job_url)
        thisurl = job_url
        response = requests.get(thisurl)
        thisdata = bytes(response.text, response.encoding).decode("gbk", "ignore")
        
        pat_title = \'<h1 title="(.*?)"\'
        pat_company = \'<p class="cname">.*?title="(.*?)"\'
        pat_money = \'<div class="tHeader tHjob">.*?<strong>(.*?)</strong>\'
        pat_msg = \'<div class="bmsg job_msg inbox">(.*?)<div class="share">\'
        
        title = re.compile(pat_title, re.S).findall(thisdata)[0]
        company = re.compile(pat_company, re.S).findall(thisdata)[0]
        money = re.compile(pat_money, re.S).findall(thisdata)[0]
        msg = re.compile(pat_msg, re.S).findall(thisdata)[0]
        print(\'-----------------------------------\')
        print("岗位: ", title)
        print("公司: ", company)
        print("薪资: ", money)
        print("岗位要求: ", msg)