wistbean / wistbean/learn_python3_spider

request error 是被反爬虫了吗

Open
#79 0 comments 0 reactions 0 assignees View on GitHub

Nobody has claimed this yet.

Dominant language
Python
Stars
22.1k
Forks
3.9k
PR merge metrics
No merged PRs in 30d

Description

mport requests
import threading
import pandas as pd
import time
from lxml import etree
from fake_useragent import UserAgent
from random import choice
from queue import Queue

代理IP池,隐藏或改变爬虫的真实IP地址。

proxies = [
('http', 'http://120.55.14.64:80'),
('http', 'http://47.96.37.159:80'),
('http', 'http://60.174.0.184:8089'),
('http', 'http://182.204.178.124:8089')
]

#随机生成useragent
ua = UserAgent()
#生成1-50页url
def url_creat():
url = 'https://nj.lianjia.com/ershoufang/pg{}/'
#生成前100页url列表
links=[url.format(i) for i in range(1,51)]
return links

#解析url并提取数据
def url_parse(url,result_queue):
try:
headers = {
'Accept-Encoding': 'gzip, deflate, br, zstd',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Cookie': 'lianjia_uuid=1482b2e7-3112-4886-a41e-9d2d2e86b44d; _jzqc=1; _jzqckmp=1; sajssdk_2015_cross_new_user=1; _ga=GA1.2.452213069.1724131249; _gid=GA1.2.1832912406.1724131249; _ga_654P0WDKYN=GS1.2.1724131249.1.0.1724131249.0.0.0; hip=apJG191KDRPikxgtTyi3OmoLYtJFJa9yCiORXQVgZ51QYD0SMeG3eATx2a1EuGGf7_iZ0YxmbRitHPS-wygw6eU7_uOz42xUXwELiqydaMqCOjZqUVrz7n77aek0s8UJGOuDAVObHUnwVJIEA9dtMtsaU2qLrjx0DsDaDb26BNk6uSjJ2owluyHyCg%3D%3D; _jzqx=1.1724131236.1724138261.2.jzqsr=cn%2Ebing%2Ecom|jzqct=/.jzqsr=hip%2Elianjia%2Ecom|jzqct=/; _ga_DX18CJBZRT=GS1.2.1724138272.1.0.1724138272.0.0.0; _ga_EYZV9X59TQ=GS1.2.1724138272.1.0.1724138272.0.0.0; select_city=320100; lianjia_ssid=aed34a25-3548-4cb0-9828-61f1b13e8284; Qs_lvt_200116=1724138352; Hm_lvt_9152f8221cb6243a53c83b956842be8a=1724138353; HMACCOUNT=67BCC534482CC88A; jzqa=1.3305053847575342600.1724131236.1724138261.1724140333.3; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221916e3adab62f-06c1cb6122e849-4c657b58-1024000-1916e3adab777c%22%2C%22%24device_id%22%3A%221916e3adab62f-06c1cb6122e849-4c657b58-1024000-1916e3adab777c%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; Qs_pv_200116=2599661905406085600%2C889815816991505000%2C2899766764332378600%2C91402511096122320%2C1000057718938937700; Hm_lpvt_9152f8221cb6243a53c83b956842be8a=1724140491; _jzqb=1.5.10.1724140333.1; _ga_E91JCCJY3Z=GS1.2.1724140344.3.1.1724140502.0.0.0; _ga_MFYNHLJT0H=GS1.2.1724140344.3.1.1724140502.0.0.0',
'Host': 'dig.lianjia.com',
'Pragma': 'no-cache',
'Referer': 'https://nj.lianjia.com/',
'sec-ch-ua': '"Not)A;Brand";v="99", "Microsoft Edge";v="127", "Chromium";v="127"',
'sec-ch-ua-mobile': '?1',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'no-cors',
'Sec-Fetch-Site': 'same-site',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent':ua.random
}
proxy = choice(proxies)
response=requests.get(url=url,headers=headers,proxies={proxy[0]: proxy[1]})
response.raise_for_status() # 检查请求是否成功
tree=etree.HTML(response.text)
#ul列表下的全部li标签
li_List=tree.xpath("//*[@class='sellListContent']/li")
results=[]

    for li in li_List:
        try:
    #标题
            title=li.xpath('./div/div/a/text()')[0]
    #网址
            link=li.xpath('./div/div/a/@href')[0]
    #位置
            postion=li.xpath('./div/div[2]/div/a/text()')[0]+li.xpath('./div/div[2]/div/a[2]/text()')[0]
    #类型
            types=li.xpath('./div/div[3]/div/text()')[0].split(' | ')[0]
    #面积
            area=li.xpath('./div/div[3]/div/text()')[0].split(' | ')[1]
    #房屋信息
            info=li.xpath('./div/div[3]/div/text()')[0].split(' | ')[2:-1]
            info=''.join(info)
    #总价
            count_price=li.xpath('.//div/div[6]/div/span/text()')[0]+'万'
    #单价
            angle_price=li.xpath('.//div/div[6]/div[2]/span/text()')[0]
            dic={'标题':title,"位置":postion,'房屋类型':types,'面积':area,"单价":angle_price,'总价':count_price,'介绍':info,"网址":link}
            print(dic)
    #将房屋信息加入总列表中
            results.append(dic)
        except IndexError:  
            print(f"Error parsing data from {url}")  
            continue  
        # 将结果放入队列中
    result_queue.put(results)  
except requests.RequestException as e:  
    print(f"Request error for {url}: {e}")  

def run():
links = url_creat()
threads=[]
result_queue = Queue()
lock = threading.Lock()
all_results=[]
#多线程爬取
for i in links:
x=threading.Thread(target=url_parse,args=(i,result_queue))
x.start()
threads.append(x)
#设置请求之间的时间间隔
time.sleep(1)
for t in threads:
t.join()

# 收集所有结果  
while not result_queue.empty():   
    page_results = result_queue.get()  # 取出的是列表的列表  
    all_results.extend(page_results[0])  # 扩展第一个元素(即包含字典的列表)  
#将全部房屋信息转化为excel
data=pd.DataFrame(all_results)
data.to_excel('房屋信息822.xlsx',index=False)

if name == 'main':
run()

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

Start at the url_parse entry point in the supplied script, especially requests.get and response.raise_for_status, then run a single generated URL to capture the actual exception or response status. The issue does not provide an error message or a named test; the work is done when the request failure's cause is identified and the expected request behavior is established.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
data-engineering
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.