用scrapy-redis分布式爬虫爬取房天下广州的租房信息

我做过用selenium爬取房天下的租房信息也用过ajax分析接口爬取房天下的租房信息但是前两次任务都是小规模地爬取爬取的量比较小。所以这次准备大规模地爬取租房信息使用scrapy-redis分布式爬虫来爬取我只有一台电脑但是我做了两个爬虫来同时爬取一个从首页中爬取出子页面的链接一个从子页面中爬虫租房的详细信息在Pycharm下同时运行两个py文件。在settings.py文件中添加如下设置ROBOTSTXT_OBEY False DUPEFILTER_CLASS scrapy_redis.dupefilter.RFPDupeFilter SCHEDULER scrapy_redis.scheduler.Scheduler SCHEDULER_PERSIST True SCHEDULER_QUEUE_CLASS scrapy_redis.queue.SpiderPriorityQueue #处理重定向 MEDIA_ALLOW_REDIRECTS True DEFAULT_REQUEST_HEADERS { #添加请求头 Accept: text/html,application/xhtmlxml,application/xml;q0.9,*/*;q0.8, Accept-Language: en, User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36, referer: https://m.fang.com/zf/bj/?jhtypezf, authority: m.fang.com, x-requested-with: XMLHttpRequest, sec-fetch-mode: cors, sec-fetch-site: same-origin } ITEM_PIPELINES { fangtianxia.pipelines.FangtianxiaPipeline: 300,#自己编写的pipeline scrapy_redis.pipelines.RedisPipeline: 400 } DOWNLOAD_DELAY 1 LOG_LEVEL DEBUG REDIS_HOST 127.0.0.1#这个端口号可以直接在spiders的py文件中添置 REDIS_PORT 6379#这个端口号可以直接在spiders的py文件中添置自定义items.pyclass FangtianxiaItem(scrapy.Item): # define the fields for your item here like: # name scrapy.Field() text scrapy.Field() dizhi scrapy.Field() zujin scrapy.Field() faxing scrapy.Field()爬取子链接的py文件zufang.py# -*- coding: utf-8 -*- import scrapy import redis from scrapy_redis.spiders import RedisSpider from ..settings import REDIS_HOST,REDIS_PORT import re class ZufangSpider(RedisSpider): name zufang # allowed_domains [fang.com] # start_urls [http://fang.com/] base_url https://m.fang.com/zf/?purpose%D7%A1%D5%ACnotGetPurpose1city%B9%E3%D6%DDrenttypeczczfaajaxGetListcitygzr0.0021985656734149206page redis_key fangtianxia:start_urls #连接到redis pool redis.ConnectionPool(hostREDIS_HOST,port REDIS_PORT,db0) redis redis.StrictRedis(connection_poolpool) for index in range(1,1000): start_url base_url str(index) redis.lpush(fangtianxia:start_urls,start_url) def parse(self, response): res response.text domainurl https://m.fang.com/zf/gz/ results re.findall(ra.*?href\\(.*?), res) # 提取子搜索页面 https://m.fang.com/zf/gz/FAGT_241762951.html for result in results: result result.split(/)[-1] href domainurl result # 拼接子url print({href:href}) self.redis.rpush(fangtianxia:house_urls,href)解析页面的文件zufang1.pyfrom scrapy_redis.spiders import RedisSpider from ..items import FangtianxiaItem class FangtianxiaSpider(RedisSpider): name fangtianxia_down redis_key fangtianxia:house_urls def parse(self,response): item FangtianxiaItem() item[text] response.xpath(/html/body/div[3]/div[1]/section[3]/div[2]/ul/li[1]/p/text()).get() item[dizhi] response.xpath(/html/body/div[3]/div[1]/section[4]/ul/li[2]/a/span/text()).get() item[zujin] response.xpath(/html/body/div[3]/div[1]/section[2]/div[1]/p/span[1]/text()).get() item[faxing] response.xpath(/html/body/div[3]/div[1]/section[2]/div[2]/ul/li[2]/p/text()).get() yield item启动文件 start.pyfrom scrapy import cmdline cmdline.execute(scrapy crawl fangtianxia_down.split())启动文件只能跑一个代码另一个代码须放在Terminal终端运行。python zufang.py在redis数据库中可以看到保存的数据将redis数据中保存的Item信息存储到mongodb中import json,redis,pymongo def main(): rediscli redis.StrictRedis(host127.0.0.1,port6379,db0) mongocli pymongo.MongoClient(hostlocalhost,port27017) db mongocli[zufang] sheet db[info] offset 0 while True: source,data rediscli.blpop([fangtianxia_down:items])#读取的数据为bytes类型 item json.loads(data.decode(utf8))#item转化为字典类型 print(type(item)) print(source) if item[text] ! None:#将值为空的信息滤掉 sheet.insert(item) offset 1 print(offset) try: print(Processing:%s%item) except KeyError: print(Error proceding: %s % item) else: continue if __name__ __main__: main()此篇爬虫主要参考这个https://www.jianshu.com/p/ba67094d2813感谢作者