丁香园防禁封策略-分布式实战

class DrugsItem(scrapy.Item): # define the fields for your item here like: #药品不存在标记 exists = scrapy.Field() #药品id drugtId = scrapy.Field() #数据 data = scrapy.Field() #标记验证码状态 msg = scrapy.Field() pass

# -*- coding: utf-8 -*- import scrapy from drugs_dxy.items import DrugsItem import re class DrugsSpider(scrapy.Spider): name = "Drugs" allowed_domains = ["dxy.cn"] size = 60 def start_requests(self): for i in xrange(50000,50000+self.size,1): url ='http://drugs.dxy.cn/drug/%d.htm' % (i) yield scrapy.Request(url=url,callback=self.parse) def parse(self, response): drug_Item = DrugsItem() drug_Item["drugtId"] = int(re.search('(\d+)',response.url).group(1)) if drug_Item["drugtId"]>=150000: return url ='http://drugs.dxy.cn/drug/%d.htm' % (drug_Item["drugtId"]+self.size) yield scrapy.Request(url=url,callback=self.parse) if '药品不存在' in response.body: drug_Item['exists'] = False yield drug_Item return if '请填写验证码继续正常访问' in response.body: drug_Item["msg"] = u'请填写验证码继续正常访问' return drug_Item["data"] = {} details = response.xpath("//dt") for detail in details: detail_name = detail.xpath('./span/text()').extract()[0].split(':')[0] if detail_name ==u'药品名称': drug_Item['data'][u'药品名称'] = {} try: detail_str = detail.xpath("./following-sibling::*[1]") detail_value = detail_str.xpath('string(.)').extract()[0].replace('\r','').replace('\t','').strip() for item in detail_value.split('\n'): item = item.replace('\r','').replace('\n','').replace('\t','').strip() name = item.split(u'：')[0] value = item.split(u'：')[1] #print name,value drug_Item['data'][u'药品名称'][name] = value except: pass else: detail_str = detail.xpath("./following-sibling::*[1]") detail_value = detail_str.xpath('string(.)').extract()[0].replace('\r','').replace('\t','').strip() #print detail_str,detail_value drug_Item['data'][detail_name] = detail_value yield drug_Item

# Enables scheduling storing requests queue in redis. SCHEDULER = "scrapy_redis.scheduler.Scheduler" # Ensure all spiders share same duplicates filter through redis. DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter" # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'scrapy_redis.pipelines.RedisPipeline': 300 } # Specify the host and port to use when connecting to Redis (optional). REDIS_HOST = '101.200.170.171' REDIS_PORT = 6379 # Custom redis client parameters (i.e.: socket timeout, etc.) REDIS_PARAMS = {} #REDIS_URL = 'redis://user:pass@hostname:9001' REDIS_PARAMS['password'] = 'itcast.cn'

# Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { 'drugs_dxy.middlewares.ProxyMiddleware': 400, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': None, }

DOWNLOAD_TIMEOUT = 90 REDIRECT_ENABLED = False

# -*- coding: utf-8 -*- import random import base64 import Queue import redis class ProxyMiddleware(object): def __init__(self, settings): self.queue = 'Proxy:queue' # 初始化代理列表 self.r = redis.Redis(host=settings.get('REDIS_HOST'),port=settings.get('REDIS_PORT'),db=1,password=settings.get('REDIS_PARAMS')['password']) @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) def process_request(self, request, spider): proxy={} source, data = self.r.blpop(self.queue) proxy['ip_port']=data proxy['user_pass']=None if proxy['user_pass'] is not None: #request.meta['proxy'] = "http://YOUR_PROXY_IP:PORT" request.meta['proxy'] = "http://%s" % proxy['ip_port'] #proxy_user_pass = "USERNAME:PASSWORD" encoded_user_pass = base64.encodestring(proxy['user_pass']) request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass print "********ProxyMiddleware have pass*****" + proxy['ip_port'] else: #ProxyMiddleware no pass print request.url, proxy['ip_port'] request.meta['proxy'] = "http://%s" % proxy['ip_port'] def process_response(self, request, response, spider): """ 检查response.status, 根据status是否在允许的状态码中决定是否切换到下一个proxy, 或者禁用proxy """ print("-------%s %s %s------" % (request.meta["proxy"], response.status, request.url)) # status不是正常的200而且不在spider声明的正常爬取过程中可能出现的 # status列表中, 则认为代理无效, 切换代理 if response.status == 200: print 'rpush',request.meta["proxy"] self.r.rpush(self.queue, request.meta["proxy"].replace('http://','')) return response def process_exception(self, request, exception, spider): """ 处理由于使用代理导致的连接异常 """ proxy={} source, data = self.r.blpop(self.queue) proxy['ip_port']=data proxy['user_pass']=None request.meta['proxy'] = "http://%s" % proxy['ip_port'] new_request = request.copy() new_request.dont_filter = True return new_request