rules=(
Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+.htm'),callback='parse_item'),
Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+.htm'),follow=True),
)
1、##
-- coding: utf-8 --
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import re
class CfSpider(CrawlSpider):
name = 'cf'
allowed_domains = ['bxjg.circ.gov.cn']
start_urls = ['http://bxjg.circ.gov.cn/web/site0/tab5240/Default.htm']
rules = (
Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+\.htm'), callback='parse_item', ),
Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+\.htm'),follow=True, ),
)
def parse_item(self, response):
item = {}
item['title']=response.xpath('//title/text()').extract()
item['publish_date'] = response.xpath("//meta[@name='docdate']/@content").extract()
print(item)
2、settings.py
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5'
DOWNLOAD_DELAY=1
