scrapy学习案例_爬取读书网书籍信息

此案例使用了crawlspider

1. 创建scrapy项目

scrapy startproject dushu 

2. 创建爬虫文件(../dushu/dushu/spiders)

scrapy genspider -t crawl read https://www.dushu.com/book/1188_1.html 

爬取思路

1. 制定规则

rules = (Rule(LinkExtractor(allow=r"/book/1188_\d+\.html"), callback="parse_item", follow=True),) 

2. 使用xpath解析response数据,找到每个数据最外层结构

img_list = response.xpath('//div[@class="bookslist"]//img') 

3. 循环img_list再次使用xpath解析其中的数据(name, src)

name = img.xpath('./@data-original').extract_first() src = img.xpath('./@alt').extract_first() 

4. 调用items.py里面的类, 把这个对象交给管道

from dushu.items import DushuItem  book = DushuItem(name=name, src=src)  yield book 

5. 设置items

name = scrapy.Field() src = scrapy.Field() 

6. 设置pipelines

将数据存储在MySQL数据库

• 预先下载好MySQL

• # 创建数据库(mysql>)  # 进入mysql>(终端) mysql -uroot -p  # 创建数据库 create database spider01 charset=utf8;  use spider01;  create table book( 	id int primary key auto-increment, 	name varchar(128), 	src varchar(128) );  # 查看book select * from book 

• # 文件导入(pipelines.py) from scrapy.utils.project import get_project_settings import pymysql 

• # 配置常量(settings.py) DB_HOST = "127.0.0.1"	# 默认ip DB_PORT = 3306			# 注意,这里是整数 DB_USER = "root" DB_PASSWORD = "<密码>" DB_NAME = "spider01" DB_CHARSET = "utf8" 

• # 添加下载到数据库的管道 class MysqlPipeline:      def open_spider(self, spider):         settings = get_project_settings()          self.host = settings['DB_HOST']         self.port = settings['DB_PORT']         self.user = settings['DB_USER']         self.password = settings['DB_PASSWORD']         self.name = settings['DB_NAME']         self.charset = settings['DB_CHARSET']          self.connect()      def connect(self):         self.conn = pymysql.connect(                 host=self.host,                 port=self.port,                 user=self.user,                 password=self.password,                 db=self.name,                 charset=self.charset         )         self.cursor = self.conn.cursor()      def process_item(self, item, spider):         sql = 'insert into book(name, src) values ("{}","{}")'.format(item['name'], item['src'])         self.cursor.execute(sql)         self.conn.commit()         return item      def close_spider(self, spider):         self.conn.close()         self.cursor.close()  

• # 解封及添加(settings.py)  ITEM_PIPELINES = {    "dushu.pipelines.DushuPipeline": 300,    "dushu.pipelines.MysqlPipeline": 301, } 

3. 运行文件

scrapy crawl read 

代码

• ../dushu/dushu/spiders/read.py

import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule from dushu.items import DushuItem   class ReadSpider(CrawlSpider):     name = "read"     allowed_domains = ["www.dushu.com"]     start_urls = ["https://www.dushu.com/book/1188_1.html"]      rules = (Rule(LinkExtractor(allow=r"/book/1188_\d+\.html"), callback="parse_item", follow=True),)       def parse_item(self, response):          img_list = response.xpath('//div[@class="bookslist"]//img')          for img in img_list:             name = img.xpath('./@data-original').extract_first()             src = img.xpath('./@alt').extract_first()              book = DushuItem(name=name, src=src)              yield book 

• ../dushu/dushu/items.py

# Define here the models for your scraped items # # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html  import scrapy   class DushuItem(scrapy.Item):     # define the fields for your item here like:     name = scrapy.Field()     src = scrapy.Field()  

• ../dushu/dushu/pipelines.py

# Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html   # useful for handling different item types with a single interface from itemadapter import ItemAdapter   class DushuPipeline:     def open_spider(self, spider):         self.fp = open('book.json', 'w', encoding='utf-8')      def process_item(self, item, spider):         self.fp.write(str(item))         return item      def close_spider(self, spider):         self.fp.close()  from scrapy.utils.project import get_project_settings import pymysql  class MysqlPipeline:      def open_spider(self, spider):         settings = get_project_settings()          self.host = settings['DB_HOST']         self.port = settings['DB_PORT']         self.user = settings['DB_USER']         self.password = settings['DB_PASSWORD']         self.name = settings['DB_NAME']         self.charset = settings['DB_CHARSET']          self.connect()      def connect(self):         self.conn = pymysql.connect(                 host=self.host,                 port=self.port,                 user=self.user,                 password=self.password,                 db=self.name,                 charset=self.charset         )         self.cursor = self.conn.cursor()      def process_item(self, item, spider):         sql = 'insert into book(name, src) values ("{}","{}")'.format(item['name'], item['src'])         self.cursor.execute(sql)         self.conn.commit()         return item      def close_spider(self, spider):         self.conn.close()         self.cursor.close() 

• ../dushu/dushu/settings.py

# Scrapy settings for dushu project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # #     https://docs.scrapy.org/en/latest/topics/settings.html #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #     https://docs.scrapy.org/en/latest/topics/spider-middleware.html  BOT_NAME = "dushu"  SPIDER_MODULES = ["dushu.spiders"] NEWSPIDER_MODULE = "dushu.spiders"   # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = "dushu (+http://www.yourdomain.com)"  # Obey robots.txt rules ROBOTSTXT_OBEY = True  # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32  # Configure a delay for requests for the same website (default: 0) # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16  # Disable cookies (enabled by default) #COOKIES_ENABLED = False  # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False  # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { #    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", #    "Accept-Language": "en", #}  # Enable or disable spider middlewares # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { #    "dushu.middlewares.DushuSpiderMiddleware": 543, #}  # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { #    "dushu.middlewares.DushuDownloaderMiddleware": 543, #}  # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { #    "scrapy.extensions.telnet.TelnetConsole": None, #}  DB_HOST = "127.0.0.1" DB_PORT = 3306 DB_USER = "root" DB_PASSWORD = "MySQL@4869" DB_NAME = "spider01" DB_CHARSET = "utf8"  # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = {    "dushu.pipelines.DushuPipeline": 300,    "dushu.pipelines.MysqlPipeline": 301, }  # Enable and configure the AutoThrottle extension (disabled by default) # See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False  # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = "httpcache" #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"  # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8"