Scrapy通过sqlite3保存数据 -云博客

Scrapy通过sqlite3保存数据

# 数据库 2024-05-05 22:57 0 139 来源：云博客

以爬取当当网作为实例 http://bj.ganji.com/fang1/chaoyang/

通过xpath获取title和price

分别贴出spider, items, pipelines的code

 1 # -*- coding: utf-8 -*- 2 import scrapy 3 from ..items import RenthouseItem 4  5 class GanjiSpider(scrapy.Spider): 6 name = ‘ganji‘ 7 # allowed_domains = [‘bj.ganji.com‘] 8 start_urls = [‘http://bj.ganji.com/fang1/chaoyang/‘] 9 10 def parse(self, response):11 #print(response)12 rh = RenthouseItem()13 title_list = response.xpath(‘//*[@class="f-list-item ershoufang-list"]/dl/dd[1]/a/text()‘).extract()14 price_list = response.xpath(‘//*[@class="f-list-item ershoufang-list"]/dl/dd[5]/div[1]/span[1]/text()‘).extract()15 # d = {}16 for i, j in zip(title_list, price_list):17 rh[‘title‘] = i18 rh[‘price‘] = j19 yield rh20 # d[‘title‘] = i21 # d[‘price‘] = j22 # yield d23 # print(i, ‘:‘, j)

 1 # -*- coding: utf-8 -*- 2  3 # Define here the models for your scraped items 4 # 5 # See documentation in: 6 # https://doc.scrapy.org/en/latest/topics/items.html 7  8 import scrapy 9 10 11 class RenthouseItem(scrapy.Item):12 # define the fields for your item here like:13 # name = scrapy.Field()14 title = scrapy.Field()15 price = scrapy.Field()16 # pass

 1 # -*- coding: utf-8 -*- 2  3 # Define your item pipelines here 4 # 5 # Don‘t forget to add your pipeline to the ITEM_PIPELINES setting 6 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 import sqlite3 8  9 class RenthousePipeline(object):10 def open_spider(self, spider):11 self.con = sqlite3.connect(‘renthouse.sqlite‘)12 self.cu = self.con.cursor() 13 14 def process_item(self, item, spider):15 #print(spider.name)16 insert_sql = ‘insert into renthouse (title, price) values ("{}", "{}")‘.format(item[‘title‘], item[‘price‘])17 #print(insert_sql)18  self.cu.execute(insert_sql)19  self.con.commit()20 return item21 22 def spider_close(self, spider):23 self.con.close()