Posted By

zwidnypublic on 11/25/14

Tagged

mysql

Versions (?)

Last Edited at 11/25/14 01:30pm

Statistics

Viewed 1444 times

Favorited by 0 user(s)

Related snippets

Scraped item to mysql in scrapy

/ Published in: Python

To store you item into mysql database.

1. I say you have been have mysql database and the table what field is the same in item class that you wang to be stored

2. I just ensure you item insert into mysql, do not ensure the performance

3. Todo

Abstract pipeline class

Highlighted TextExpand

// File: myscrapy/items.py
..

# -*- coding: utf-8 -*-

import scrapy    
    
    
class FundItem(scrapy.Item):                                                                                                        
    stock = scrapy.Field()  # è‚¡ç¥¨åç§°                                          
    deadline = scrapy.Field()  # æˆªæ¢æ—¥æœŸ                                       
    fund = scrapy.Field()  # åŸºé‡‘åç§°                                           
    code = scrapy.Field()  # åŸºé‡‘ä»£ç                                            
    count = scrapy.Field()  # æŒä»“æ•°é‡(è‚¡)                                      
    percentcir = scrapy.Field()  # å æµé€šè‚¡æ¯”ä¾‹(%)                              
    svalue = scrapy.Field()  # æŒè‚¡å¸‚å€¼ï¼ˆå…ƒï¼‰                                   
    percentnet = scrapy.Field()  # å å‡€å€¼æ¯”ä¾‹ï¼ˆ%ï¼‰

// File: myscrapy/pipelines.py
..
# -*- coding: utf-8 -*-

from twisted.enterprise import adbapi    
from scrapy.utils.project import get_project_settings    
    
settings = get_project_settings()

class FundPipeline(object):
    # The table you items.FundItem class map to, my table is named fund    
    insert_sql = """insert into fund (%s) values ( %s )"""    
    
    def __init__(self):    
        dbargs = settings.get('DB_CONNECT')    
        db_server = settings.get('DB_SERVER')    
        dbpool = adbapi.ConnectionPool(db_server, **dbargs)    
        self.dbpool = dbpool    
    
    def __del__(self):    
        self.dbpool.close()    
    
    def process_item(self, item, spider):    
        self.insert_data(item, self.insert_sql)    
        return item    
    
    def insert_data(self, item, insert):    
        keys = item.fields.keys()    
        fields = u','.join(keys)    
        qm = u','.join([u'%s'] * len(keys))    
        sql = insert % (fields, qm)    
        data = [item[k] for k in keys]
        return self.dbpool.runOperation(sql, data)

// File: myscrapy/settings.py
..
# -*- coding: utf-8 -*-
...

ITEM_PIPELINES = {    
    'myscrapy.pipelines.FundPipeline': 500,    
}

DB_SERVER = 'MySQLdb'            # For detail, please see twisted doc
DB_CONNECT = {    
    'db': 'finance',             # Your db   
    'user': 'root',              # 
    'passwd': 'root',            # 
    'host': '10.19.80.197',      # Your Server
    'charset': 'utf8',    
    'use_unicode': True,    
}   
...

Report this snippet Tweet

Comments

Subscribe to comments

Comment:

You need to login to post a comment.