scrapy中怎么进行异步插入数据库mysql
程序员文章站
2022-05-10 19:30:59
...
import pymysql
from twisted.enterprise import adbapi
from pymysql import cursors
class MSQLPipeline(object):
@classmethod
def from_setting(cls,settings):
db_params = dict(
host=settings['MYSQL_HOST'],
user=settings['MYSQL_USER'],
password=settings['MYSQL_PASSWD'],
port=settings['MYSQL_PORT'],
db=settings['MYSQL_DBNAME'],
charset=settings['MYSQL_CHARSET'],
use_unicode= True,
cursorclass=cursors.Cursor
)
db_pool = adbapi.ConnectionPool('pymysql',**db_params)
return cls(db_pool)
def __init__(self,db_pool):
self.db_pool = db_pool
def process_item(self,item,spider):
query = self.db_pool.runTnteraction(self.insert_item,item)
query.addErrback(self.handler_error,item,spider)
return item
def handler_error(self,failure,item,spider):
print(failure)
def insert_item(self,cursor,item):
sql = "INSERT INTO jobs(job_name,org_name,job_location,max_money,min_money,date)VALUES (%s,%s,%s,%s,%s,%s)"
# 执行sql
cursor.execute(sql, (
item['job_name'], item['org_name'], item['job_location'], item['max_money'], item['min_money'], item['date']))
在pipeline中自定义一个类来实现,数据库的配置直接从settings中读取,可以通过修改settings中的配置来实现
代码的重复使用。
下一篇: 谢谢你的包容