scrapy 使用postgres异步存储代码

scrapy 使用postgres异步存储代码

# -*- coding: utf-8 -*-# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: twisted.internet import reactorfrom twisted.python import logfrom txpostgres import txpostgresimport timeclass Jianshu2Pipeline(object): def __init__(self): self.conn = txpostgres.Connection() parm = dict( database="test", user="postgres", password="postgres", host="localhost", port="5432" ) self.d = self.conn.connect(**parm) @property def _sql(self): return "insert into jianshu(title, url) values (%s,%s);" def process_item(self, item, spider): # run the interaction, making sure that if the insert fails, the table won't be # left behind created but empty tuple_parm = (item.get('title')[0], item.get('url')) # print('item:{}'.format(item)) # print('parm:{}'.format(tuple_parm)) # print('call process_item') self.d.addCallback(lambda _: self.conn.runInteraction(self.interaction, self._sql, tuple_parm)) # close the connection, log any errors and stop the reactor return item def interaction(self, cur, sql, tuple_parm): """ A callable that will execute inside a transaction. """ # the parameter is a txpostgres Cursor self.d = cur.execute(sql, tuple_parm) # self.d.addCallback(lambda _: cur.execute('insert into test values (%s)', (1,))) # self.d.commit() print('call interaction') # yield self.d time.sleep(1) return self.d def close_spider(self, spider): print('call close_spider') # self.d.addCallback(lambda _: self.conn.close()) # self.d.addErrback(log.err) # self.d.addBoth(lambda _: reactor.stop()) pass # spider (Spider 对象) – 被关闭的spider # 可选实现，当spider被关闭时，这个方法被调用

懂得,原来世界如此简单!

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。