1.使用pipeline进行批量导入数据。包含先使用rpush插入数据,然后使用expire改动过期时间
class Redis_Handler(Handler):
def connect(self):
#print self.host,self.port,self.table
self.conn = Connection(self.host,self.port,self.table)
def execute(self, action_name):
filename = "/tmp/temp.txt"
batch_size = 10000
with open(filename) as file:
try:
count = 0
pipeline_redis = self.conn.client.pipeline()
for lines in file:
(key,value) = lines.split(',')
count = count + 1
if len(key)>0:
pipeline_redis.rpush(key,value.strip())
if not count % batch_size:
pipeline_redis.execute()
count = 0
#send the last batch
pipeline_redis.execute()
except Exception:
print 'redis add error'