Revision: 8527
Initial Code
Initial URL
Initial Description
Initial Title
Initial Tags
Initial Language
at September 26, 2008 12:10 by tkf
Initial Code
import numpy
import os
import cPickle as pickle
from datetime import datetime
def dt2sec(dt):
return dt.microseconds / 1000.0 + dt.seconds
data = [ { "a" : numpy.arange( 100, dtype=float ),
"b" : numpy.arange( 100, dtype=float ).reshape(10,10) }
for i in range(10)]
pickle_name = "data.pickle"
ff_dir = "fromfile"
os.mkdir( ff_dir )
#----- write data
print "write data"
t0 = datetime.now()
pickle.dump(data, file(pickle_name,"w"))
t1 = datetime.now()
print "pickle.dump :", dt2sec(t1-t0)
t0 = datetime.now()
for i,d in enumerate(data):
idir = os.path.join(ff_dir,str(i))
os.mkdir( idir )
for name, array in d.items():
# write to fromfile/{i}/{name}
array.tofile(os.path.join(idir,name))
t1 = datetime.now()
print "tofile :", dt2sec(t1-t0)
#----- read data
print "read data"
t0 = datetime.now()
pickle_data = pickle.load(file(pickle_name))
#print pickle_data
t1 = datetime.now()
print "pickle.load :", dt2sec(t1-t0)
t0 = datetime.now()
ff_data = []
for i in os.listdir(ff_dir):
idir = os.path.join(ff_dir,i)
tmp = {}
for name in os.listdir(idir):
tmp[name] = numpy.fromfile(os.path.join(idir,name))
ff_data.append(tmp)
#print ff_data
t1 = datetime.now()
print "fromfile :", dt2sec(t1-t0)
## write data
## pickle.dump : 9.328
## tofile : 2.946
## read data
## pickle.load : 15.423
## fromfile : 1.858
Initial URL
Initial Description
Initial Title
pickle(cPickle) vs numpy tofile/fromfile
Initial Tags
python
Initial Language
Python