From ea6dcd901b88585731c563a18ca750a2ff3bb152 Mon Sep 17 00:00:00 2001 From: Sveinung Rundhovde Date: Wed, 19 Sep 2018 15:28:24 +0200 Subject: [PATCH] Reduce memory usage when loading nexus plot file DataFrame was constructed from a list of dicts, using a lot of memory. Constructing frame column by column signifficantly reduces memory usage. --- python/nex/nex.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/nex/nex.py b/python/nex/nex.py index e88b37b..dee05a8 100644 --- a/python/nex/nex.py +++ b/python/nex/nex.py @@ -11,14 +11,16 @@ def load(filename): """ raw = _load(filename) - df = pandas.DataFrame([{ - 'timestep': d.timestep, - 'time': d.time, - 'classname': d.classname, - 'instancename': d.instancename, - 'varname': d.varname, - 'value': d.value, - } for d in raw._data]) + D = raw._data + df = pandas.DataFrame() + + df['timestep'] = [d.timestep for d in D] + df['time'] = [d.time for d in D] + df['classname'] = [d.classname for d in D] + df['instancename'] = [d.instancename for d in D] + df['varname'] = [d.varname for d in D] + df['value'] = [d.value for d in D] + df.start_date = datetime.datetime( year=raw._header.year, month=raw._header.month,