From 677d324291d532f1e9c322fe95a233a87b2c8ebb Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 15 Feb 2018 17:06:11 -0700 Subject: [PATCH 1/2] Bug fix release has more fool-proof set writing. --- CHANGES.txt | 1 + README.md | 2 +- gdxpds/__init__.py | 2 +- gdxpds/gdx.py | 30 +++++++++++++++++++++++++++++- gdxpds/test/test_details.py | 8 ++++---- setup.py | 2 +- 6 files changed, 37 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3e86418..50aeb60 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,4 @@ +v1.0.3, 02/15/18 -- fix up set 'Value' column when write to GDX v1.0.2, 01/19/18 -- instructions and additional filepaths for running pytest on installed package v1.0.1, 09/12/17 -- workaround bug fix for library conflicts between gdxcc and pandas diff --git a/README.md b/README.md index b66f81e..15b3ccf 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ pip install git+https://github.com/NREL/gdx-pandas.git@master or ```bash -pip install git+https://github.com/NREL/gdx-pandas.git@v1.0.2 +pip install git+https://github.com/NREL/gdx-pandas.git@v1.0.3 ``` Versions are listed at https://github.com/NREL/gdx-pandas/releases. diff --git a/gdxpds/__init__.py b/gdxpds/__init__.py index 67ddaea..0be20a1 100644 --- a/gdxpds/__init__.py +++ b/gdxpds/__init__.py @@ -43,7 +43,7 @@ ''' -__version__ = '1.0.2' +__version__ = '1.0.3' import logging import os diff --git a/gdxpds/gdx.py b/gdxpds/gdx.py index f5d9ce8..2393d55 100644 --- a/gdxpds/gdx.py +++ b/gdxpds/gdx.py @@ -51,6 +51,7 @@ from collections import MutableSequence import copy +from ctypes import c_bool from enum import Enum import logging from numbers import Number @@ -654,6 +655,9 @@ def dataframe(self, data): self._dataframe.columns = self.dims + self.value_col_names else: self._dataframe = pds.DataFrame(data,columns=self.dims + self.value_col_names) + + if self.data_type == GamsDataType.Set: + self._fixup_set_value() return def _init_dataframe(self): @@ -665,10 +669,31 @@ def _init_dataframe(self): cols = self._dataframe.columns tmpcols = [col if col != '*' else 'aaa' for col in cols ] self._dataframe.columns = tmpcols - self._dataframe[colname] = self._dataframe[colname].astype(bool) + self._dataframe[colname] = self._dataframe[colname].astype(c_bool) self._dataframe.columns = cols return + def _fixup_set_value(self): + """ + Tricky to get boolean set values to come through right. + isinstance(True,Number) == True and float(True) = 1, but + isinstance(c_bool(True),Number) == False, and this keeps the default + value of 0.0. + + Could just test for isinstance(,bool), but this fix has the added + advantage of speaking the GDX bindings data type language, and also + fills in any missing values, so users no longer need to actually specify + self.dataframe['Value'] = True. + """ + assert self.data_type == GamsDataType.Set + + colname = self._dataframe.columns[-1] + assert colname == self.value_col_names[0], "Unexpected final column in Set dataframe" + self._dataframe[colname].fillna(value=True,inplace=True) + # self._dataframe[colname].astype(c_bool,copy=False) + self._dataframe[colname] = self._dataframe[colname].apply(lambda x: c_bool(x)) + return + @property def num_records(self): if self.loaded: @@ -728,6 +753,9 @@ def write(self,index=None): if not self.loaded: raise Error("Cannot write unloaded symbol {}.".format(repr(self.name))) + if self.data_type == GamsDataType.Set: + self._fixup_set_value() + if index is not None: self._index = index diff --git a/gdxpds/test/test_details.py b/gdxpds/test/test_details.py index d1c1e06..e122307 100644 --- a/gdxpds/test/test_details.py +++ b/gdxpds/test/test_details.py @@ -39,6 +39,7 @@ [/LICENSE] ''' +from ctypes import c_bool import logging import os import subprocess as subp @@ -158,12 +159,11 @@ def test_from_scratch_sets(manage_rundir): data = pds.DataFrame([['u' + str(i)] for i in range(1,11)]) data['Value'] = True gdx[-1].dataframe = data - assert gdx[-1].dataframe[gdx[-1].dataframe.columns[-1]].dtype == bool + assert isinstance(gdx[-1].dataframe[gdx[-1].dataframe.columns[-1]].values[0], c_bool) gdx.append(gdxpds.gdx.GdxSymbol('my_other_set',gdxpds.gdx.GamsDataType.Set,dims=['u'])) data = pds.DataFrame([['u' + str(i)] for i in range(1,11)],columns=['u']) data['Value'] = True - gdx[-1].dataframe = gdx[-1].dataframe.append(data) - assert gdx[-1].dataframe[gdx[-1].dataframe.columns[-1]].dtype == bool + gdx[-1].dataframe = gdx[-1].dataframe.append(data) gdx.write(os.path.join(outdir,'my_sets.gdx')) with gdxpds.gdx.GdxFile(lazy_load=False) as gdx: gdx.read(os.path.join(outdir,'my_sets.gdx')) @@ -172,4 +172,4 @@ def test_from_scratch_sets(manage_rundir): assert sym.dims[0] == 'u' assert sym.data_type == gdxpds.gdx.GamsDataType.Set assert sym.num_records == 10 - assert sym.dataframe[sym.dataframe.columns[-1]].dtype == bool + assert isinstance(sym.dataframe[sym.dataframe.columns[-1]].values[0],c_bool) diff --git a/setup.py b/setup.py index 340e264..f664faa 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name = 'gdxpds', - version = "1.0.2", + version = "1.0.3", author = 'Elaine T. Hale', author_email = 'elaine.hale@nrel.gov', packages = ['gdxpds', 'gdxpds.test'], From 7a6d4357f4dd3cb79becc51f2e37e9e8a7fadc1f Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 15 Feb 2018 17:08:40 -0700 Subject: [PATCH 2/2] removing unneeded line of code --- gdxpds/gdx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gdxpds/gdx.py b/gdxpds/gdx.py index 2393d55..fad41a7 100644 --- a/gdxpds/gdx.py +++ b/gdxpds/gdx.py @@ -690,7 +690,6 @@ def _fixup_set_value(self): colname = self._dataframe.columns[-1] assert colname == self.value_col_names[0], "Unexpected final column in Set dataframe" self._dataframe[colname].fillna(value=True,inplace=True) - # self._dataframe[colname].astype(c_bool,copy=False) self._dataframe[colname] = self._dataframe[colname].apply(lambda x: c_bool(x)) return