From 112641787eae20252d35b8066923255a96c080f5 Mon Sep 17 00:00:00 2001
From: Sally <sally.matson@gmail.com>
Date: Thu, 9 Jan 2025 14:03:09 +0000
Subject: [PATCH] adding tests for corrupted files

---
 tests/core/data/garbage.csv       |   2 +
 tests/core/data/garbage.xlsx      | Bin 0 -> 10895 bytes
 tests/core/test_readers.py        |  71 +++++++++++++++++++++++++++++-
 virtual_ecosystem/core/readers.py |  19 +++++---
 4 files changed, 86 insertions(+), 6 deletions(-)
 create mode 100644 tests/core/data/garbage.csv
 create mode 100644 tests/core/data/garbage.xlsx

diff --git a/tests/core/data/garbage.csv b/tests/core/data/garbage.csv
new file mode 100644
index 000000000..c55856dbd
--- /dev/null
+++ b/tests/core/data/garbage.csv
@@ -0,0 +1,2 @@
+a,b,c
+1,2,"3
\ No newline at end of file
diff --git a/tests/core/data/garbage.xlsx b/tests/core/data/garbage.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..bd39652b1bcc79fce379dfadb6b99037085f9b9d
GIT binary patch
literal 10895
zcmeHN2UncenI4*efdfMmT{I1%4-AMRMDH^+(S<aC21YP26u}5Yx8<|hnAp2fa^f7t
zaam^_7bRYwxFzx0ImdQXENdmp@nWmkXG?Oj*6zt)*yow|`U7cw_76BZ&s*+$?|k3A
zZ@UAc)9T|JsoSFq(CKt3I{ROK_s+zSmre)itJ6j30=$pa&Ca+TGw$J@<vGWKE$_^<
zE19$28Q?l^eC)6PzQrCu5(wHM@(m5(&7e9^uI0|Jssj!}1H!Y8R6>04!;3GE9)sd^
zie0#&7BXtG1Uy_Ut7efC5Hr)FgI|2RD_9#q=^ctMF=+-PwCGr{`N5;$k@3|9vBSgA
zPJ)=zr$9hgQ9XE#1uSkNYA-jYp)U;T#ozEPgP<~P0!n)ySXk+YFF_b&A_8L4nYlV7
z!`v2MD5<MpdlzZzOd06Tw0A+92LV>l0&AxbG&1T9o(~d)XxbjpO7^uC#c4)S1}}bp
zDjVwB@wF8QK1<LkqBv?|IF~-I2BnA%T-P}z`d$e04A?rTDhjL>Bw%R+bYw(==<#DB
zg^#tzq)z8Rzw~r{M_EZ0v<;cL^6OG49iY*KH9j8j2l+B5fJhL1E2vfeOWD7!*+eO)
z^8`GEhD~O~9#3n6;-l-luXDlFo1X@bT5LsE3PS4|o_DwwOnKygZCT&q9`(S{kt9GH
zGc{i|6Sl0~xchxWt#zq}=9C`hcMB*~Td_a>?(YsH7<yQlK24k8ss?1mVQrzRz9KP%
zN0u`~m$Ydpr%wB4rL#3Os@?=OY;pGr=uMX83}s`%X<#n_(i%mwsj;l``r`uGmsr4i
zE5HEuvNkqFUh#kzsJ2^ljkEs(*Pq<nenM1z6=WIv!_D*T6btt8&tF-C;!;P8>+PN>
zV=w1(=MSYW50AN^0o)D1jjZMJb^y9ki^3sT$xR@(8DNu}qmQmWwG3Vo`2Gb5;L2IB
zB+=zd3G{EMd|A=&nPi*Cmzhum{fx`BG&?_KpPilJ*A`6lPBZGIbka`5GQNBSV?Z4r
zS~`da)Prmw0o@Ru)N<#wL~d9^>9pR(ckf&6o!Wj{%;UUqPPJF5n`_11#qhRGGt#u3
z1_5V4`J(3O4?xKUIr*?gw`;Ph(6M<)X2CysLsM%H59)6460)(STMS(e2=1x<gRRpe
zN4e`Ry-E%)FsuC*U{DMKO6zm~_|!WM%<zI3l@km{{pWAVf@5@2oah?2K%1Ce4nEF@
zkAmhcAHPD6mmIHB<?*S?oVo-1VV`B=)dzwJW&@Z|C+05w5W+pz18&U(8vrp=8rkxB
zYj6<2Y!IB_%~%H-V*r6U5O<R_#<T?b>Br4j8G!Qw<qiPu%x~3m<Ebj5hXBj=bE1sj
z`ibSBsGTCF`r4obfRu9>wVj;t%1+6H!Q}=MWQkaZh~}ZgHYUXao=>OK$O06cT!3WV
z0DRIg=*~kG;C>l*e#`6uj+-ZPAfiF~wA_dlv1)v%xI2Em*KoKC7GqvbKP5o70Q8`S
z#B8W5eKKyQBk6)#bI%bTGhXcA+-T^n=fPWecEELX7JL(wm}~++0_q=B38gw++&YXM
zolLI=5R-BtqlIlk;EaJxVFdt}e>wra+RhCuu^t2l?w7lqW#R>cc$^_nc~s0tOm^-%
zlr`us91Lh*;gp6Y)g@rybN}bbM`dcMMC7Lz?pqFH_*B|~8{dU;nC>?y*DONMwt)dE
z@G=+?;mN90=*|$ZID0oZ%ZBIBm4gB8AQfn;ZDtlWyW!<08A*YEEhM)J*!>u9*Xg=K
zh#;|*r|CoyK4oroS<|5>FW+|%<Fo}vT_K-D_Y2ro1f9&8nze+|a5^ZJ@k#`(f&>xU
zkFG;wKiUvW4aE2|E1v|=@0)kd)rGR0rY^lIR`8$}u%<IdZ4eF%>(Cj`;H_JWxu3|@
zYH(9C7%(QHNWV^Z=$T5*#n&JC_aFTtkjVwaXElO<@)+1YvG7iy3)vkQ84VX-p(&0J
z0hkj@9ioE=)qnYFnl?&HgleMWn?OVV`MtMgiVpxBplg38x@}k_lbd_u2a?hw^?ppE
z<ZkXpFpSGZv~ReKDz#;@3SUO!bOBztjO6?1?kj2MyQx<OKBmJZ_wD`%Cv>%0!Sk6&
zsuuE>pf5&+fxAsCzFtshz?IiCA=6U|RpQ2HVQ6DqttOvYh@I<$$#XCS%tv*_Z0JC^
z?@ELVvfPjB1wH|)@R@{7ei7l?43r=~WPuM+Va|aXSqcVZ7<aCpR2DpN^E*Y85c#Lg
zlM9fEAkWvZtFJ>7bpQ7+&uW8YXqjvW0@fa|N5nFV7>EsQIjmpaX%H#uIBdQFvgBkN
zPd_CC@L*5^$jVb4tYB|;^Ibqw(k30b@W<!QL(sQ?oBavkjDRN<>}*fQlIPGLtOhH+
z#nyrv*!0usVz5$uDO;FuINS>+k0?;z&am({OVJa1NS-PY^p{5;CfD3h+x!QP*xHIu
z2K1?WW`XwwGo=~?SVZ)Bdu|^LRGSgA@ZgXUDEWujk}^`gCT18bpiIPQgo?7qm+4~s
zM$YcT_U{eS1wODOgD&vXjQq~PyRe{7b?il}ZCsm8H#)-`RF;ZSMIwzMWDlxyf(^+W
z`x>9j#tsaA{F`iE=uLAnzWZ(vlp~3$S^PFz71|ljHUrFv5YF_?Y>y3X-wVZ8Y`rST
znJNLeVIYlnfK_yjn>Tjats%W#vqLd~^dJ=@ZHmF6>q9~5B?Ftsi*JAL&;b>p$qL(j
z58;W5$be-=8Hf>aI#%RS5h_RHG1BK0mC;I4CCrxrMiJn@f0%w}&Hxpl##XQ|o(5B)
zffm&r#?N<xBGlV-%uLU<PH)|PhF2zVM1;C0`enpZ7K^%@vs8~of(`GsicZ%LSG$hM
zNM+M1a}+SDW&`zPqnh*o`a04o5wi?jb6f7ImYf!7%3$he0`!`Dx-#;bp=IxpIpL96
z!EMS{U0F8e>2AzfTwu&UfNG5;5uC-63aZ`!)37txF!#V3$Ostn!03O!nJDlxAi(?R
zAkUi5o~2Xg3uLcu$p?$!o5Daz$zZ@|kC!lw&+Jsk1cr6&v^O3DYXSvoF3=QMGn-pG
zAlh=6cR+R@SW6qj*lPwW?C+vqTCYC@&h!of2`qZL&#00nf2eWNZkgn+m^!;L%-Vkd
z(T?c*mhpa<B@ebrSeYrzBt$bTmA`!VG)wn{W7=Wz@UQ#i;I+SJhSQP$AfOs#6BT^*
zMA?lON@U0;$(d%A1ne7g)eLQRq!%j_;DNoJN`Y9RFZR&4Dmw~B(p32!o^OQt-?p~E
z0lwG1WGa+Vg$Ut%|H5tq^q^F>5AH<ycVFri29ECrRyxwb8KOFFa10dN(F}f!5nrDN
z)8NC?1IZwU)(gb5@`H3@);LoLNEG3_acnMZl&#Xvd<<F(Tp8`tuS4aqpiMPUww21k
zoIgMIEEFb*AR~$WK>5I@F#TU%pR~8A7Q{~m$TA=9-|wVNH5v>8L0$Xo16i^_<JT{I
z@iyz5@iTO<j#=kNwYmH9C{r_wlHuBZ1loqr$YgZMq&8XUBWmnn=f&rt7EZ7Wv6My`
zUEt<Fh}2IOi=y!|Xk2|lEVB%%28OmAq$cCp24`h<b*IvEVDWN@IRv5-^xiozm4!01
z77fXw79@zsDJ^$HE5G)0@Hnw~`+fW>w7iPnpFJ?S(;;J{M|Bg)?{B3uDY|7THwZAV
zf%>{EW#4Bxg|RUh?1W+GNOKs^a=W1cc`Z}?6x&CzvyzYWZR-VJ)eVA&gMgAR|5-IF
z8G|p27Htw4lnBYQoB3fNg_jo%NiyEemv#?Ad9o{ygDKZ-%Iv3aX{%NS7~@QJI;j1_
zYLQ9Fq~Sxd+E@EfL|g=P?!w5D7(G)xs)wus30hiHTXO^H{~6F%=eFn&Ag+)R>TpF=
zvbr3{#R@*Ycl$LuDDt$JSX;RuR>k$d0n;(%j~jzJRLxXiH~?Y{xQK42Z8ua+SZrDT
zVX#kt%%BX={=g%AognZOON@PVaFNeo#ErvT7Td7i3c;$o@lurfL<&ZLX)3ygZ>4Mh
z<AoQmpTGCg-^xtmd%qH;W`kd?l<yg3uJ2+w6lAl(27rJDkSq5j*EOp>L!}EE7kK?y
z|8(vlh+KY9y&M{Ix_~8K+?By708E@Rllrc-X27WQ^gMV=57-7Dol;+*qUooi#lQ(v
zt7indEjdi2$}l>{w;!xWMc-j$RDli=`ygDRW{aqIIllfSB@BQ5rziR*lIVWG2x8^{
zYoHNPm4B2oQcn57?BlxhGK6}GnH~Kgh|S89&Axs42VlU02ypthR7Rh@0LcwA`v4x_
z?(XD@lOVp3eG_0-0Zvd;I6Vrig7{(@1?+HEIz&UsQMS`O8M|$e!J-H6hb$4@w{Qwd
z>Il;ZNTVP9lq<vpPNRT_1EQzF_AvW^l>;L0Z9Nl_qe8Ge30tVD1J;phz$T~(EM$fp
z*bo8EOaQu7<EK}0r)R88K@FZ?%gzR2eEaO5-hB3yOss43P*jV~9IOzl3MB?P8KdLe
z)#Y<V;7N2Y!&RW10@c~7V&VMy6eHezStJNZ7BMRVoOxfnnUQb%=y_<pCZk20f)dLs
zre~h_rL$5dMk#MSB?ItrdVPTI+3<;DI8|;i7t~g2pw8Caebolpcker_F@T!~#&&M8
z`YCpVxNtD0+s1%9AqR301oW1J=_Xl;z^aP9jJ97K7bVS~{x}ZG<RdP=4aRc*zX;Qb
z&DOD0w7oN*M`!Wb4WLqEP`0cBXWESU(sRoyfuUAK&2jN*5PMDo9bgB=>EpmPwA0P=
zKGqM<LmELm;~7<np5GQO!r%u2%<U<xAeB}w`pHzt57Ei=mNBYJ4*{4(Xk2A6A3UKM
zq0GMju#s^Z1F<i;)tyrd9oTS+y!Ymhq1bC!yn>tGg>)|lWma1ELK_zAZGLmvuLjqj
z!+_fpaO-TIJb(4-QoAuBNo67o%)<BI1k*58-mU3%&{C6F@QqBOE}39!1oRH8DAUuO
zP_8}`%cP$Yn_4OZI__VeP=Ra5z&Yq_P#N!%CG<8!V2=hHaENh51eYXo%S<4iMo=%}
zBc3)^bXBJq91+7RL;wq@gP*;-bI@@0dReCN@(+Fvy$CcLB1Wb$TQ<)I7}~@7bC~xh
z1Me_(%oR}=-|DWU=UW*h#x##FWVHb~>_EfJMxa+E{xVA<yjqOWeUPN1pwG{$GdE!1
z2V#Z^lw<kbK#%~zdT!%P!!g@5J<B)Ea3djCpT6~vCq`K#x87i~1AHJ*#{C2HCNM^&
zAH-teE*3!=hSm+md(;+UdcS((x5Lg7ZjcB3_(eXh5flm%RChG6eO`WdEe67AI40&Q
z5uX3Cy7ut)Qz}}p(zBiP25YujrKM*b$Jq&L$I&5K7@xTD3venTczEuH-Hgtt`0RCX
z4f~+wgQ_@p>2a|qs3wC^9B2qs3@W<)vyIXa9tPlfxG@bp%)){?40JlV>HqtAb1xQ*
zf$+<@e*9g9UVh$Oa4)+Y3)=seGHjmT$5levWS$5ZfgJW2tBgA#K72M2+<x^@aPIj5
ztadC8B^(mt21JLb%y}TqcJo4?9*P4r4>Bj%z+(z}zuLOc3k~r6v5Ifny%%4`;Kr7N
zWtO9+HW_Y!djgnU>?fUc91ju8(ec6-aE3%M7~hhZb8Slo`GZF{oAs=3pc%xR3gd%-
zBm+MQ1X^<AMFS54Ehg?&A-VfY42#{F;FDaVdg+AA{)eBiNeR+5txq&>W;NjXX#~;@
z8na9+*LsURr6Euqios?5gcZ21zOz=^!E*r(A~O8#-bcPX`}u1avS<L64NWo<&YEY#
z-``<nnbw4e8NA_(pMj69gE4s;m}1N-U}-W-6!2BixDhS|l0~Qj#(>S(z$V$uSEvv{
z6O1;}v(<c#6QuL^+otC<@H?2<&5sLM4Js^MQ1#kGrazm4zOOua5|RY8vr0AaO`mLi
z@}oW;yNiVgc!63$^ouW3k^OTT1T?ZNfZm<@pJRbO07kxw&AlpEn@LDA_{*R<2{O&Z
z*HHta=GMUtJJ%~)kpFuRv7Z8b=@7RC3~&E*lw&@fXQDg)>(}SO1-)x&Q0QWw^wU?w
z3V!8XFiQhxz7b3c6;ECTfJ&3;Fz$P<GLVVvscOizu>Ify`w_aUR0b6g-aMAtFSPD6
sw!-WY24@`vgeP@?2Zw(B=O;{M7NGO6O1BK<wtPLKixxDHjQ*qhUxO{}-2eap

literal 0
HcmV?d00001

diff --git a/tests/core/test_readers.py b/tests/core/test_readers.py
index 5c3bbdba7..323774974 100644
--- a/tests/core/test_readers.py
+++ b/tests/core/test_readers.py
@@ -4,6 +4,7 @@
 from logging import CRITICAL, DEBUG, INFO
 
 import pytest
+from pandas.errors import ParserError
 from xarray import DataArray
 
 from tests.conftest import log_check
@@ -93,7 +94,11 @@ def test_func():
     ],
 )
 def test_load_netcdf(shared_datadir, caplog, file, file_var, exp_err, expected_log):
-    """Test the netdcf variable loader."""
+    """Test the netdcf variable loader.
+
+    The FileNotFoundError test is dependent on running the test_file_format_loader tests
+    first, and will fail if run independently. See TODO above.
+    """
 
     from virtual_ecosystem.core.readers import load_netcdf
 
@@ -105,6 +110,70 @@ def test_load_netcdf(shared_datadir, caplog, file, file_var, exp_err, expected_l
     log_check(caplog, expected_log)
 
 
+@pytest.mark.parametrize(
+    argnames=["file", "file_var", "exp_err", "expected_log"],
+    argvalues=[
+        (
+            "not_there.csv",
+            "irrelevant",
+            pytest.raises(FileNotFoundError),
+            ((CRITICAL, "Data file not found"),),
+        ),
+        (
+            "garbage.csv",
+            "irrelevant",
+            pytest.raises(ParserError),
+            ((CRITICAL, "Could not load data from"),),
+        ),
+        (
+            "reader_test.csv",
+            "missing",
+            pytest.raises(KeyError),
+            ((CRITICAL, "Variable missing not found in"),),
+        ),
+        (
+            "reader_test.csv",
+            "var1",
+            does_not_raise(),
+            (),
+        ),
+        (
+            "garbage.xlsx",
+            "irrelevant",
+            pytest.raises(Exception),
+            ((CRITICAL, "Unidentified exception opening"),),
+        ),
+        (
+            "reader_test.xlsx",
+            "missing",
+            pytest.raises(KeyError),
+            ((CRITICAL, "Variable missing not found in"),),
+        ),
+        (
+            "reader_test.xlsx",
+            "var1",
+            does_not_raise(),
+            (),
+        ),
+    ],
+)
+def test_load_dataframe(shared_datadir, caplog, file, file_var, exp_err, expected_log):
+    """Test the netdcf variable loader.
+
+    The FileNotFoundError test is dependent on running the test_file_format_loader tests
+    first, and will fail if run independently. See TODO above.
+    """
+
+    from virtual_ecosystem.core.readers import load_from_dataframe
+
+    with exp_err:
+        darray = load_from_dataframe(shared_datadir / file, file_var)
+        assert isinstance(darray, DataArray)
+
+    # Check the error reports
+    log_check(caplog, expected_log)
+
+
 @pytest.mark.parametrize(
     argnames=[
         "filename",
diff --git a/virtual_ecosystem/core/readers.py b/virtual_ecosystem/core/readers.py
index 9fbb65322..19302d66f 100644
--- a/virtual_ecosystem/core/readers.py
+++ b/virtual_ecosystem/core/readers.py
@@ -35,6 +35,7 @@ def new_function_to_load_tif_data(...):
 from pathlib import Path
 
 from pandas import read_csv, read_excel
+from pandas.errors import ParserError
 from xarray import DataArray, load_dataset
 
 from virtual_ecosystem.core.logger import LOGGER
@@ -150,7 +151,11 @@ def load_from_dataframe(file: Path, var_name: str) -> DataArray:
 
     Raises:
         FileNotFoundError: with bad file path names.
-        ValueError: if the file data is not readable.
+        ParserError: if the csv data is not readable.
+        Exception: if the excel data is not readable.
+
+    Note: the general exception is used because of the variety of exceptions that are
+    possible with read_excel.
     """
 
     to_raise: Exception
@@ -161,13 +166,17 @@ def load_from_dataframe(file: Path, var_name: str) -> DataArray:
         if file_type == ".csv":
             dataset = read_csv(file)
         else:
-            dataset = read_excel(file)
+            dataset = read_excel(file, engine="openpyxl")
     except FileNotFoundError:
         to_raise = FileNotFoundError(f"Data file not found: {file}")
         LOGGER.critical(to_raise)
         raise to_raise
-    except ValueError as err:
-        to_raise = ValueError(f"Could not load data from {file}: {err}")
+    except ParserError as err:
+        to_raise = ParserError(f"Could not load data from {file}: {err}.")
+        LOGGER.critical(to_raise)
+        raise to_raise
+    except Exception as err:
+        to_raise = Exception(f"Unidentified exception opening {file}: {err}")
         LOGGER.critical(to_raise)
         raise to_raise
 
@@ -177,7 +186,7 @@ def load_from_dataframe(file: Path, var_name: str) -> DataArray:
         LOGGER.critical(to_raise)
         raise to_raise
 
-    return dataset[var_name]
+    return dataset[var_name].to_xarray()
 
 
 def load_to_dataarray(