Update to new version of bgen-reader and fix pass all tests.

Author: CarlKCarlK

pysnptools/distreader/bgen.py

@@ -145,7 +145,16 @@ def _apply_iid_function(self,samples):
         if self._iid_function is not default_iid_function:
             return np.array([self._iid_function(sample) for sample in samples],dtype='str')
         else:
-            samples_np = np.stack(np.core.defchararray.split(samples,',',maxsplit=2))
+            try:
+                samples_np = np.stack(np.core.defchararray.split(samples,',',maxsplit=2))
+            except:
+                def split_and_fill(s):
+                    fields = s.split(',',2)
+                    if len(fields)==1:
+                        return ['0',fields[0]]
+                    else:
+                        return fields
+                samples_np = np.array([split_and_fill(sample) for sample in samples])
             if samples_np.shape[1]==1:
                 samples_np = np.stack([np.full(samples.shape,'0'),samples_np.reshape(-1)],axis=1)
             row = samples_np
@@ -173,14 +182,14 @@ def _run_once(self):
 
         assert os.path.exists(self.filename), "Expect file to exist ('{0}')".format(self.filename)
         #!!!cmkassert os.path.getsize(self.filename)<2**31, "For now, Python cannot access files larger than about 2G bytes (see https://github.com/limix/bgen-reader-py/issues/29)"
-        verbose = logging.getLogger().level >= logging.INFO
+        verbose = logging.getLogger().level <= logging.INFO
 
-        self._bgen = open_bgen(self.filename,self._sample,verbose)
-        self._row = self._apply_iid_function(self._bgen.samples)
-        self._col = self._apply_sid_function(self._bgen.ids,self._bgen.rsids)
+        self._open_bgen = open_bgen(self.filename,self._sample,verbose)
+        self._row = self._apply_iid_function(self._open_bgen.samples)
+        self._col = self._apply_sid_function(self._open_bgen.ids,self._open_bgen.rsids)
         self._col_property = np.zeros((len(self._col),3),dtype='float')
-        self._col_property[:,0] = self._bgen.chromosomes
-        self._col_property[:,2] = self._bgen.positions
+        self._col_property[:,0] = self._open_bgen.chromosomes
+        self._col_property[:,2] = self._open_bgen.positions
 
         self._assert_iid_sid_pos(check_val=False)
 
@@ -191,8 +200,8 @@ def _read(self, iid_index_or_none, sid_index_or_none, order, dtype, force_python
         if order=='A':
             order='F'
 
-        #cmk assert self._bgen.nalleles unique is 2, phased is all false, ploidy is 2
-        val = self._bgen.read((iid_index_or_none,sid_index_or_none),dtype=dtype,order=order)
+        #cmk assert self._open_bgen.nalleles unique is 2, phased is all false, ploidy is 2
+        val = self._open_bgen.read((iid_index_or_none,sid_index_or_none),dtype=dtype,order=order)
         return val
 
     def __repr__(self): 
@@ -214,8 +223,8 @@ def flush(self):
         '''
         if hasattr(self,'_ran_once') and self._ran_once:
             self._ran_once = False
-            if hasattr(self,'_bgen') and self._bgen is not None:  # we need to test this because Python doesn't guarantee that __init__ was fully run
-                del self._bgen
+            if hasattr(self,'_bgen') and self._open_bgen is not None:  # we need to test this because Python doesn't guarantee that __init__ was fully run
+                del self._open_bgen
 
     @staticmethod
     def write(filename, distreader, bits=16, compression=None, sample_function=default_sample_function, id_rsid_function=default_id_rsid_function, iid_function=default_iid_function, sid_function=default_sid_function, block_size=None, qctool_path=None, cleanup_temp_files=True):
@@ -272,7 +281,7 @@ def write(filename, distreader, bits=16, compression=None, sample_function=defau
         dir, file = os.path.split(filename)
         if dir=='':
             dir='.'
-        metadatanpz =  file+'.metadata.npz'
+        metadatanpz =  open_bgen._metadatapath_from_filename(file)
         samplefile =  os.path.splitext(file)[0]+'.sample'
         genfile =  os.path.splitext(file)[0]+'.gen'
         olddir = os.getcwd()
@@ -374,7 +383,7 @@ def copyinputs(self, copier):
         copier.input(self.filename)
         if self._sample is not None:
             copier.input(self._sample)
-        metadata2 = self.filename + ".metadata.npz"
+        metadata2 = open_bgen._metadatapath_from_filename(self.filename)
         if os.path.exists(metadata2):
             copier.input(metadata2)
 
@@ -520,8 +529,9 @@ def test_read1(self):
         pstutil.create_directory_if_necessary(file_to)
         if os.path.exists(file_to+".metadata"):
             os.remove(file_to+".metadata")
-        if os.path.exists(file_to+".metadata.npz"):
-            os.remove(file_to+".metadata.npz")
+        meta = open_bgen._metadatapath_from_filename(file_to)
+        if os.path.exists(meta):
+            os.remove(meta)
         shutil.copy(file_from,file_to)
 
         for loop_index in range(2):
@@ -542,12 +552,12 @@ def test_read1(self):
             bgen = Bgen(file_to,iid_function,sid_function=sid_function)
             assert bgen.sid[0]=='SNPID_2,RSID_2'
 
-        os.remove(file_to+".metadata.npz")
+        os.remove(bgen._open_bgen._metadatapath_from_filename(file_to))
         sid_function = lambda id,rsid: '{0},{1}'.format(id,rsid)
         bgen = Bgen(file_to,iid_function,sid_function=sid_function)
         assert bgen.sid[0]=='SNPID_2,RSID_2'
 
-        os.remove(file_to+".metadata.npz")
+        os.remove(bgen._open_bgen._metadatapath_from_filename(file_to))
         bgen = Bgen(file_to,iid_function,sid_function='rsid')
         assert np.array_equal(bgen.iid[0],['sample_001', 'sample_001'])
         assert bgen.sid[0]=='RSID_2'

pysnptools/distreader/test.py

@@ -165,13 +165,13 @@ def test_subset_view(self):
         result = distreader2.read(view_ok=True)
         self.assertFalse(distreader2 is result)
         result2 = result[:,:].read()
-        self.assertFalse(sp.may_share_memory(result2.val,result.val))
+        self.assertFalse(np.may_share_memory(result2.val,result.val))
         result3 = result[:,:].read(view_ok=True)
-        self.assertTrue(sp.may_share_memory(result3.val,result.val))
+        self.assertTrue(np.may_share_memory(result3.val,result.val))
         result4 = result3.read()
-        self.assertFalse(sp.may_share_memory(result4.val,result3.val))
+        self.assertFalse(np.may_share_memory(result4.val,result3.val))
         result5 = result4.read(view_ok=True)
-        self.assertTrue(sp.may_share_memory(result4.val,result5.val))
+        self.assertTrue(np.may_share_memory(result4.val,result5.val))
 
 
 
@@ -231,7 +231,7 @@ def test_writes(self):
                             shutil.rmtree(filename)
                         ret = writer(filename,distdata)
                         assert ret is not None
-                        for subsetter in [None, sp.s_[::2,::3]]:
+                        for subsetter in [None, np.s_[::2,::3]]:
                             reader = constructor(filename)
                             _fortesting_JustCheckExists().input(reader)
                             subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]]
@@ -479,11 +479,11 @@ def factory_iterator():
 
         for iid_index_list in [range(N_original), range(N_original//2), range(N_original - 1,0,-2)]:
             for snp_index_list in [range(snps_to_read_count), range(snps_to_read_count//2), range(snps_to_read_count - 1,0,-2)]:
-                reference_snps, reference_dtype = TestDistNaNCNC(iid_index_list, snp_index_list, snp_reader_factory_distnpz(), sp.float64, "C", "False", None, None).read_and_standardize()
+                reference_snps, reference_dtype = TestDistNaNCNC(iid_index_list, snp_index_list, snp_reader_factory_distnpz(), np.float64, "C", "False", None, None).read_and_standardize()
                 for distreader_factory in [snp_reader_factory_distnpz, 
                                             snp_reader_factory_snpmajor_hdf5, snp_reader_factory_iidmajor_hdf5
                                             ]:
-                    for dtype in [sp.float64,sp.float32]:
+                    for dtype in [np.float64,np.float32]:
                         for order in ["C", "F"]:
                             for force_python_only in [False, True]:
                                 distreader = distreader_factory()
@@ -529,7 +529,7 @@ def runTest(self, result = None):
         assert not np.array_equal(snps[0,0],snps[0,0]) #without SnpReader's standardization NaN's stay NaN's
         assert np.allclose(snps[0,1],[.1,.2,.7])
         if self.reference_snps is not None:
-            self.assertTrue(np.allclose(self.reference_snps, snps, rtol=1e-04 if dtype == sp.float32 or self.reference_dtype == sp.float32 else 1e-12,equal_nan=True))
+            self.assertTrue(np.allclose(self.reference_snps, snps, rtol=1e-04 if dtype == np.float32 or self.reference_dtype == np.float32 else 1e-12,equal_nan=True))
 
 
 # We do it this way instead of using doctest.DocTestSuite because doctest.DocTestSuite requires modules to be pickled, which python doesn't allow.
@@ -604,7 +604,7 @@ def getTestSuite():
     return test_suite
 
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=logging.WARN)
 
     if False:
         from pysnptools.snpreader import Bed

pysnptools/pstreader/test.py

@@ -105,7 +105,7 @@ def _zero(c):
                                 the_class.write(filename,pstdata)
                                 reader = the_class(filename) if suffix!='hdf5' else the_class(filename,block_size=3)
                                 _fortesting_JustCheckExists().input(reader)
-                                for subsetter in [None, sp.s_[::2,::3]]:
+                                for subsetter in [None, np.s_[::2,::3]]:
                                     subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]]
                                     expected = pstdata if subsetter is None else pstdata[subsetter[0],subsetter[1]].read()
                                     for order in ['C','F','A']:

pysnptools/pysnptools.pyproj

@@ -5,7 +5,7 @@
     <SchemaVersion>2.0</SchemaVersion>
     <ProjectGuid>{60f1cb4a-9da3-47c2-9b89-60ac1ce93347}</ProjectGuid>
     <ProjectHome />
-    <StartupFile>distreader\bgen.py</StartupFile>
+    <StartupFile>distreader\test.py</StartupFile>
     <SearchPath>..\;..\..\bgen-reader-py</SearchPath>
     <WorkingDirectory>.\snpreader</WorkingDirectory>
     <OutputPath>.</OutputPath>

pysnptools/test.py

@@ -376,7 +376,7 @@ def test_some_std(self):
 
         from pysnptools.standardizer.identity import Identity
         from pysnptools.standardizer.diag_K_to_N import DiagKtoN
-        for dtype in [sp.float64,sp.float32]:
+        for dtype in [np.float64,np.float32]:
             for std in [Unit(),Beta(1,25),Identity(),DiagKtoN()]:
                 s = str(std)
                 np.random.seed(0)
@@ -418,7 +418,7 @@ def standardize(self,snpreader):
         make sure blocked standardize yields same result as regular standardize
         """
 
-        for dtype in [sp.float64,sp.float32]:
+        for dtype in [np.float64,np.float32]:
 
             snps = snpreader.read(order='F',force_python_only=True,dtype=dtype).val
             self.assertEqual(dtype, snps.dtype)
@@ -562,13 +562,13 @@ def test_subset_view(self):
         result = snpreader2.read(view_ok=True)
         self.assertFalse(snpreader2 is result)
         result2 = result[:,:].read()
-        self.assertFalse(sp.may_share_memory(result2.val,result.val))
+        self.assertFalse(np.may_share_memory(result2.val,result.val))
         result3 = result[:,:].read(view_ok=True)
-        self.assertTrue(sp.may_share_memory(result3.val,result.val))
+        self.assertTrue(np.may_share_memory(result3.val,result.val))
         result4 = result3.read()
-        self.assertFalse(sp.may_share_memory(result4.val,result3.val))
+        self.assertFalse(np.may_share_memory(result4.val,result3.val))
         result5 = result4.read(view_ok=True)
-        self.assertTrue(sp.may_share_memory(result4.val,result5.val))
+        self.assertTrue(np.may_share_memory(result4.val,result5.val))
 
     def test_load_and_standardize_hdf5(self):
         snpreader2 = SnpHdf5(self.currentFolder + "/examples/toydata.snpmajor.snp.hdf5")
@@ -610,7 +610,7 @@ def load_and_standardize(self, snpreader2, snpreader3):
         iid_index_list = range(N_original - 1,0,-2)
         snpreader3 = snpreader3[iid_index_list,:]
 
-        for dtype in [sp.float64,sp.float32]:
+        for dtype in [np.float64,np.float32]:
 
             G2 = snpreader2.read(order='F',force_python_only=True).val
             G2 = Unit().standardize(G2, block_size=10000, force_python_only=True)
@@ -808,7 +808,7 @@ def test_writes(self):
                             shutil.rmtree(filename)
                         ret = writer(filename,snpdata)
                         assert ret is not None
-                        for subsetter in [None, sp.s_[::2,::3]]:
+                        for subsetter in [None, np.s_[::2,::3]]:
                             reader = constructor(filename)
                             _fortesting_JustCheckExists().input(reader)
                             subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]]
@@ -875,12 +875,12 @@ def factory_iterator():
         for iid_index_list in [range(N_original), range(N_original//2), range(N_original - 1,0,-2)]:
             for snp_index_list in [range(snps_to_read_count), range(snps_to_read_count//2), range(snps_to_read_count - 1,0,-2)]:
                 for standardizer in [Unit(),Beta(1,25)]:
-                    reference_snps, reference_dtype = NaNCNCTestCases(iid_index_list, snp_index_list, standardizer, snp_reader_factory_bed(), sp.float64, "C", "False", None, None).read_and_standardize()
+                    reference_snps, reference_dtype = NaNCNCTestCases(iid_index_list, snp_index_list, standardizer, snp_reader_factory_bed(), np.float64, "C", "False", None, None).read_and_standardize()
                     for snpreader_factory in [snp_reader_factory_bed, 
                                              snp_reader_factory_snpmajor_hdf5, snp_reader_factory_iidmajor_hdf5,
                                              snp_reader_factory_dat
                                               ]:
-                        for dtype in [sp.float64,sp.float32]:
+                        for dtype in [np.float64,np.float32]:
                             for order in ["C", "F"]:
                                 for force_python_only in [False, True]:
                                     snpreader = snpreader_factory()
@@ -929,7 +929,7 @@ def runTest(self, result = None):
         self.assertTrue(snps[0,0] == 0)
         self.assertTrue(np.all(snps[:,1] == 0))
         if self.reference_snps is not None:
-            self.assertTrue(np.allclose(self.reference_snps, snps, rtol=1e-04 if dtype == sp.float32 or self.reference_dtype == sp.float32 else 1e-12))
+            self.assertTrue(np.allclose(self.reference_snps, snps, rtol=1e-04 if dtype == np.float32 or self.reference_dtype == np.float32 else 1e-12))