Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make ssu and faithpd use the shared library #63

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Make ssu and faithpd use the shared library #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3c1b044
Add su::biom::load_n_samples and use in su.cpp
sfiligoi Jan 23, 2025
6838090
Remove unnecessary header includes
sfiligoi Jan 23, 2025
1efade6
Expose the partial functions in libssu
sfiligoi Jan 23, 2025
897ce3e
Make libssu trully thread-safe
sfiligoi Jan 24, 2025
817b7d7
Make ssu and faithpd use the shared library
sfiligoi Jan 24, 2025
518c319
Add missing header include
sfiligoi Jan 24, 2025
b45d9dc
Use explicit object list in rapi_test dependency
sfiligoi Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 11 additions & 17 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,20 @@ PLATFORM := $(shell uname -s)
COMPILER := $(shell ($(CXX) -v 2>&1) | tr A-Z a-z )

ifeq ($(PLATFORM),Darwin)
all: api main install test_binaries
all: api install main install_main test_binaries

else

# Note: important that all_nv is after all_cpu_basic and all_nv_avx2 for tests to work
all: all_cpu_basic all_nv_avx2 all_nv all_combined test_binaries_nv

all_cpu_basic: api_cpu_basic main_cpu_basic install_cpu_basic
all_cpu_basic: api_cpu_basic install_cpu_basic

all_nv: api_nv main_nv install_nv
all_nv: api_nv install_nv

all_nv_avx2: api_nv_avx2 main_nv_avx2 install_nv_avx2
all_nv_avx2: api_nv_avx2 install_nv_avx2

all_combined: api_combined install_combined
all_combined: api_combined install_combined main install_main

endif

Expand Down Expand Up @@ -50,28 +50,22 @@ api_combined:
main:
cd src && make main

main_cpu_basic:
export BUILD_VARIANT=cpu_basic ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make main

main_nv:
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make main

main_nv_avx2:
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv_avx2 ; export BUILD_FULL_OPTIMIZATION=True ; cd src && make main
install_main:
cd src && make install

########### install

install:
cd src && make install
cd src && make install_lib

install_cpu_basic:
export BUILD_VARIANT=cpu_basic ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make install
export BUILD_VARIANT=cpu_basic ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make install_lib

install_nv:
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make install
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv ; export BUILD_FULL_OPTIMIZATION=False ; cd src && make install_lib

install_nv_avx2:
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv_avx2 ; export BUILD_FULL_OPTIMIZATION=True ; cd src && make install
. ./setup_nv_h5.sh; export BUILD_VARIANT=nv_avx2 ; export BUILD_FULL_OPTIMIZATION=True ; cd src && make install_lib

install_combined:
cd combined && make install
Expand Down
4 changes: 1 addition & 3 deletions combined/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ libssu.so: libssu.o
$(CC) -shared -o libssu.so libssu.o -fPIC -ldl $(LDFLAGS)

install: libssu.so
rm -f ${PREFIX}/lib//libssu.so; cp libssu.so ${PREFIX}/lib/
rm -f ${PREFIX}/bin/ssu; cp ssu ${PREFIX}/bin/
rm -f ${PREFIX}/bin/faithpd; cp faithpd ${PREFIX}/bin/
rm -f ${PREFIX}/lib/libssu.so; cp libssu.so ${PREFIX}/lib/

clean:
rm -f libssu.o libssu.so
44 changes: 0 additions & 44 deletions combined/faithpd
View file
Open in desktop

This file was deleted.

146 changes: 107 additions & 39 deletions combined/libssu.c
View file
Open in desktop

Large diffs are not rendered by default.

47 changes: 0 additions & 47 deletions combined/ssu
View file
Open in desktop

This file was deleted.

21 changes: 12 additions & 9 deletions src/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.PHONY: all main api test test_binaries install clean rapi_test
.PHONY: all main api test test_binaries install install_lib clean rapi_test

CXX := h5c++

Expand Down Expand Up @@ -138,7 +138,7 @@ endif

CPPFLAGS += -Wall -std=c++17 -pedantic -I. $(OPT) -fPIC -L$(PREFIX)/lib

all: api main install
all: api install_lib main install

main: $(SSU) $(FPD)

Expand All @@ -153,11 +153,11 @@ test_ska: test_ska.cpp tree.o tsv.o test_su.cpp biom.o biom_inmem.o biom_subsamp
test_api: test_api.cpp tree.o tsv.o test_su.cpp biom.o biom_inmem.o biom_subsampled.o unifrac.o skbio_alt.o api.o $(UNIFRAC_FILES)
$(CXX) $(CPPFLAGS) $(EXEFLAGS) test_api.cpp -o test_api tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o $(UNIFRAC_FILES) unifrac.o skbio_alt.o api.o -llz4 $(BLASLIB) -lpthread

$(SSU): su.cpp tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES)
$(CXX) $(CPPFLAGS) $(EXEFLAGS) su.cpp -o $(SSU) tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lhdf5_cpp -llz4 $(BLASLIB) -lpthread
$(SSU): su.cpp api.hpp biom.hpp $(PREFIX)/lib/lib$(SSU).so
$(CXX) $(CPPFLAGS) $(EXEFLAGS) su.cpp -o $@ -l$(SSU)

$(FPD): faithpd.cpp tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES)
$(CXX) $(CPPFLAGS) $(EXEFLAGS) faithpd.cpp -o $(FPD) tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lhdf5_cpp -llz4 $(BLASLIB) -lpthread
$(FPD): faithpd.cpp api.hpp $(PREFIX)/lib/lib$(SSU).so
$(CXX) $(CPPFLAGS) $(EXEFLAGS) faithpd.cpp -o $@ -l$(SSU)

lib$(SSU).so: tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES)
$(CXX) $(LDDFLAGS) -o lib$(SSU).so tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lc -llz4 $(BLASLIB) -L$(PREFIX)/lib -noshlib -lhdf5_cpp -lhdf5_hl_cpp -lhdf5_hl -lhdf5
Expand Down Expand Up @@ -195,16 +195,19 @@ test: test_binaries
./test_ska
./test_api

install: lib$(SSU).so $(SSU) $(FPD)
rm -f ${PREFIX}/lib//lib$(SSU).so; cp lib$(SSU).so ${PREFIX}/lib/
install_lib: lib$(SSU).so
rm -f ${PREFIX}/lib/lib$(SSU).so; cp lib$(SSU).so ${PREFIX}/lib/


install: $(SSU) $(FPD)
rm -f ${PREFIX}/bin/$(SSU); cp $(SSU) ${PREFIX}/bin/
rm -f ${PREFIX}/bin/$(FPD); cp $(FPD) ${PREFIX}/bin/
mkdir -p ${PREFIX}/include/unifrac
rm -f ${PREFIX}/include/unifrac/task_parameters.hpp; cp task_parameters.hpp ${PREFIX}/include/unifrac/
rm -f ${PREFIX}/include/unifrac/api.hpp; cp api.hpp ${PREFIX}/include/unifrac/
rm -f ${PREFIX}/include/unifrac/status_enum.hpp; cp status_enum.hpp ${PREFIX}/include/unifrac/

rapi_test: main
rapi_test: tree.o biom.o biom_inmem.o biom_subsampled.o tsv.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES)
mkdir -p ~/.R
if [ -e ~/.R/Makevars ] ; \
then \
Expand Down
14 changes: 7 additions & 7 deletions src/api.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ EXTERN IOStatus write_vec(const char* filename, r_vec* result);
* unknown_method : the requested method is unknown.
*/

ComputeStatus partial(const char* biom_filename, const char* tree_filename,
EXTERN ComputeStatus partial(const char* biom_filename, const char* tree_filename,
const char* unifrac_method, bool variance_adjust, double alpha,
bool bypass_tips, unsigned int n_substeps, unsigned int stripe_start,
unsigned int stripe_stop, partial_mat_t** result);
Expand Down Expand Up @@ -750,7 +750,7 @@ ComputeStatus partial(const char* biom_filename, const char* tree_filename,
* ### FOOTER ###
* <MAGIC> : char, e.g., SSU-PARTIAL-01, same as starting magic
*/
IOStatus write_partial(const char* filename, const partial_mat_t* result);
EXTERN IOStatus write_partial(const char* filename, const partial_mat_t* result);

/* Read a partial matrix object
*
Expand Down Expand Up @@ -780,7 +780,7 @@ IOStatus read_partial(const char* filename, partial_mat_t** result);
* bad_header : header seems malformed
* unexpected_end : format end not found in expected location
*/
IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out);
EXTERN IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out);

/* Read a stripe of a partial matrix
*
Expand All @@ -796,12 +796,12 @@ IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** res
* bad_header : header seems malformed
* unexpected_end : format end not found in expected location
*/
IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx);
EXTERN IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx);

/*
* Description TBD
*/
MergeStatus validate_partial(const partial_dyn_mat_t* const * partial_mats, int n_partials);
EXTERN MergeStatus validate_partial(const partial_dyn_mat_t* const * partial_mats, int n_partials);

/* Merge partial results
*
Expand Down Expand Up @@ -863,7 +863,7 @@ MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int
* sample_id_consistency : samples described by stripes are inconsistent
* square_mismatch : inconsistency on denotation of square matrix
*/
MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result);
EXTERN MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result);

/* Merge partial results
*
Expand All @@ -879,7 +879,7 @@ MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int
* sample_id_consistency : samples described by stripes are inconsistent
* square_mismatch : inconsistency on denotation of square matrix
*/
MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result);
EXTERN MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result);


// Find eigen values and vectors
Expand Down
27 changes: 8 additions & 19 deletions src/biom.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,33 +15,22 @@
using namespace H5;
using namespace su;

/* datasets defined by the BIOM 2.x spec */
const std::string OBS_INDPTR = std::string("/observation/matrix/indptr");
const std::string OBS_INDICES = std::string("/observation/matrix/indices");
const std::string OBS_DATA = std::string("/observation/matrix/data");
const std::string OBS_IDS = std::string("/observation/ids");

const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr");
const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices");
const std::string SAMPLE_DATA = std::string("/sample/matrix/data");
const std::string SAMPLE_IDS = std::string("/sample/ids");

biom::biom(std::string filename)
: biom_inmem(true)
, has_hdf5_backing(true) {
file = H5File(filename.c_str(), H5F_ACC_RDONLY);

/* establish the datasets */
obs_indices = file.openDataSet(OBS_INDICES.c_str());
obs_data = file.openDataSet(OBS_DATA.c_str());
sample_indices = file.openDataSet(SAMPLE_INDICES.c_str());
sample_data = file.openDataSet(SAMPLE_DATA.c_str());
obs_indices = file.openDataSet(OBS_INDICES);
obs_data = file.openDataSet(OBS_DATA);
sample_indices = file.openDataSet(SAMPLE_INDICES);
sample_data = file.openDataSet(SAMPLE_DATA);

/* cache IDs and indptr */
load_ids(OBS_IDS.c_str(), obs_ids);
load_ids(SAMPLE_IDS.c_str(), sample_ids);
load_indptr(OBS_INDPTR.c_str(), obs_indptr);
load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr);
load_ids(OBS_IDS, obs_ids);
load_ids(SAMPLE_IDS, sample_ids);
load_indptr(OBS_INDPTR, obs_indptr);
load_indptr(SAMPLE_INDPTR, sample_indptr);

/* cache shape and nnz info */
n_samples = sample_ids.size();
Expand Down
24 changes: 24 additions & 0 deletions src/biom.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,17 @@ namespace su {
biom& operator= (const biom&) = delete;

private:
/* datasets defined by the BIOM 2.x spec */
static constexpr const char * OBS_INDPTR = "/observation/matrix/indptr";
static constexpr const char * OBS_INDICES = "/observation/matrix/indices";
static constexpr const char * OBS_DATA = "/observation/matrix/data";
static constexpr const char * OBS_IDS = "/observation/ids";

static constexpr const char * SAMPLE_INDPTR = "/sample/matrix/indptr";
static constexpr const char * SAMPLE_INDICES = "/sample/matrix/indices";
static constexpr const char * SAMPLE_DATA = "/sample/matrix/data";
static constexpr const char * SAMPLE_IDS = "/sample/ids";

bool has_hdf5_backing = false;

// cache both index pointers into both CSC and CSR representations
Expand Down Expand Up @@ -95,6 +106,19 @@ namespace su {
public:
uint32_t nnz; // the total number of nonzero entries

// used by ssu for helper messages
static inline size_t load_n_samples(const char* filename) {
const char* path = SAMPLE_IDS;
H5::H5File file(filename, H5F_ACC_RDONLY);
H5::DataSet ds_ids = file.openDataSet(path);
H5::DataSpace dataspace = ds_ids.getSpace();

hsize_t dims[1];
dataspace.getSimpleExtentDims(dims, NULL);

return dims[0];
}

// for unit testing
bool is_sample_indptr(const std::vector<uint32_t>& other) const { return sample_indptr==other; }
bool is_obs_indptr(const std::vector<uint32_t>& other) const { return obs_indptr==other; }
Expand Down
3 changes: 0 additions & 3 deletions src/faithpd.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
#include <iomanip>
#include "api.hpp"
#include "cmd.hpp"
#include "tree.hpp"
#include "biom.hpp"
#include "unifrac.hpp"


void usage() {
Expand Down
Loading
Loading