more efficient & accurate strong hierarchy

Author: Kieran Elmes

R/pint.R

@@ -112,23 +112,9 @@ interaction_lasso <- function(X, Y, n = dim(X)[1], p = dim(X)[2], lambda_min = -
     }
     rm(tmp)
 
-    if (approximate_hierarchy && depth > 1) {
-        if (verbose) {
-            print("Finding main effects")
-        }
-        main_only <- .Call(lasso_, X, Ym, lambda_min, lambda_max, halt_error_diff, max_interaction_distance, max_nz_beta*2, max_lambdas, verbose, log_filename, 1, log_level_enum, estimate_unbiased, use_intercept, num_threads, check_duplicates, continuous_X)
-        main_only_result <- process_result(X, main_only[[1]])
-        # we'll use these to allow the second run to report all identical columns again.
-        if (check_duplicates) {
-            all_equiv <- unique(unlist(main_only_result$main_effects$equivalent))
-        } else {
-            all_equiv <- unique(unlist(main_only_result$main_effects$i))
-        }
-
-        X <- X[,all_equiv]
-    }
 
-    result <- .Call(lasso_, X, Ym, lambda_min, lambda_max, halt_error_diff, max_interaction_distance, max_nz_beta, max_lambdas, verbose, log_filename, depth, log_level_enum, estimate_unbiased, use_intercept, num_threads, check_duplicates, continuous_X)
+
+    result <- .Call(lasso_, X, Ym, lambda_min, lambda_max, halt_error_diff, max_interaction_distance, max_nz_beta, max_lambdas, verbose, log_filename, depth, log_level_enum, estimate_unbiased, use_intercept, num_threads, check_duplicates, continuous_X, approximate_hierarchy)
 
     rm(Ym)
 

coverage-badge.svg

@@ -16,8 +16,8 @@ if (length(args >= 2)) {
   # f <- "../data/simulated_8k/n8000_p4000_SNR5_nbi40_nbij800_nlethals200_viol0_91159.rds"
   # f <- "../data/simulated_small_data/n1000_p100_SNR5_nbi0_nbij100_nlethals0_viol0_11754.rds"
   # f <- "../data/simulated_large_data/n10000_p1000_SNR10_nbi0_nbij1000_nlethals0_viol0_11504.rds"
-  f <- "../data/simulated_8k/n2000_p1000_SNR5_nbi10_nbij200_nlethals50_viol0_11057.rds"
-  # f <- "../data/simulated_8k/n8000_p4000_SNR5_nbi40_nbij800_nlethals200_viol0_78715.rds"
+  # f <- "../data/simulated_8k/n2000_p1000_SNR5_nbi10_nbij200_nlethals50_viol0_11057.rds"
+  f <- "../data/simulated_8k/n8000_p4000_SNR5_nbi40_nbij800_nlethals200_viol0_78715.rds"
   # f <- "../infx_lasso_data/3way_data_to_run/n1000_p100_SNR4_nbi10_nbij252_nbijk1666_nlethals0_70443.rds"
   # f <- "./weirdly_slow_case/n1000_p100_SNR10_nbi0_nbij100_nlethals0_viol0_33859.rds"
   # f <- "./antibio_data.rds"
@@ -43,7 +43,7 @@ Y <- d$Y
 # result <- interaction_lasso(X, Y, depth = 3)
 # result <- interaction_lasso(X, Y, depth = 2)
 # result <- interaction_lasso(X, Y, depth = 2, max_nz_beta = 150, estimate_unbiased = TRUE, num_threads = 4, verbose=TRUE, strong_hierarchy = TRUE, check_duplicates = TRUE, continuous_X = TRUE)
-result <- interaction_lasso(X, Y, depth = 2, max_nz_beta = 150, estimate_unbiased = TRUE, num_threads = 4, verbose=TRUE, approximate_hierarchy = FALSE, check_duplicates = TRUE, continuous_X = FALSE)
+result <- interaction_lasso(X, Y, depth = 2, max_nz_beta = 250, estimate_unbiased = TRUE, num_threads = 4, verbose=TRUE, approximate_hierarchy = TRUE, check_duplicates = TRUE, continuous_X = FALSE)
 # print(result)
 
 # q()

install_and_run.R

@@ -275,7 +275,7 @@ SEXP lasso_(SEXP X_, SEXP Y_, SEXP lambda_min_, SEXP lambda_max_,
     SEXP halt_error_diff_, SEXP max_interaction_distance_, SEXP max_nz_beta_,
     SEXP max_lambdas_, SEXP verbose_, SEXP log_filename_, SEXP depth_, SEXP log_level_,
     SEXP estimate_unbiased_, SEXP use_intercept_, SEXP use_cores_,
-    SEXP check_duplicates_, SEXP continuous_X_)
+    SEXP check_duplicates_, SEXP continuous_X_, SEXP use_hierarchy_)
 {
     double* x = REAL(X_);
     double* y = REAL(Y_);
@@ -293,6 +293,7 @@ SEXP lasso_(SEXP X_, SEXP Y_, SEXP lambda_min_, SEXP lambda_max_,
     int_fast64_t use_cores = asInteger(use_cores_);
     bool check_duplicates = asLogical(check_duplicates_);
     bool continuous_X = asLogical(continuous_X_);
+    bool use_hierarchy = asLogical(use_hierarchy_);
 
     VERBOSE = verbose;
 
@@ -359,7 +360,8 @@ SEXP lasso_(SEXP X_, SEXP Y_, SEXP lambda_min_, SEXP lambda_max_,
     Lasso_Result lasso_result = simple_coordinate_descent_lasso(
         xmatrix, Y, n, p, max_interaction_distance, asReal(lambda_min_),
         asReal(lambda_max_), max_lambdas, verbose, halt_error_diff, log_level, NULL, 0,
-        max_nz_beta, log_filename, depth, estimate_unbiased, use_intercept, check_duplicates, &ci);
+        max_nz_beta, log_filename, depth, estimate_unbiased, use_intercept, check_duplicates, &ci,
+        use_hierarchy);
     float final_lambda = lasso_result.final_lambda;
     float regularized_intercept = lasso_result.regularized_intercept;
     float unbiased_intercept = lasso_result.unbiased_intercept;
@@ -392,7 +394,7 @@ SEXP lasso_(SEXP X_, SEXP Y_, SEXP lambda_min_, SEXP lambda_max_,
 }
 
 static const R_CallMethodDef CallEntries[] = {
-    { "lasso_", (DL_FUNC)&lasso_, 17 },
+    { "lasso_", (DL_FUNC)&lasso_, 18 },
     { "read_log_", (DL_FUNC)&read_log_, 1 },
     { NULL, NULL, 0 }
 };

src/Pint.cpp

@@ -122,6 +122,13 @@ typedef struct {
     // int_fast64_t total_found_hash_count;
 } IndiCols;
 
+typedef struct {
+    robin_hood::unordered_flat_map<int_fast64_t, float> beta1;
+    robin_hood::unordered_flat_map<int_fast64_t, float> beta2;
+    robin_hood::unordered_flat_map<int_fast64_t, float> beta3;
+    int_fast64_t p;
+} Beta_Value_Sets;
+
 #include "s8b.h"
 #include "sparse_matrix.h"
 #include "tuple_val.h"
@@ -149,12 +156,6 @@ struct AS_Entry {
     std::vector<float> real_vals;
 };
 
-typedef struct {
-    robin_hood::unordered_flat_map<int_fast64_t, float> beta1;
-    robin_hood::unordered_flat_map<int_fast64_t, float> beta2;
-    robin_hood::unordered_flat_map<int_fast64_t, float> beta3;
-    int_fast64_t p;
-} Beta_Value_Sets;
 
 typedef struct {
     Beta_Value_Sets regularized_result;

src/liblasso.h

@@ -244,7 +244,7 @@ void subproblem_only(Iter_Vars* vars, float lambda, float* rowsum,
 int_fast64_t run_lambda_iters_pruned(Iter_Vars* vars, float lambda, float* rowsum,
     float* old_rowsum, Active_Set* active_set,
     int_fast64_t depth, const bool use_intercept, IndiCols* indi, const bool check_duplicates,
-    struct continuous_info* cont_inf)
+    struct continuous_info* cont_inf, const bool use_hierarchy)
 {
     XMatrixSparse Xc = vars->Xc;
     X_uncompressed Xu = vars->Xu;
@@ -354,7 +354,8 @@ int_fast64_t run_lambda_iters_pruned(Iter_Vars* vars, float lambda, float* rowsu
         clock_gettime(CLOCK_MONOTONIC_RAW, &start_time);
         auto working_set_results = update_working_set(vars->Xu, Xc, rowsum, wont_update, p, n, lambda,
             updateable_items, count_may_update, active_set,
-            thread_caches, last_max, depth, indi, &new_cols, max_interaction_distance, check_duplicates, cont_inf);
+            thread_caches, last_max, depth, indi, &new_cols, max_interaction_distance, check_duplicates, cont_inf,
+            use_hierarchy, beta_sets);
         bool increased_set = working_set_results.first;
         auto vals_to_remove = working_set_results.second;
         for (auto val : vals_to_remove) {
@@ -496,7 +497,7 @@ Lasso_Result simple_coordinate_descent_lasso(
     float hed, enum LOG_LEVEL log_level,
     const char** job_args, int_fast64_t job_args_num,
     int_fast64_t mnz_beta, const char* log_filename, int_fast64_t depth,
-    const bool estimate_unbiased, const bool use_intercept, const bool check_duplicates, struct continuous_info* cont_inf)
+    const bool estimate_unbiased, const bool use_intercept, const bool check_duplicates, struct continuous_info* cont_inf, const bool use_hierarchy)
 {
     const bool continuous_X = cont_inf->use_cont;
     int_fast64_t max_nz_beta = mnz_beta;
@@ -727,7 +728,7 @@ Lasso_Result simple_coordinate_descent_lasso(
         int_fast64_t last_iter_count = 0;
 
         nz_beta += run_lambda_iters_pruned(&iter_vars_pruned, lambda, rowsum,
-            old_rowsum, &active_set, depth, use_intercept, &indi, check_duplicates, cont_inf);
+            old_rowsum, &active_set, depth, use_intercept, &indi, check_duplicates, cont_inf, use_hierarchy);
 
         {
             int_fast64_t nonzero = beta_sets.beta1.size() + beta_sets.beta2.size() + beta_sets.beta3.size();

src/regression.cpp

@@ -30,7 +30,7 @@ Lasso_Result simple_coordinate_descent_lasso(
     float lambda_min, float lambda_max, int_fast64_t max_iter, const bool VERBOSE,
     float halt_beta_diff,
     enum LOG_LEVEL log_level, const char** job_args, int_fast64_t job_args_num,
-    int_fast64_t max_nz_beta, const char* log_filename, int_fast64_t depth, const bool estimate_unbiased, const bool use_intercept, const bool check_duplicates, struct continuous_info* cont_inf);
+    int_fast64_t max_nz_beta, const char* log_filename, int_fast64_t depth, const bool estimate_unbiased, const bool use_intercept, const bool check_duplicates, struct continuous_info* cont_inf, const bool use_hierarchy);
 float update_intercept_cyclic(float intercept, int_fast64_t** X, float* Y,
     robin_hood::unordered_flat_map<int_fast64_t, float>* beta, int_fast64_t n, int_fast64_t p);
 // Changes update_beta_cyclic(XMatrixSparse xmatrix_sparse, float *Y,

src/regression.h

@@ -37,8 +37,8 @@ void free_row_set(struct row_set rs)
 }
 
 struct row_set row_list_without_columns(XMatrixSparse Xc, X_uncompressed Xu,
-    bool* remove,
-    Thread_Cache* thread_caches, struct continuous_info* ci)
+    bool* remove, Thread_Cache* thread_caches, struct continuous_info* ci,
+    const bool use_hierarchy, Beta_Value_Sets* beta_sets)
 {
     int_fast64_t p = Xc.p;
     int_fast64_t n = Xc.n;
@@ -59,8 +59,10 @@ struct row_set row_list_without_columns(XMatrixSparse Xc, X_uncompressed Xu,
         for (int_fast64_t i = 0; i < Xu.host_row_nz[row]; i++) {
             int_fast64_t col = Xu.host_X_row[Xu.host_row_offsets[row] + i];
             if (!remove[col]) {
-                row_cache[row_pos] = col;
-                row_pos++;
+                if (!use_hierarchy || beta_sets->beta1.contains(col)) {
+                    row_cache[row_pos] = col;
+                    row_pos++;
+                }
             }
         }
 

src/sparse_matrix.cpp

@@ -58,7 +58,7 @@ XMatrixSparse sparse_X_from_X(int_fast64_t** X, int_fast64_t n, int_fast64_t p,
     int_fast64_t shuffle);
 XMatrixSparse sparsify_X(int_fast64_t** X, int_fast64_t n, int_fast64_t p);
 
-struct row_set row_list_without_columns(XMatrixSparse Xc, X_uncompressed Xu, bool* remove, Thread_Cache* thread_caches, struct continuous_info* ci);
+struct row_set row_list_without_columns(XMatrixSparse Xc, X_uncompressed Xu, bool* remove, Thread_Cache* thread_caches, struct continuous_info* ci, const bool use_hierarchy, Beta_Value_Sets* beta_sets);
 void free_row_set(struct row_set rs);
 X_uncompressed construct_host_X(XMatrixSparse* Xc);
 void free_host_X(X_uncompressed *Xu);

src/sparse_matrix.h

@@ -166,7 +166,7 @@ char update_working_set_cpu(struct XMatrixSparse Xc,
     X_uncompressed Xu, float* rowsum,
     bool* wont_update, int_fast64_t p, int_fast64_t n, float lambda,
     int_fast64_t* updateable_items, int_fast64_t count_may_update,
-    float* last_max, int_fast64_t depth, IndiCols* indicols, robin_hood::unordered_flat_set<int_fast64_t>* new_cols, int_fast64_t max_interaction_distance, const bool check_duplicates, struct continuous_info* ci)
+    float* last_max, int_fast64_t depth, IndiCols* indicols, robin_hood::unordered_flat_set<int_fast64_t>* new_cols, int_fast64_t max_interaction_distance, const bool check_duplicates, struct continuous_info* ci, const bool use_hierarchy)
 {
     int_fast64_t* host_X = Xu.host_X;
     int_fast64_t* host_col_nz = Xu.host_col_nz;
@@ -239,22 +239,28 @@ char update_working_set_cpu(struct XMatrixSparse Xc,
                 sum_with_col[main] += rowsum_diff; //TODO: slow
             if (depth > 1) {
                 int_fast64_t ri = 0;
-                int_fast64_t jump_dist = relevant_row_set.row_lengths[row_main]/2;
+                const int_fast64_t row_length = relevant_row_set.row_lengths[row_main];
                 const int_fast64_t* row = relevant_row_set.rows[row_main];
-                int_fast64_t tmp = row[ri];
-                while (tmp != main) {
-                    if (tmp < main)
-                        ri += jump_dist;
-                    else if (tmp > main)
-                        ri -= jump_dist;
-                    else if (tmp == main)
-                        break;
-                    jump_dist = std::max((int_fast64_t)1,jump_dist/2);
-                    tmp = row[ri];
+                if (use_hierarchy) {
+                    // we can't assume row_main is in the reduced row matrix.
+                    while (ri < row_length && row[ri] < main)
+                        ri++;
+                } else {
+                    int_fast64_t jump_dist = row_length/2;
+                    int_fast64_t tmp = row[ri];
+                    while (tmp != main) {
+                        if (tmp < main)
+                            ri += jump_dist;
+                        else if (tmp > main)
+                            ri -= jump_dist;
+                        else if (tmp == main)
+                            break;
+                        jump_dist = std::max((int_fast64_t)1,jump_dist/2);
+                        tmp = row[ri];
+                    }
+                    ri++;
                 }
-                ri++;
 
-                const int_fast64_t row_length = relevant_row_set.row_lengths[row_main];
                 for (; ri < row_length; ri++) {
                     int_fast64_t inter = relevant_row_set.rows[row_main][ri];
                     float inter_val = 1.0;
@@ -549,15 +555,15 @@ std::pair<bool, std::vector<int_fast64_t>> update_working_set(X_uncompressed Xu,
     Active_Set* as, Thread_Cache* thread_caches,
     float* last_max,
     int_fast64_t depth, IndiCols *indicols, robin_hood::unordered_flat_set<int_fast64_t>* new_cols,
-    int_fast64_t max_interaction_distance, bool check_duplicates, struct continuous_info* ci)
+    int_fast64_t max_interaction_distance, bool check_duplicates, struct continuous_info* ci, bool use_hierarchy, Beta_Value_Sets* beta_sets)
 {
-    struct row_set new_row_set = row_list_without_columns(Xc, Xu, wont_update, thread_caches, ci);
+    struct row_set new_row_set = row_list_without_columns(Xc, Xu, wont_update, thread_caches, ci, use_hierarchy, beta_sets);
     std::vector<int_fast64_t> vals_to_remove;
     if (check_duplicates)
         vals_to_remove = update_main_indistinguishable_cols(Xu, wont_update, new_row_set, indicols, new_cols, ci);
     char increased_set = update_working_set_cpu(
         Xc, new_row_set, thread_caches, as, Xu, rowsum, wont_update, p, n, lambda,
-        updateable_items, count_may_update, last_max, depth, indicols, new_cols, max_interaction_distance, check_duplicates, ci);
+        updateable_items, count_may_update, last_max, depth, indicols, new_cols, max_interaction_distance, check_duplicates, ci, use_hierarchy);
 
     free_row_set(new_row_set);
     return std::make_pair(increased_set, vals_to_remove);

src/update_working_set.cpp

@@ -11,7 +11,7 @@ std::pair<bool, std::vector<int_fast64_t>> update_working_set(
     float* rowsum, bool* wont_update, int_fast64_t p, int_fast64_t n,
     float lambda, int_fast64_t* updateable_items, int_fast64_t count_may_update, Active_Set* as,
     Thread_Cache* thread_caches,
-    float* last_max, int_fast64_t depth, IndiCols* indicols, robin_hood::unordered_flat_set<int_fast64_t>* new_cols, int_fast64_t max_interaction_distance, const bool check_duplicates, struct continuous_info* ci);
+    float* last_max, int_fast64_t depth, IndiCols* indicols, robin_hood::unordered_flat_set<int_fast64_t>* new_cols, int_fast64_t max_interaction_distance, const bool check_duplicates, struct continuous_info* ci, bool use_hierarchy, Beta_Value_Sets* beta_sets);
 
 void free_inter_cache(int_fast64_t p);
 //struct OpenCL_Setup setup_working_set_kernel(