Merge branch 'f-#6-combine'

- Combine stack traces with the help of the profile package.

.Rbuildignore

@@ -3,8 +3,6 @@
 ^CMakeLists\.txt$
 ^cmake-build-debug$
 ^clion-test\.R$
-^.*\.prof$
-^.*\.prof\.out$
 ^README\.Rmd$
 ^README-.*\.png$
 ^\.travis\.yml$
@@ -16,3 +14,18 @@
 ^src/go/src/\.idea$
 ^src/main\..*$
 ^_pkgdown\.yml$
+^.*\.out$
+^configure\.ac$
+^m4$
+^config\.log$
+^autom4te\.cache$
+^aclocal\.m4$
+^ltmain\.sh$
+^compile$
+^config\.guess$
+^config\.sub$
+^install-sh$
+^missing$
+^config\.status$
+^src/stamp-h1$
+^src/config\.h$

.gitignore

@@ -4,9 +4,20 @@
 /.idea
 clion-test.R
 cmake-build-debug
-*.prof
-*.prof.out
 inst/doc
 src/pprof
 src/go/src/.idea/
 src/main.*
+/Rprof.out
+/config.log
+/autom4te.cache
+/aclocal.m4
+/ltmain.sh
+/compile
+/config.guess
+/config.sub
+/install-sh
+/missing
+/config.status
+/src/stamp-h1
+/src/config.h

.travis.yml

@@ -41,13 +41,19 @@ jobs:
       apt:
         packages:
         - libgoogle-perftools-dev
+        - libprotoc-dev
+        - libprotobuf-dev
+        - protobuf-compiler
     env:
     - KIND=apt
     - BUILD_PKGDOWN=true
   - addons:
       apt:
         packages:
         - libunwind8-dev
+        - libprotoc-dev
+        - libprotobuf-dev
+        - protobuf-compiler
     env:
     - KIND=github
     before_install:

DESCRIPTION

@@ -3,7 +3,7 @@ Title: Joint Profiling of Native and R Code
 Version: 0.0-1
 Authors@R: c(
       person("Kirill", "Müller", role = c("aut", "cre"), email = "[email protected]"),
-      person("Google, Inc.", role = "cph", comment = "Bundled pprof sources")
+      person("Google, Inc.", role = "cph", comment = "Bundled pprof sources, and code extracted from gperftools")
     )
 Description: Simultaneously collects profiling data at both R and native levels
     (using 'Rprof' and 'gprofiler') and provides a unified view of the collected data. Work in progress.
@@ -15,11 +15,16 @@ BugReports: https://github.com/r-prof/gprofiler/issues
 URL: https://github.com/r-prof/gprofiler, http://r-prof.github.io/gprofiler
 LinkingTo: Rcpp
 Imports:
-    dplyr,
+    profile,
     profvis,
-    tidyr,
-    Rcpp
-SystemRequirements: C++11
+    rlang,
+    tibble,
+    Rcpp,
+    RProtoBuf
+Remotes:
+    r-prof/profile
+SystemRequirements: C++11, Linux (OS X and Solaris not supported), Go, libprofiler (libgoogle-perftools-dev on Ubuntu)
+OS_type: unix
 Suggests: 
     DBI,
     knitr,

NAMESPACE

@@ -3,10 +3,10 @@
 export(callback1_r)
 export(callback2_r)
 export(callback3_r)
-export(get_profiler_traces)
 export(show_profiler_pdf)
 export(start_profiler)
 export(stop_profiler)
+import(rlang)
+importFrom(RProtoBuf,FileInputStream)
 importFrom(Rcpp,sourceCpp)
-importFrom(tidyr,"%>%")
 useDynLib(gprofiler,.registration = TRUE)

R/combine.R

@@ -0,0 +1,130 @@
+combine_profiles <- function(path, prof_path, out_path) {
+  ds_rprof <- profile::read_rprof(out_path)
+
+  proto_path <- tempfile("gprofiler", fileext = ".pb.gz")
+  system2(
+    get_pprof_path(),
+    c("-proto", "-output", shQuote(proto_path), shQuote(prof_path))
+  )
+  ds_pprof <- profile::read_pprof(proto_path)
+
+  stopifnot(sum(ds_pprof$samples$value) == sum(ds_rprof$samples$value))
+  stopifnot(ds_rprof$samples$value == 1)
+
+  ds_pprof <- shift_ids(ds_pprof, ds_rprof)
+  ds_rprof <- expand_samples(ds_rprof)
+  ds_pprof <- expand_samples(ds_pprof)
+
+  ds_combined <- combine_ds(ds_rprof, ds_pprof)
+  ds_merged <- patch_combined_ds(ds_combined)
+  ds_pruned <- prune_ds(ds_merged)
+
+  profile::write_rprof(ds_pruned, path)
+}
+
+shift_ids <- function(ds, ds_base) {
+  d_location_id <- max(ds_base$locations$location_id)
+  d_function_id <- max(ds_base$functions$function_id)
+
+  ds$samples$locations <- map(ds$samples$locations, function(x) {
+    x$location_id <- x$location_id + d_location_id
+    x
+  })
+  ds$locations$location_id <- ds$locations$location_id + d_location_id
+
+  ds$locations$function_id <- ds$locations$function_id + d_function_id
+  ds$functions$function_id <- ds$functions$function_id + d_function_id
+
+  profile::validate_profile_v1(ds)
+  ds
+}
+
+expand_samples <- function(ds) {
+  new_samples <- tibble::tibble(
+    value = 1L,
+    locations = rep(ds$samples$locations, ds$samples$value)
+  )
+
+  ds$samples <- new_samples
+  profile::validate_profile_v1(ds)
+  ds
+}
+
+combine_ds <- function(ds_rprof, ds_pprof) {
+  ds_rprof$samples$.pprof_locations <- ds_pprof$samples$locations
+  ds_rprof$locations <- tibble::as_tibble(rbind(
+    strip_dots(ds_rprof$locations), strip_dots(ds_pprof$locations)
+  ))
+  ds_rprof$functions <- tibble::as_tibble(rbind(
+    strip_dots(ds_rprof$functions), strip_dots(ds_pprof$functions)
+  ))
+  profile::validate_profile_v1(ds_rprof)
+  ds_rprof
+}
+
+strip_dots <- function(x) {
+  x[grep("^[.]", names(x))] <- NULL
+  x
+}
+
+patch_combined_ds <- function(ds_combined) {
+  ds_combined$samples$.rprof_locations <- ds_combined$samples$locations
+
+  . <- merge(ds_combined$locations, ds_combined$functions, by = "function_id", all.x = TRUE)
+  . <- tibble::as_tibble(.)
+  locations_flat <- .
+
+  ds_combined$samples$locations <- map2(
+    ds_combined$samples$.rprof_locations,
+    ds_combined$samples$.pprof_locations,
+    patch_locations,
+    list(locations_flat)
+  )
+  ds_combined
+}
+
+patch_locations <- function(rprof_locations, pprof_locations, locations_flat) {
+  . <- rprof_locations
+  . <- tibble::rowid_to_column(., "rprof_id")
+  . <- merge(., locations_flat, by = "location_id", sort = FALSE)
+  . <- .[order(.$rprof_id), ]
+  . <- tibble::as_tibble(., rownames = NULL)
+  rprof_locations_full <- .
+  stopifnot(rprof_locations$location_id == rprof_locations_full$location_id)
+
+  call_idx <- which(rprof_locations_full$name == ".Call")
+
+  if (length(call_idx) == 0) {
+    return(rprof_locations)
+  }
+
+  call_idx <- call_idx[[1]]
+
+  . <- pprof_locations
+  . <- tibble::rowid_to_column(., "pprof_id")
+  . <- merge(., locations_flat, by = "location_id", sort = FALSE)
+  . <- .[order(.$pprof_id), ]
+  . <- tibble::as_tibble(., rownames = NULL)
+  pprof_locations_full <- .
+  stopifnot(pprof_locations$location_id == pprof_locations_full$location_id)
+
+  eval_idx <- which(pprof_locations_full$name == "Rf_eval")
+  if (length(eval_idx) == 0) {
+    eval_idx <- length(pprof_locations_full$name)
+  } else {
+    eval_idx <- max(eval_idx[[1L]] - 1L, 1L)
+    if (pprof_locations_full$name[[eval_idx]] == "<?>") eval_idx <- eval_idx - 1L
+  }
+
+  tibble::tibble(
+    location_id = c(
+      pprof_locations_full$location_id[seq2(1L, eval_idx)],
+      rprof_locations_full$location_id[seq2(call_idx + 1L, nrow(rprof_locations_full))]
+    )
+  )
+}
+
+prune_ds <- function(ds) {
+  # FIXME: prune unused locations and functions
+  ds
+}