Notes on `arrow` about rams HOT 2 CLOSED

library(arrow)
artab <- arrow_table(int=1:1000000, value=runif(1000000))

library(dplyr)
library(data.table) # for %between% operator syntax

req_vals <- data.frame(lower_bound=runif(200)) %>%
  mutate(upper_bound=lower_bound+0.0002)

eval_method <- function(){
  window_req_string <- req_vals %>%
    summarise(req_cmd=paste0("value%between%c(", lower_bound, ", ", upper_bound, 
                             ")", collapse="|")) %>%
    pull(req_cmd)
  full_arrow_req <- paste0(
    'artab %>% filter(', window_req_string, ') %>% dplyr::collect()'
  )
  eval(parse(text=full_arrow_req)) %>%
    arrange(int)
}

lapply_method <- function(){
  req_list <- split(req_vals, seq_len(nrow(req_vals))) %>%
    lapply(unlist)
  lapply(req_list, function(window) {
    artab %>%   
      filter(value%between%window) %>%   
      collect()
  }) %>% 
    bind_rows() %>%
    distinct() %>%
    arrange(int)
}

identical(arrow_output_eval, arrow_output_lapply)
microbenchmark::microbenchmark(eval_method(), lapply_method(), times = 3, check = "identical")