|
5 | 5 | #' By default, `bind_tweets` binds into a data frame containing tweets (from data_*id*.json files). |
6 | 6 | #' |
7 | 7 | #' If users is TRUE, it binds into a data frame containing user information (from users_*id*.json). |
| 8 | +#' |
| 9 | +#' For the "tidy" format, parallel processing with furrr is supported. In order to enable parallel processing, workers need to be set manually through [future::plan()]. See examples |
8 | 10 | #' |
9 | 11 | #' @param data_path string, file path to directory of stored tweets data saved as data_*id*.json and users_*id*.json |
10 | 12 | #' @param user If `FALSE`, this function binds JSON files into a data frame containing tweets; data frame containing user information otherwise. Ignore if `output_format` is not NA |
|
16 | 18 | #' \item{"tidy"}{Tidy format; all essential columns are available} |
17 | 19 | #' } |
18 | 20 | #' |
19 | | -#' @param parallel_workers Number of threads used for parallel processing. Defaults to all detected threads. Only supported if the output_format is "tidy" |
20 | | -#' @param auto_set_plan Should the parallelization plan be set automatically? If True, the function automatically sets up (and ends) a multisession with the specified amount of parallel workers. If False, a future::plan() session needs to be set up manually for parallelization |
21 | | -#' |
22 | 21 | #' @return a data.frame containing either tweets or user information |
23 | 22 | #' @export |
24 | 23 | #' |
|
32 | 31 | #' |
33 | 32 | #' # bind json files in the directory "data" into a "tidy" data frame / tibble |
34 | 33 | #' bind_tweets(data_path = "data/", user = TRUE, output_format = "tidy") |
| 34 | +#' |
| 35 | +#' # bind json files in the directory "data" into a "tidy" data frame / tibble with parallel computing |
| 36 | +#' ## set up a multisession |
| 37 | +#' future::plan("multisession") |
| 38 | +#' ## run the function - note that no additional arguments are required |
| 39 | +#' bind_tweets(data_path = "data/", user = TRUE, output_format = "tidy") |
| 40 | +#' ## Shut down parallel workers |
| 41 | +#' future::plan("sequential") |
35 | 42 | #' } |
36 | | -bind_tweets <- function(data_path, user = FALSE, verbose = TRUE, output_format = NA, parallel_workers = parallel::detectCores(), auto_set_plan = TRUE) { |
| 43 | +#' |
| 44 | +bind_tweets <- function(data_path, user = FALSE, verbose = TRUE, output_format = NA) { |
37 | 45 | if (!is.na(output_format)) { |
38 | | - return(.flat(data_path, output_format = output_format, parallel_workers = parallel_workers, auto_set_plan = auto_set_plan)) |
| 46 | + return(.flat(data_path, output_format = output_format)) |
39 | 47 | } |
40 | 48 | if(user) { |
41 | 49 | files <- ls_files(data_path, "^users_") |
@@ -137,18 +145,14 @@ convert_json <- function(data_file, output_format = "tidy") { |
137 | 145 | return(file.path(dirname(data_filename), paste0("users_", ids, ".json"))) |
138 | 146 | } |
139 | 147 |
|
140 | | -.flat <- function(data_path, output_format = "tidy", parallel_workers, auto_set_plan) { |
| 148 | +.flat <- function(data_path, output_format = "tidy") { |
141 | 149 | if (!output_format %in% c("tidy", "raw")) { |
142 | 150 | stop("Unknown format.", call. = FALSE) |
143 | 151 | } |
144 | 152 | data_files <- ls_files(data_path, "^data_") |
145 | 153 | if (output_format == "raw") { |
146 | 154 | return(convert_json(data_files, output_format = "raw")) |
147 | 155 | } |
148 | | - if (auto_set_plan == TRUE && parallel_workers > 1) { |
149 | | - session_plan <- future::plan(future::multisession, workers = parallel_workers) |
150 | | - on.exit(future::plan(session_plan)) |
151 | | - } |
152 | 156 | return(furrr::future_map_dfr(data_files, convert_json, output_format = output_format)) |
153 | 157 | } |
154 | 158 |
|
|
0 commit comments