|
5 | 5 | #' By default, `bind_tweets` binds into a data frame containing tweets (from data_*id*.json files). |
6 | 6 | #' |
7 | 7 | #' If users is TRUE, it binds into a data frame containing user information (from users_*id*.json). |
| 8 | +#' |
| 9 | +#' For the "tidy" format, parallel processing with furrr is supported. In order to enable parallel processing, workers need to be set manually through [future::plan()]. See examples |
8 | 10 | #' |
9 | 11 | #' @param data_path string, file path to directory of stored tweets data saved as data_*id*.json and users_*id*.json |
10 | 12 | #' @param user If `FALSE`, this function binds JSON files into a data frame containing tweets; data frame containing user information otherwise. Ignore if `output_format` is not NA |
|
15 | 17 | #' \item{"raw"}{List of data frames; Note: not all data frames are in Boyce-Codd 3rd Normal Form} |
16 | 18 | #' \item{"tidy"}{Tidy format; all essential columns are available} |
17 | 19 | #' } |
| 20 | +#' |
18 | 21 | #' @return a data.frame containing either tweets or user information |
19 | 22 | #' @export |
20 | 23 | #' |
|
28 | 31 | #' |
29 | 32 | #' # bind json files in the directory "data" into a "tidy" data frame / tibble |
30 | 33 | #' bind_tweets(data_path = "data/", user = TRUE, output_format = "tidy") |
| 34 | +#' |
| 35 | +#' # bind json files in the directory "data" into a "tidy" data frame / tibble with parallel computing |
| 36 | +#' ## set up a multisession |
| 37 | +#' future::plan("multisession") |
| 38 | +#' ## run the function - note that no additional arguments are required |
| 39 | +#' bind_tweets(data_path = "data/", user = TRUE, output_format = "tidy") |
| 40 | +#' ## Shut down parallel workers |
| 41 | +#' future::plan("sequential") |
31 | 42 | #' } |
| 43 | +#' |
32 | 44 | bind_tweets <- function(data_path, user = FALSE, verbose = TRUE, output_format = NA) { |
33 | 45 | if (!is.na(output_format)) { |
34 | 46 | return(.flat(data_path, output_format = output_format)) |
@@ -142,7 +154,7 @@ convert_json <- function(data_file, output_format = "tidy") { |
142 | 154 | if (output_format == "raw") { |
143 | 155 | return(convert_json(data_files, output_format = "raw")) |
144 | 156 | } |
145 | | - return(purrr::map_dfr(data_files, convert_json, output_format = output_format)) |
| 157 | + return(furrr::future_map_dfr(data_files, convert_json, output_format = output_format)) |
146 | 158 | } |
147 | 159 |
|
148 | 160 | .gen_raw <- function(df, pkicol = "id", pki_name = "tweet_id") { |
|
0 commit comments