async with crul

Asynchronous requests with crul.

There are two interfaces to asynchronous requests in crul:

Simple async: any number of URLs, all treated with the same curl options, headers, etc., and only one HTTP method type at a time.
Varied request async: build any type of request and execute all asynchronously.

The first option takes less thinking, less work, and is good solution when you just want to hit a bunch of URLs asynchronously.

The second option is ideal when you want to set curl options/headers on each request and/or want to do different types of HTTP methods on each request.

One thing to think about before using async is whether the data provider is okay with it. It’s possible that a data provider’s service may be brought down if you do too many async requests.

library("crul")

simple async

Build request objcect with 1 or more URLs

(cc <- Async$new(
  urls = c(
    'https://httpbin.org/get?a=5',
    'https://httpbin.org/get?a=5&b=6',
    'https://httpbin.org/ip'
  )
))
#> <crul async connection> 
#>   urls: 
#>    https://httpbin.org/get?a=5
#>    https://httpbin.org/get?a=5&b=6
#>    https://httpbin.org/ip

Make request with any HTTP method

(res <- cc$get())
#> [[1]]
#> <crul response> 
#>   url: https://httpbin.org/get?a=5&b=6
#>   request_headers: 
#>     useragent: libcurl/7.51.0 r-curl/2.4 crul/0.3.4
#>   response_headers: 
#>     status: HTTP/1.1 200 OK
#>     connection: keep-alive
#>     server: gunicorn/19.7.1
#>     date: Fri, 31 Mar 2017 18:09:47 GMT
#>     content-type: application/json
#>     access-control-allow-origin: *
#>     access-control-allow-credentials: true
#>     content-length: 319
#>     via: 1.1 vegur
#>   params: 
#>     a: 5
#>     b: 6
#>   status: 200
#> 
#> [[2]]
#> <crul response> 
#>   url: https://httpbin.org/ip
#>   request_headers: 
#>     useragent: libcurl/7.51.0 r-curl/2.4 crul/0.3.4
#>   response_headers: 
#>     status: HTTP/1.1 200 OK
#>     connection: keep-alive
#>     server: gunicorn/19.7.1
#>     date: Fri, 31 Mar 2017 18:09:47 GMT
#>     content-type: application/json
#>     access-control-allow-origin: *
#>     access-control-allow-credentials: true
#>     content-length: 31
#>     via: 1.1 vegur
#>   status: 200
#> 
#> [[3]]
#> <crul response> 
#>   url: https://httpbin.org/get?a=5
#>   request_headers: 
#>     useragent: libcurl/7.51.0 r-curl/2.4 crul/0.3.4
#>   response_headers: 
#>     status: HTTP/1.1 200 OK
#>     connection: keep-alive
#>     server: gunicorn/19.7.1
#>     date: Fri, 31 Mar 2017 18:09:47 GMT
#>     content-type: application/json
#>     access-control-allow-origin: *
#>     access-control-allow-credentials: true
#>     content-length: 300
#>     via: 1.1 vegur
#>   params: 
#>     a: 5
#>   status: 200

You get back a list matching length of the number of input URLs

Access object variables and methods just as with HttpClient results, here just one at a time.

res[[1]]$url
#> [1] "https://httpbin.org/get?a=5&b=6"
res[[1]]$success()
#> [1] TRUE
res[[1]]$parse("UTF-8")
#> [1] "{\n  \"args\": {\n    \"a\": \"5\", \n    \"b\": \"6\"\n  }, \n  \"headers\": {\n    \"Accept\": \"*/*\", \n    \"Accept-Encoding\": \"gzip, deflate\", \n    \"Connection\": \"close\", \n    \"Host\": \"httpbin.org\", \n    \"User-Agent\": \"libcurl/7.51.0 r-curl/2.4 crul/0.3.4\"\n  }, \n  \"origin\": \"46.189.67.31\", \n  \"url\": \"https://httpbin.org/get?a=5&b=6\"\n}\n"

Or apply access/method calls aross many results, e.g., parse all results

lapply(res, function(z) z$parse("UTF-8"))
#> [[1]]
#> [1] "{\n  \"args\": {\n    \"a\": \"5\", \n    \"b\": \"6\"\n  }, \n  \"headers\": {\n    \"Accept\": \"*/*\", \n    \"Accept-Encoding\": \"gzip, deflate\", \n    \"Connection\": \"close\", \n    \"Host\": \"httpbin.org\", \n    \"User-Agent\": \"libcurl/7.51.0 r-curl/2.4 crul/0.3.4\"\n  }, \n  \"origin\": \"46.189.67.31\", \n  \"url\": \"https://httpbin.org/get?a=5&b=6\"\n}\n"
#> 
#> [[2]]
#> [1] "{\n  \"origin\": \"46.189.67.31\"\n}\n"
#> 
#> [[3]]
#> [1] "{\n  \"args\": {\n    \"a\": \"5\"\n  }, \n  \"headers\": {\n    \"Accept\": \"*/*\", \n    \"Accept-Encoding\": \"gzip, deflate\", \n    \"Connection\": \"close\", \n    \"Host\": \"httpbin.org\", \n    \"User-Agent\": \"libcurl/7.51.0 r-curl/2.4 crul/0.3.4\"\n  }, \n  \"origin\": \"46.189.67.31\", \n  \"url\": \"https://httpbin.org/get?a=5\"\n}\n"

varied request async

req1 <- HttpRequest$new(
  url = "https://httpbin.org/get?a=5",
  opts = list(
    verbose = TRUE
  )
)
req1$get()
#> <crul http request> get
#>   url: https://httpbin.org/get?a=5
#>   curl options: 
#>     verbose: TRUE
#>   proxies: 
#>   headers:

req2 <- HttpRequest$new(
  url = "https://httpbin.org/post?a=5&b=6"
)
req2$post(body = list(a = 5))
#> <crul http request> post
#>   url: https://httpbin.org/post?a=5&b=6
#>   curl options: 
#>   proxies: 
#>   headers:

(res <- AsyncVaried$new(req1, req2))
#> <crul async varied connection> 
#>   requests: 
#>    get: https://httpbin.org/get?a=5 
#>    post: https://httpbin.org/post?a=5&b=6

Make requests asynchronously

res$request()

Parse all results

res$parse()
#> [1] "{\n  \"args\": {\n    \"a\": \"5\"\n  }, \n  \"headers\": {\n    \"Accept\": \"*/*\", \n    \"Accept-Encoding\": \"gzip, deflate\", \n    \"Connection\": \"close\", \n    \"Host\": \"httpbin.org\", \n    \"User-Agent\": \"libcurl/7.51.0 r-curl/2.4 crul/0.3.4\"\n  }, \n  \"origin\": \"46.189.67.31\", \n  \"url\": \"https://httpbin.org/get?a=5\"\n}\n"                                                                                                                                                                                                                                                                                                       
#> [2] "{\n  \"args\": {\n    \"a\": \"5\", \n    \"b\": \"6\"\n  }, \n  \"data\": \"\", \n  \"files\": {}, \n  \"form\": {\n    \"a\": \"5\"\n  }, \n  \"headers\": {\n    \"Accept\": \"*/*\", \n    \"Accept-Encoding\": \"gzip, deflate\", \n    \"Connection\": \"close\", \n    \"Content-Length\": \"137\", \n    \"Content-Type\": \"multipart/form-data; boundary=------------------------a6d02fc2ae5c450e\", \n    \"Expect\": \"100-continue\", \n    \"Host\": \"httpbin.org\", \n    \"User-Agent\": \"libcurl/7.51.0 r-curl/2.4 crul/0.3.4\"\n  }, \n  \"json\": null, \n  \"origin\": \"46.189.67.31\", \n  \"url\": \"https://httpbin.org/post?a=5&b=6\"\n}\n"

lapply(res$parse(), jsonlite::prettify)
#> [[1]]
#> {
#>     "args": {
#>         "a": "5"
#>     },
#>     "headers": {
#>         "Accept": "*/*",
#>         "Accept-Encoding": "gzip, deflate",
#>         "Connection": "close",
#>         "Host": "httpbin.org",
#>         "User-Agent": "libcurl/7.51.0 r-curl/2.4 crul/0.3.4"
#>     },
#>     "origin": "46.189.67.31",
#>     "url": "https://httpbin.org/get?a=5"
#> }
#>  
#> 
#> [[2]]
#> {
#>     "args": {
#>         "a": "5",
#>         "b": "6"
#>     },
#>     "data": "",
#>     "files": {
#> 
#>     },
#>     "form": {
#>         "a": "5"
#>     },
#>     "headers": {
#>         "Accept": "*/*",
#>         "Accept-Encoding": "gzip, deflate",
#>         "Connection": "close",
#>         "Content-Length": "137",
#>         "Content-Type": "multipart/form-data; boundary=------------------------a6d02fc2ae5c450e",
#>         "Expect": "100-continue",
#>         "Host": "httpbin.org",
#>         "User-Agent": "libcurl/7.51.0 r-curl/2.4 crul/0.3.4"
#>     },
#>     "json": null,
#>     "origin": "46.189.67.31",
#>     "url": "https://httpbin.org/post?a=5&b=6"
#> }
#>

Status codes

res$status_code()
#> [1] 200 200

async with crul

Scott Chamberlain

2017-03-31

simple async

varied request async