From 9ebd2f3051c7ae1dfc64c4c7856ded89fdbf9b22 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Fri, 18 Aug 2023 12:08:09 -0300
Subject: [PATCH 01/36] basic port of async utils

---
 r/adbcdrivermanager/src/async.cc | 216 +++++++++++++++++++++++++++++++
 1 file changed, 216 insertions(+)
 create mode 100644 r/adbcdrivermanager/src/async.cc

diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
new file mode 100644
index 0000000000..2ea523a183
--- /dev/null
+++ b/r/adbcdrivermanager/src/async.cc
@@ -0,0 +1,216 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#define R_NO_REMAP
+#include <R.h>
+#include <Rinternals.h>
+
+#include <deque>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include <adbc.h>
+
+#include "radbc.h"
+
+class Task {
+ public:
+  Task() : worker(nullptr) {}
+
+  std::thread* worker;
+
+  static void FinalizeXptr(SEXP xptr) {
+    Task* task = reinterpret_cast<Task*>(R_ExternalPtrAddr(xptr));
+    if (task->worker != nullptr) {
+      delete task->worker;
+    }
+
+    delete task;
+  }
+
+  static SEXP MakeXptr(SEXP shelter = R_NilValue) {
+    SEXP xptr = PROTECT(R_MakeExternalPtr(new Task(), R_NilValue, shelter));
+    R_RegisterCFinalizer(xptr, &FinalizeXptr);
+    UNPROTECT(1);
+    return xptr;
+  }
+};
+
+class CallbackQueue {
+ public:
+  // Because it is used in C++ frames where a longjmp may occur,
+  // the members of this struct must be trivially destructible:
+  // Any destruction that needs to occur must occur via an SEXP
+  // finalizer set on the return_value_xptr.
+  struct RCallback {
+    // An environment containing a function named "callback". This callback
+    // is executed as callback(return_code, error_xptr, return_value_int,
+    // return_value_xptr). The environment may contain other items that need to stay valid
+    // for the lifetime of the task (e.g., inputs).
+    SEXP env_sexp;
+
+    // An external pointer to an AdbcError*
+    SEXP error_xptr;
+
+    // An external pointer to return_value_ptr (with proper finalizer set as applicable)
+    SEXP return_value_xptr;
+
+    // A return code (e.g., AdbcStatusCode). A return code of 0 indicates success.
+    int return_code;
+
+    // The external pointer address of error_xptr
+    AdbcError* return_error;
+
+    // An integer return value. This is needed for execute query, which returns both
+    // rows affected and (potentially) an array stream.
+    int64_t return_value_int;
+
+    // The external pointer address of return_value_xptr
+    void* return_value_ptr;
+  };
+
+  RCallback InitCallback(SEXP callback_env, SEXP return_value_xptr = R_NilValue,
+                         SEXP error_xptr = R_NilValue) {
+    RCallback out{callback_env, error_xptr, return_value_xptr, NA_INTEGER, nullptr, 0,
+                  nullptr};
+    if (error_xptr != R_NilValue) {
+      out.return_error = reinterpret_cast<AdbcError*>(R_ExternalPtrAddr(error_xptr));
+    }
+
+    if (return_value_xptr != R_NilValue) {
+      out.return_value_ptr = R_ExternalPtrAddr(return_value_xptr);
+    }
+
+    R_PreserveObject(out.env_sexp);
+    R_PreserveObject(out.error_xptr);
+    R_PreserveObject(out.return_value_xptr);
+
+    return out;
+  }
+
+  // Add a callback to the queue with thread safety. This can
+  // (should) be called from another thread.
+  void AddCallback(RCallback callback) {
+    std::lock_guard<std::mutex> lock(callbacks_lock_);
+    pending_callbacks_.push_back(callback);
+  }
+
+  // Try to run all callbacks. Callbacks should be written such that
+  // they do not error; however, this function is written such that
+  // an erroring callback will simply result in a (potentially)
+  // incompletely executed callback queue.
+  int64_t RunPending() {
+    int64_t n_run = 0;
+    while (!pending_callbacks_.empty()) {
+      // RunCallback() may may longjmp
+      RCallback callback = GetNextCallback();
+      RunCallback(callback);
+      n_run++;
+    }
+
+    return n_run;
+  }
+
+  // Pop a callback from the end of the queue with thread safety.
+  RCallback GetNextCallback() {
+    std::lock_guard<std::mutex> lock(callbacks_lock_);
+    RCallback callback = pending_callbacks_.front();
+    pending_callbacks_.pop_front();
+    return callback;
+  }
+
+  // Transfer SEXP ownership of callback members to the stack and run
+  // the callback. This must be called from the main R thread.
+  void RunCallback(RCallback callback) {
+    // Transfer responsibility of releasing SEXPs to the stack such that
+    // they will be garbage collected if any of the R calls below error
+    SEXP env_sexp = PROTECT(callback.env_sexp);
+    SEXP error_xptr = PROTECT(callback.error_xptr);
+    SEXP return_value_xptr = PROTECT(callback.return_value_xptr);
+    R_ReleaseObject(env_sexp);
+    R_ReleaseObject(error_xptr);
+    R_ReleaseObject(return_value_xptr);
+
+    SEXP callback_sym = PROTECT(Rf_install("callback"));
+    SEXP return_code_sexp = PROTECT(Rf_ScalarInteger(callback.return_code));
+    SEXP callback_call =
+        PROTECT(Rf_lang3(callback_sym, return_code_sexp, return_value_xptr));
+
+    Rf_eval(callback_call, env_sexp);
+    UNPROTECT(6);
+  }
+
+  static void FinalizeXptr(SEXP xptr) {
+    CallbackQueue* queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(xptr));
+    delete queue;
+  }
+
+  static SEXP MakeXptr(SEXP shelter = R_NilValue) {
+    SEXP xptr = PROTECT(R_MakeExternalPtr(new CallbackQueue(), R_NilValue, shelter));
+    R_RegisterCFinalizer(xptr, &FinalizeXptr);
+    UNPROTECT(1);
+    return xptr;
+  }
+
+ private:
+  std::deque<RCallback> pending_callbacks_;
+  std::mutex callbacks_lock_;
+};
+
+// TODO: unlike other parts of this package, these functions make use of
+// C++ standard library functions that might throw exceptions in functions
+// that are directly called from R. These will segfault if an exception is
+// thrown. cpp11 handles this using BEGIN_CPP11 an END_CPP11...we would need
+// similar for safety here.
+
+extern "C" SEXP RAdbcNewCallbackQueue(void) { return CallbackQueue::MakeXptr(); }
+
+extern "C" SEXP RAdbcCallbackQueueRunCallbacks(SEXP callback_queue_xptr) {
+  // TODO: Check callback_queue_xptr class
+  auto queue = reinterpret_cast<CallbackQueue*>(callback_queue_xptr);
+  return Rf_ScalarReal(queue->RunPending());
+}
+
+extern "C" SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr, SEXP array_stream_xptr,
+                                  SEXP array_xptr, SEXP callback_env) {
+  // TODO: check array_stream/array/callback queue classes using utils in radbc.h
+
+  auto queue = reinterpret_cast<CallbackQueue*>(callback_queue_xptr);
+  auto array_stream =
+      reinterpret_cast<struct ArrowArrayStream*>(R_ExternalPtrAddr(array_stream_xptr));
+  auto array = reinterpret_cast<struct ArrowArray*>(R_ExternalPtrAddr(array_xptr));
+
+  // Task handle to ensure the thread pointer is cleaned up
+  SEXP task_xptr = PROTECT(Task::MakeXptr());
+  SEXP task_symbol = PROTECT(Rf_install("task"));
+  Rf_setVar(task_symbol, task_xptr, callback_env);
+  UNPROTECT(1);
+
+  auto task = reinterpret_cast<Task*>(R_ExternalPtrAddr(task_xptr));
+  CallbackQueue::RCallback callback = queue->InitCallback(callback_env);
+  task->worker = new std::thread([array_stream, array, callback, queue] {
+      CallbackQueue::RCallback callback_out = callback;
+      callback_out.return_code = array_stream->get_next(array_stream, array);
+      queue->AddCallback(callback_out);
+    });
+
+  UNPROTECT(1);
+  return R_NilValue;
+}

From e3df089e3444b7c7960fc38023fccde3ab7238e5 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Fri, 18 Aug 2023 12:59:10 -0300
Subject: [PATCH 02/36] with passing test

---
 r/adbcdrivermanager/R/async.R                 | 63 +++++++++++++++++++
 r/adbcdrivermanager/R/error.R                 |  7 ++-
 r/adbcdrivermanager/src/async.cc              | 43 +++++++------
 .../tests/testthat/test-async.R               | 26 ++++++++
 4 files changed, 119 insertions(+), 20 deletions(-)
 create mode 100644 r/adbcdrivermanager/R/async.R
 create mode 100644 r/adbcdrivermanager/tests/testthat/test-async.R

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
new file mode 100644
index 0000000000..930f883821
--- /dev/null
+++ b/r/adbcdrivermanager/R/async.R
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+adbc_callback_queue <- function() {
+  .Call(RAdbcNewCallbackQueue)
+}
+
+adbc_callback_queue_run_pending <- function(queue) {
+  .Call(RAdbcCallbackQueueRunPending, queue)
+}
+
+adbc_array_stream_get_next_async <- function(stream, callback,
+                                             schema = stream$get_schema(),
+                                             queue = adbc_callback_queue()) {
+  callback <- as_adbc_callback(callback)
+  array <- nanoarrow::nanoarrow_allocate_array()
+  nanoarrow::nanoarrow_array_set_schema(array, schema, validate = FALSE)
+  .Call(RAdbcArrayStreamGetNextAsync, queue, stream, array, callback)
+
+  invisible(queue)
+}
+
+as_adbc_callback <- function(x) {
+  if (inherits(x, "adbc_callback")) {
+    x
+  } else if (is.function(x)) {
+    adbc_callback(x)
+  } else {
+    stop(sprintf("Can't create adbc_callback() from object of type '%s'", class(x)[1]))
+  }
+}
+
+adbc_callback <- function(on_success, ..., on_error = stop_for_error) {
+  callback_env <- new.env(parent = emptyenv())
+  callback_env$sheltered_objects = list(...)
+
+  force(on_success)
+  force(on_error)
+  callback_env$callback <- function(status, error, return_value_xptr) {
+    if (!identical(status, 0L)) {
+      try(on_error(status, error))
+    } else {
+      try(on_success(return_value_xptr))
+    }
+  }
+
+  class(callback_env) <- "adbc_callback"
+  callback_env
+}
diff --git a/r/adbcdrivermanager/R/error.R b/r/adbcdrivermanager/R/error.R
index a7c3eeb8d8..82ee9cf269 100644
--- a/r/adbcdrivermanager/R/error.R
+++ b/r/adbcdrivermanager/R/error.R
@@ -52,7 +52,12 @@ adbc_allocate_error <- function(shelter = NULL, use_legacy_error = NULL) {
 
 stop_for_error <- function(status, error) {
   if (!identical(status, 0L)) {
-    error <- .Call(RAdbcErrorProxy, error)
+    if (inherits(error, "adbc_error")) {
+      error <- .Call(RAdbcErrorProxy, error)
+    } else {
+      error <- list()
+    }
+
     error$status <- status
     error$status_code_message <- .Call(RAdbcStatusCodeMessage, status)
     if (!is.null(error$message)) {
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 2ea523a183..8ca8c3750b 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -78,18 +78,14 @@ class CallbackQueue {
     // The external pointer address of error_xptr
     AdbcError* return_error;
 
-    // An integer return value. This is needed for execute query, which returns both
-    // rows affected and (potentially) an array stream.
-    int64_t return_value_int;
-
     // The external pointer address of return_value_xptr
     void* return_value_ptr;
   };
 
   RCallback InitCallback(SEXP callback_env, SEXP return_value_xptr = R_NilValue,
                          SEXP error_xptr = R_NilValue) {
-    RCallback out{callback_env, error_xptr, return_value_xptr, NA_INTEGER, nullptr, 0,
-                  nullptr};
+    RCallback out{callback_env, error_xptr, return_value_xptr,
+                  NA_INTEGER,   nullptr,    nullptr};
     if (error_xptr != R_NilValue) {
       out.return_error = reinterpret_cast<AdbcError*>(R_ExternalPtrAddr(error_xptr));
     }
@@ -148,12 +144,21 @@ class CallbackQueue {
     R_ReleaseObject(error_xptr);
     R_ReleaseObject(return_value_xptr);
 
+    // Release the dependence of the task on this callback queue
+    SEXP task_sym = PROTECT(Rf_install("task"));
+    SEXP task_xptr = PROTECT(Rf_findVarInFrame(env_sexp, task_sym));
+    R_SetExternalPtrProtected(task_xptr, R_NilValue);
+    UNPROTECT(2);
+
+    // Set up the call to run the callback
     SEXP callback_sym = PROTECT(Rf_install("callback"));
     SEXP return_code_sexp = PROTECT(Rf_ScalarInteger(callback.return_code));
     SEXP callback_call =
-        PROTECT(Rf_lang3(callback_sym, return_code_sexp, return_value_xptr));
+        PROTECT(Rf_lang4(callback_sym, return_code_sexp, error_xptr, return_value_xptr));
 
+    // Run the callback
     Rf_eval(callback_call, env_sexp);
+
     UNPROTECT(6);
   }
 
@@ -182,34 +187,34 @@ class CallbackQueue {
 
 extern "C" SEXP RAdbcNewCallbackQueue(void) { return CallbackQueue::MakeXptr(); }
 
-extern "C" SEXP RAdbcCallbackQueueRunCallbacks(SEXP callback_queue_xptr) {
+extern "C" SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr) {
   // TODO: Check callback_queue_xptr class
-  auto queue = reinterpret_cast<CallbackQueue*>(callback_queue_xptr);
+  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
   return Rf_ScalarReal(queue->RunPending());
 }
 
-extern "C" SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr, SEXP array_stream_xptr,
-                                  SEXP array_xptr, SEXP callback_env) {
+extern "C" SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr,
+                                             SEXP array_stream_xptr, SEXP array_xptr,
+                                             SEXP callback_env) {
   // TODO: check array_stream/array/callback queue classes using utils in radbc.h
-
-  auto queue = reinterpret_cast<CallbackQueue*>(callback_queue_xptr);
+  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
   auto array_stream =
       reinterpret_cast<struct ArrowArrayStream*>(R_ExternalPtrAddr(array_stream_xptr));
   auto array = reinterpret_cast<struct ArrowArray*>(R_ExternalPtrAddr(array_xptr));
 
   // Task handle to ensure the thread pointer is cleaned up
-  SEXP task_xptr = PROTECT(Task::MakeXptr());
+  SEXP task_xptr = PROTECT(Task::MakeXptr(callback_queue_xptr));
   SEXP task_symbol = PROTECT(Rf_install("task"));
   Rf_setVar(task_symbol, task_xptr, callback_env);
   UNPROTECT(1);
 
   auto task = reinterpret_cast<Task*>(R_ExternalPtrAddr(task_xptr));
-  CallbackQueue::RCallback callback = queue->InitCallback(callback_env);
+  CallbackQueue::RCallback callback = queue->InitCallback(callback_env, array_xptr);
   task->worker = new std::thread([array_stream, array, callback, queue] {
-      CallbackQueue::RCallback callback_out = callback;
-      callback_out.return_code = array_stream->get_next(array_stream, array);
-      queue->AddCallback(callback_out);
-    });
+    CallbackQueue::RCallback callback_out = callback;
+    callback_out.return_code = array_stream->get_next(array_stream, array);
+    queue->AddCallback(callback_out);
+  });
 
   UNPROTECT(1);
   return R_NilValue;
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
new file mode 100644
index 0000000000..31995d3996
--- /dev/null
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+test_that("aync array_stream$get_next", {
+  stream <- nanoarrow::basic_array_stream(list(1:5))
+  queue <- adbc_array_stream_get_next_async(stream, function(array) {
+    expect_identical(nanoarrow::convert_array(array), 1:5)
+  })
+
+  Sys.sleep(0.5)
+  expect_identical(adbc_callback_queue_run_pending(queue), 1)
+})

From e3f6b07e21056d777b90f0d5228b61c5ae219164 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Fri, 18 Aug 2023 13:22:50 -0300
Subject: [PATCH 03/36] some notes

---
 r/adbcdrivermanager/R/async.R    |  1 +
 r/adbcdrivermanager/src/async.cc | 16 +++++++++++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 930f883821..eb2cc377cc 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -27,6 +27,7 @@ adbc_array_stream_get_next_async <- function(stream, callback,
                                              schema = stream$get_schema(),
                                              queue = adbc_callback_queue()) {
   callback <- as_adbc_callback(callback)
+  callback$args <- list(stream)
   array <- nanoarrow::nanoarrow_allocate_array()
   nanoarrow::nanoarrow_array_set_schema(array, schema, validate = FALSE)
   .Call(RAdbcArrayStreamGetNextAsync, queue, stream, array, callback)
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 8ca8c3750b..02e9378ff7 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -30,6 +30,11 @@
 
 #include "radbc.h"
 
+// A thin wrapper around a std::thread() that ensures that the thread
+// does not leak. This could also maybe just be an external pointer to
+// a std::thread*. The external pointer to the Task holds a strong
+// reference to the external pointer to the CallbackQueue that is
+// released just before the callback is run.
 class Task {
  public:
   Task() : worker(nullptr) {}
@@ -39,6 +44,8 @@ class Task {
   static void FinalizeXptr(SEXP xptr) {
     Task* task = reinterpret_cast<Task*>(R_ExternalPtrAddr(xptr));
     if (task->worker != nullptr) {
+      // TODO: check task->worker->joinable()?
+      task->worker->join();
       delete task->worker;
     }
 
@@ -53,6 +60,7 @@ class Task {
   }
 };
 
+// A thread-safe queue of callbacks to execute
 class CallbackQueue {
  public:
   // Because it is used in C++ frames where a longjmp may occur,
@@ -61,9 +69,9 @@ class CallbackQueue {
   // finalizer set on the return_value_xptr.
   struct RCallback {
     // An environment containing a function named "callback". This callback
-    // is executed as callback(return_code, error_xptr, return_value_int,
-    // return_value_xptr). The environment may contain other items that need to stay valid
-    // for the lifetime of the task (e.g., inputs).
+    // is executed as callback(return_code, error_xptr, return_value_xptr). The
+    // environment may contain other items that need to stay valid for the lifetime of the
+    // task (e.g., inputs).
     SEXP env_sexp;
 
     // An external pointer to an AdbcError*
@@ -82,6 +90,8 @@ class CallbackQueue {
     void* return_value_ptr;
   };
 
+  // Initialize a callback and preserve its SEXP members. This must
+  // be called from the main R thread.
   RCallback InitCallback(SEXP callback_env, SEXP return_value_xptr = R_NilValue,
                          SEXP error_xptr = R_NilValue) {
     RCallback out{callback_env, error_xptr, return_value_xptr,

From 26aad2fcd705d1015a08d47484bd7a552c5d005c Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Fri, 18 Aug 2023 13:24:45 -0300
Subject: [PATCH 04/36] more notes

---
 r/adbcdrivermanager/src/async.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 02e9378ff7..bc7438bbec 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -60,13 +60,14 @@ class Task {
   }
 };
 
-// A thread-safe queue of callbacks to execute
+// A thread-safe queue of callbacks to execute.
 class CallbackQueue {
  public:
-  // Because it is used in C++ frames where a longjmp may occur,
+  // Because the RCallback is used in C++ frames where a longjmp may occur,
   // the members of this struct must be trivially destructible:
   // Any destruction that needs to occur must occur via an SEXP
-  // finalizer set on the return_value_xptr.
+  // finalizer set on the return_value_xptr. This struct is intentionally
+  // copyable and is passed by value.
   struct RCallback {
     // An environment containing a function named "callback". This callback
     // is executed as callback(return_code, error_xptr, return_value_xptr). The

From 059fc569533f4fb0beda7893f10ef8efad2c0c23 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@dunnington.ca>
Date: Thu, 31 Aug 2023 09:47:50 -0300
Subject: [PATCH 05/36] Update r/adbcdrivermanager/tests/testthat/test-async.R
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Kirill Müller <krlmlr@users.noreply.github.com>
---
 r/adbcdrivermanager/tests/testthat/test-async.R | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 31995d3996..5416ae044d 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -17,10 +17,13 @@
 
 test_that("aync array_stream$get_next", {
   stream <- nanoarrow::basic_array_stream(list(1:5))
+  async_called <- FALSE
   queue <- adbc_array_stream_get_next_async(stream, function(array) {
+    async_called <<- TRUE
     expect_identical(nanoarrow::convert_array(array), 1:5)
   })
 
-  Sys.sleep(0.5)
+  expect_false(async_called)
   expect_identical(adbc_callback_queue_run_pending(queue), 1)
+  expect_true(async_called)
 })

From 4f25aa4977a5a0056309622383f968bedab08648 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Tue, 5 Sep 2023 11:19:00 -0300
Subject: [PATCH 06/36] add promises/later integration to async

---
 r/adbcdrivermanager/DESCRIPTION               |  2 +
 r/adbcdrivermanager/R/async.R                 | 72 +++++++++++++++++++
 r/adbcdrivermanager/R/error.R                 | 17 ++++-
 .../tests/testthat/test-async.R               | 21 +++++-
 4 files changed, 110 insertions(+), 2 deletions(-)

diff --git a/r/adbcdrivermanager/DESCRIPTION b/r/adbcdrivermanager/DESCRIPTION
index cbd53577ac..4810c28c6a 100644
--- a/r/adbcdrivermanager/DESCRIPTION
+++ b/r/adbcdrivermanager/DESCRIPTION
@@ -17,6 +17,8 @@ Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.2.3
 Suggests:
+    later,
+    promises,
     testthat (>= 3.0.0),
     withr
 Config/testthat/edition: 3
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index eb2cc377cc..638fe8e990 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -35,6 +35,78 @@ adbc_array_stream_get_next_async <- function(stream, callback,
   invisible(queue)
 }
 
+adbc_array_stream_get_next_promise <- function(stream,
+                                               schema = stream$get_schema(),
+                                               loop = NULL, delay = 0) {
+  if (is.null(loop)) {
+    loop <- later::current_loop()
+  }
+
+  queue <- callback_queue_from_later_loop(loop)
+
+  later_loop_schedule_run_pending(loop, delay = delay)
+
+  promises::promise(function(resolve, reject) {
+    callback <- adbc_callback(
+      on_success = resolve,
+      on_error = function(status, error) {
+        reject(adbc_error_message(status, error))
+      }
+    )
+
+    adbc_array_stream_get_next_async(
+      stream,
+      callback,
+      schema = schema,
+      queue = queue
+    )
+  })
+}
+
+later_loop_schedule_run_pending <- function(loop, delay = 0) {
+  later::later(function() {
+    queue <- callback_queue_from_later_loop(loop)
+    n_run <- adbc_callback_queue_run_pending(queue)
+    if (n_run > 0) {
+      later_loop_schedule_run_pending(loop, delay = delay)
+    } else {
+      string_id <- as.character(loop$id)
+      callback_queue_cache[[string_id]] <- adbc_callback_queue()
+    }
+  }, delay = delay, loop = loop)
+}
+
+callback_queue_from_later_loop <- function(loop) {
+  string_id <- as.character(loop$id)
+  if (is.null(callback_queue_cache[[string_id]])) {
+    callback_queue_cache[[string_id]] <- adbc_callback_queue()
+  }
+
+  callback_queue_cache[[string_id]]
+}
+
+callback_queue_cache <- new.env(parent = emptyenv())
+
+# test helper to force a promise to resolve
+later_loop_wait <- function(loop = NULL, timeout = 10, delay = 0.01) {
+  if (is.null(loop)) {
+    loop <- later::current_loop()
+  }
+
+  start <- Sys.time()
+  while (!later::loop_empty(loop)) {
+    elapsed <- difftime(Sys.time(), start, units = "secs")
+    if (elapsed > timeout) {
+      stop(sprintf("later_loop_wait() timed out at %s seconds", elapsed))
+    }
+
+    later::run_now(loop = loop)
+    Sys.sleep(delay)
+  }
+
+  invisible(loop)
+}
+
 as_adbc_callback <- function(x) {
   if (inherits(x, "adbc_callback")) {
     x
diff --git a/r/adbcdrivermanager/R/error.R b/r/adbcdrivermanager/R/error.R
index 82ee9cf269..0d7923e37a 100644
--- a/r/adbcdrivermanager/R/error.R
+++ b/r/adbcdrivermanager/R/error.R
@@ -58,7 +58,6 @@ stop_for_error <- function(status, error) {
       error <- list()
     }
 
-    error$status <- status
     error$status_code_message <- .Call(RAdbcStatusCodeMessage, status)
     if (!is.null(error$message)) {
       msg <- paste(error$status_code_message, error$message, sep=": ")
@@ -84,6 +83,22 @@ stop_for_error <- function(status, error) {
   }
 }
 
+adbc_error_message <- function(status, error) {
+  if (!identical(status, 0L)) {
+    if (inherits(error, "adbc_error")) {
+      error <- .Call(RAdbcErrorProxy, error)
+    } else {
+      error <- list()
+    }
+
+    error$status <- status
+    error$status_code_message <- .Call(RAdbcStatusCodeMessage, status)
+    if (!is.null(error$message)) error$message else error$status_code_message
+  } else {
+    "OK"
+  }
+}
+
 #' @export
 print.adbc_error <- function(x, ...) {
   str(x, ...)
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 5416ae044d..3b4253a480 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-test_that("aync array_stream$get_next", {
+test_that("async array_stream$get_next() works", {
   stream <- nanoarrow::basic_array_stream(list(1:5))
   async_called <- FALSE
   queue <- adbc_array_stream_get_next_async(stream, function(array) {
@@ -27,3 +27,22 @@ test_that("aync array_stream$get_next", {
   expect_identical(adbc_callback_queue_run_pending(queue), 1)
   expect_true(async_called)
 })
+
+test_that("async array_stream$get_next() promises/later integration works", {
+  skip_if_not_installed("later")
+  skip_if_not_installed("promises")
+
+  stream <- nanoarrow::basic_array_stream(list(1:5))
+  loop <- later::create_loop()
+  async_called <- FALSE
+
+  promise <- adbc_array_stream_get_next_promise(stream, loop = loop)
+  expect_true(promises::is.promising(promise))
+  promise$then(function(x) {
+    async_called <<- TRUE
+    expect_identical(nanoarrow::convert_array(x), 1:5)
+  })
+
+  later_loop_wait(loop)
+  expect_true(async_called)
+})

From 3ff0cb24c797e0b1656f0d5c47d7262572437927 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Tue, 5 Sep 2023 11:55:57 -0300
Subject: [PATCH 07/36] add execute query async

---
 r/adbcdrivermanager/R/async.R                 | 17 ++++++-
 r/adbcdrivermanager/src/async.cc              | 30 ++++++++++++
 .../tests/testthat/test-async.R               | 47 +++++++++++++++++++
 3 files changed, 92 insertions(+), 2 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 638fe8e990..f0f2343eb0 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -35,6 +35,19 @@ adbc_array_stream_get_next_async <- function(stream, callback,
   invisible(queue)
 }
 
+adbc_statement_execute_query_async <- function(statement, callback,
+                                               schema = stream$get_schema(),
+                                               queue = adbc_callback_queue()) {
+  callback <- as_adbc_callback(callback)
+  stream_out <- nanoarrow::nanoarrow_allocate_array_stream()
+  callback$args <- list(statement, stream_out)
+
+  error <- adbc_allocate_error()
+  .Call(RAdbcStatementExecuteQueryAsync, queue, statement, stream_out, error, callback)
+
+  invisible(queue)
+}
+
 adbc_array_stream_get_next_promise <- function(stream,
                                                schema = stream$get_schema(),
                                                loop = NULL, delay = 0) {
@@ -125,9 +138,9 @@ adbc_callback <- function(on_success, ..., on_error = stop_for_error) {
   force(on_error)
   callback_env$callback <- function(status, error, return_value_xptr) {
     if (!identical(status, 0L)) {
-      try(on_error(status, error))
+      on_error(status, error)
     } else {
-      try(on_success(return_value_xptr))
+      on_success(return_value_xptr)
     }
   }
 
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index bc7438bbec..1cb20b2ce0 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -204,6 +204,36 @@ extern "C" SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr) {
   return Rf_ScalarReal(queue->RunPending());
 }
 
+extern "C" SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr,
+                                                SEXP statement_xptr, SEXP out_stream_xptr,
+                                                SEXP error_xptr, SEXP callback_env) {
+  // TODO: check array_stream/array/callback queue classes using utils in radbc.h
+  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
+  auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
+  auto out_stream = adbc_from_xptr<ArrowArrayStream>(out_stream_xptr);
+  auto error = adbc_from_xptr<AdbcError>(error_xptr);
+
+  // Task handle to ensure the thread pointer is cleaned up
+  SEXP task_xptr = PROTECT(Task::MakeXptr(callback_queue_xptr));
+  SEXP task_symbol = PROTECT(Rf_install("task"));
+  Rf_setVar(task_symbol, task_xptr, callback_env);
+  UNPROTECT(1);
+
+  auto task = reinterpret_cast<Task*>(R_ExternalPtrAddr(task_xptr));
+  CallbackQueue::RCallback callback =
+      queue->InitCallback(callback_env, out_stream_xptr, error_xptr);
+  task->worker = new std::thread([statement, out_stream, error, callback, queue] {
+    CallbackQueue::RCallback callback_out = callback;
+    int64_t rows_affected = -1;
+    callback_out.return_code =
+        AdbcStatementExecuteQuery(statement, out_stream, &rows_affected, error);
+    queue->AddCallback(callback_out);
+  });
+
+  UNPROTECT(1);
+  return R_NilValue;
+}
+
 extern "C" SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr,
                                              SEXP array_stream_xptr, SEXP array_xptr,
                                              SEXP callback_env) {
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 3b4253a480..b7b75ef3c9 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -28,6 +28,53 @@ test_that("async array_stream$get_next() works", {
   expect_true(async_called)
 })
 
+test_that("async adbc_statement_execute_query() can return a value", {
+  db <- adbc_database_init(adbc_driver_monkey())
+  con <- adbc_connection_init(db)
+  input <- data.frame(x = 1:5)
+  stmt <- adbc_statement_init(con, input)
+
+  async_called <- FALSE
+  queue <- adbc_statement_execute_query_async(stmt, function(stream) {
+    async_called <<- TRUE
+    expect_identical(nanoarrow::convert_array_stream(stream), input)
+  })
+
+  expect_false(async_called)
+  expect_identical(adbc_callback_queue_run_pending(queue), 1)
+  expect_true(async_called)
+})
+
+test_that("async adbc_statement_execute_query() can return an error", {
+  db <- adbc_database_init(adbc_driver_void())
+  con <- adbc_connection_init(db)
+  stmt <- adbc_statement_init(con)
+
+  async_called <- FALSE
+  queue <- adbc_statement_execute_query_async(
+    stmt,
+    adbc_callback(
+      on_success = function(stream) {
+        stop("This should not be called")
+      },
+      on_error = function(status, error) {
+        expect_false(status == 0)
+        expect_s3_class(error, "adbc_error")
+        async_called <<- TRUE
+        stop_for_error(status, error)
+      }
+    )
+  )
+
+  expect_false(async_called)
+  expect_error(
+    adbc_callback_queue_run_pending(queue),
+    class = "adbc_status_not_implemented"
+  )
+  expect_true(async_called)
+  expect_identical(adbc_callback_queue_run_pending(queue), 0)
+})
+
 test_that("async array_stream$get_next() promises/later integration works", {
   skip_if_not_installed("later")
   skip_if_not_installed("promises")

From b2236c3acf8ff8476bb637dd099f7b7d9ac737c0 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Tue, 5 Sep 2023 12:47:04 -0300
Subject: [PATCH 08/36] test chains

---
 r/adbcdrivermanager/R/async.R                 | 22 +++++++++-
 .../tests/testthat/test-async.R               | 44 ++++++++++++++++++-
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index f0f2343eb0..ba95336adf 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -36,7 +36,6 @@ adbc_array_stream_get_next_async <- function(stream, callback,
 }
 
 adbc_statement_execute_query_async <- function(statement, callback,
-                                               schema = stream$get_schema(),
                                                queue = adbc_callback_queue()) {
   callback <- as_adbc_callback(callback)
   stream_out <- nanoarrow::nanoarrow_allocate_array_stream()
@@ -76,6 +75,27 @@ adbc_array_stream_get_next_promise <- function(stream,
   })
 }
 
+adbc_statement_execute_query_promise <- function(statement, loop = NULL, delay = 0) {
+  if (is.null(loop)) {
+    loop <- later::current_loop()
+  }
+
+  queue <- callback_queue_from_later_loop(loop)
+
+  later_loop_schedule_run_pending(loop, delay = delay)
+
+  promises::promise(function(resolve, reject) {
+    callback <- adbc_callback(
+      on_success = resolve,
+      on_error = function(status, error) {
+        reject(adbc_error_message(status, error))
+      }
+    )
+
+    adbc_statement_execute_query_async(statement, callback, queue = queue)
+  })
+}
+
 later_loop_schedule_run_pending <- function(loop, delay = 0) {
   later::later(function() {
     queue <- callback_queue_from_later_loop(loop)
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index b7b75ef3c9..84d90db244 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -84,7 +84,6 @@ test_that("async array_stream$get_next() promises/later integration works", {
   async_called <- FALSE
 
   promise <- adbc_array_stream_get_next_promise(stream, loop = loop)
-  expect_true(promises::is.promising(promise))
   promise$then(function(x) {
     async_called <<- TRUE
     expect_identical(nanoarrow::convert_array(x), 1:5)
@@ -93,3 +92,46 @@ test_that("async array_stream$get_next() promises/later integration works", {
   later_loop_wait(loop)
   expect_true(async_called)
 })
+
+test_that("async adbc_execute_query() promises/later integration works", {
+  skip_if_not_installed("later")
+  skip_if_not_installed("promises")
+
+  db <- adbc_database_init(adbc_driver_monkey())
+  con <- adbc_connection_init(db)
+  input <- data.frame(x = 1:5)
+  stmt <- adbc_statement_init(con, input)
+
+  loop <- later::create_loop()
+  async_called <- FALSE
+
+  adbc_statement_execute_query_promise(stmt, loop = loop)$then(function(x) {
+    async_called <<- TRUE
+    expect_identical(nanoarrow::convert_array_stream(x), input)
+  })
+
+  later_loop_wait(loop)
+  expect_true(async_called)
+})
+
+test_that("async adbc_execute_query() promises/later integration can error", {
+  skip_if_not_installed("later")
+  skip_if_not_installed("promises")
+
+  db <- adbc_database_init(adbc_driver_void())
+  con <- adbc_connection_init(db)
+  stmt <- adbc_statement_init(con)
+
+  loop <- later::create_loop()
+  async_called <- FALSE
+
+  adbc_statement_execute_query_promise(stmt, loop = loop)$then(
+    onFulfilled = function(x) stop("should not be called"),
+    onRejected = function(x) {
+      async_called <<- TRUE
+    }
+  )
+
+  later_loop_wait(loop)
+  expect_true(async_called)
+})

From d43f5dbcd1066e4ed19b8fa5de9210802194b389 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Tue, 5 Sep 2023 12:57:35 -0300
Subject: [PATCH 09/36] test chaining

---
 .../tests/testthat/test-async.R               | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 84d90db244..b32b3acc7f 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -135,3 +135,28 @@ test_that("async adbc_execute_query() promises/later integration can error", {
   later_loop_wait(loop)
   expect_true(async_called)
 })
+
+test_that("execute_query and get_next promises can be chained", {
+  skip_if_not_installed("later")
+  skip_if_not_installed("promises")
+
+  db <- adbc_database_init(adbc_driver_monkey())
+  con <- adbc_connection_init(db)
+  input <- data.frame(x = 1:5)
+  stmt <- adbc_statement_init(con, input)
+
+  loop <- later::create_loop()
+  async_called <- FALSE
+
+  later::with_loop(loop, {
+    adbc_statement_execute_query_promise(stmt)$then(function(x) {
+      adbc_array_stream_get_next_promise(x)
+    })$then(function(x) {
+      async_called <<- TRUE
+      expect_identical(nanoarrow::convert_array(x), input)
+    })
+
+    later_loop_wait()
+    expect_true(async_called)
+  })
+})

From c05f11fcbf8d5fbb3d30c5ac7af75c7ba8969a13 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@voltrondata.com>
Date: Tue, 5 Sep 2023 12:58:39 -0300
Subject: [PATCH 10/36] fix error

---
 r/adbcdrivermanager/R/error.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/r/adbcdrivermanager/R/error.R b/r/adbcdrivermanager/R/error.R
index 0d7923e37a..397008edf3 100644
--- a/r/adbcdrivermanager/R/error.R
+++ b/r/adbcdrivermanager/R/error.R
@@ -58,6 +58,7 @@ stop_for_error <- function(status, error) {
       error <- list()
     }
 
+    error$status <- status
     error$status_code_message <- .Call(RAdbcStatusCodeMessage, status)
     if (!is.null(error$message)) {
       msg <- paste(error$status_code_message, error$message, sep=": ")

From 344814c58c0cbd1cfafa826cd4272fc9bc249c39 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Thu, 2 May 2024 23:14:28 -0300
Subject: [PATCH 11/36] fix init

---
 r/adbcdrivermanager/src/init.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index ad7ff6dcb9..4913bc2b57 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -20,6 +20,13 @@
 #include <Rinternals.h>
 
 /* generated by tools/make-callentries.R */
+SEXP RAdbcNewCallbackQueue(void);
+SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
+SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr, SEXP statement_xptr,
+                                     SEXP out_stream_xptr, SEXP error_xptr,
+                                     SEXP callback_env);
+SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr, SEXP array_stream_xptr,
+                                  SEXP array_xptr, SEXP callback_env);
 SEXP RAdbcVoidDriverInitFunc(void);
 SEXP RAdbcMonkeyDriverInitFunc(void);
 SEXP RAdbcLogDriverInitFunc(void);
@@ -102,6 +109,10 @@ SEXP RAdbcXptrEnv(SEXP xptr);
 SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot);
 
 static const R_CallMethodDef CallEntries[] = {
+    {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
+    {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
+    {"RAdbcStatementExecuteQueryAsync", (DL_FUNC)&RAdbcStatementExecuteQueryAsync, 5},
+    {"RAdbcArrayStreamGetNextAsync", (DL_FUNC)&RAdbcArrayStreamGetNextAsync, 4},
     {"RAdbcVoidDriverInitFunc", (DL_FUNC)&RAdbcVoidDriverInitFunc, 0},
     {"RAdbcMonkeyDriverInitFunc", (DL_FUNC)&RAdbcMonkeyDriverInitFunc, 0},
     {"RAdbcLogDriverInitFunc", (DL_FUNC)&RAdbcLogDriverInitFunc, 0},

From de938d9f5cd806b694e6345ee234dbd2df6237e4 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Fri, 3 May 2024 00:25:37 -0300
Subject: [PATCH 12/36] maybe start on a future-based method

---
 r/adbcdrivermanager/src/async.cc | 71 ++++++++++++++++++++++++++++++++
 r/adbcdrivermanager/src/init.c   |  6 +++
 2 files changed, 77 insertions(+)

diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 1cb20b2ce0..7404a7042f 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -21,6 +21,7 @@
 #include <Rinternals.h>
 
 #include <deque>
+#include <future>
 #include <mutex>
 #include <string>
 #include <thread>
@@ -30,6 +31,76 @@
 
 #include "radbc.h"
 
+struct RAdbcAsyncTask {
+  AdbcError* return_error{nullptr};
+  int* return_code{nullptr};
+  double* rows_affected{nullptr};
+  void* return_value_ptr{nullptr};
+
+  std::string error_message;
+  std::unique_ptr<std::future<void>> result;
+};
+
+template <>
+inline const char* adbc_xptr_class<RAdbcAsyncTask>() {
+  return "adbc_async_task";
+}
+
+static void FinalizeTaskXptr(SEXP task_xptr) {
+  auto task = reinterpret_cast<RAdbcAsyncTask*>(R_ExternalPtrAddr(task_xptr));
+  if (task != nullptr) {
+    delete task;
+  }
+}
+
+extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
+  const char* names[] = {"error_xptr", "return_code", "rows_affected", "result_xptr", ""};
+  SEXP task_prot = PROTECT(Rf_mkNamed(VECSXP, names));
+  SET_VECTOR_ELT(task_prot, 0, error_xptr);
+  SET_VECTOR_ELT(task_prot, 1, Rf_allocVector(INTSXP, 1));
+  SET_VECTOR_ELT(task_prot, 2, Rf_allocVector(REALSXP, 1));
+
+  auto task = new RAdbcAsyncTask();
+  SEXP task_xptr = PROTECT(R_MakeExternalPtr(task, R_NilValue, task_prot));
+  Rf_setAttrib(task_xptr, R_ClassSymbol, Rf_mkString("adbc_async_task"));
+  R_RegisterCFinalizer(task_xptr, &FinalizeTaskXptr);
+
+  task->return_error = adbc_from_xptr<AdbcError>(error_xptr);
+  task->return_code = INTEGER(VECTOR_ELT(task_prot, 1));
+  task->rows_affected = REAL(VECTOR_ELT(task_prot, 2));
+
+  UNPROTECT(1);
+  return R_NilValue;
+}
+
+extern "C" SEXP RAdbcAsyncTaskData(SEXP task_xptr) {
+  if (!Rf_inherits(task_xptr, adbc_xptr_class<RAdbcAsyncTask>())) {
+    Rf_error("task must inherit from 'adbc_async_task'");
+  }
+
+  return R_ExternalPtrProtected(task_xptr);
+}
+
+extern "C" SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  int duration_ms = adbc_as_int(duration_ms_sexp);
+
+  if (task->result == nullptr) {
+    return Rf_mkString("not_started");
+  }
+
+  std::future_status status =
+      task->result->wait_for(std::chrono::milliseconds(duration_ms));
+  switch (status) {
+    case std::future_status::timeout:
+      return Rf_mkString("timeout");
+    case std::future_status::ready:
+      return Rf_mkString("ready");
+    default:
+      return Rf_mkString("unknown");
+  }
+}
+
 // A thin wrapper around a std::thread() that ensures that the thread
 // does not leak. This could also maybe just be an external pointer to
 // a std::thread*. The external pointer to the Task holds a strong
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 4913bc2b57..0dbbc2fa5a 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -20,6 +20,9 @@
 #include <Rinternals.h>
 
 /* generated by tools/make-callentries.R */
+SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
+SEXP RAdbcAsyncTaskData(SEXP task_xptr);
+SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcNewCallbackQueue(void);
 SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
 SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr, SEXP statement_xptr,
@@ -109,6 +112,9 @@ SEXP RAdbcXptrEnv(SEXP xptr);
 SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot);
 
 static const R_CallMethodDef CallEntries[] = {
+    {"RAdbcAsyncTaskNew", (DL_FUNC)&RAdbcAsyncTaskNew, 1},
+    {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
+    {"RAdbcAsyncTaskWait", (DL_FUNC)&RAdbcAsyncTaskWait, 2},
     {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
     {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
     {"RAdbcStatementExecuteQueryAsync", (DL_FUNC)&RAdbcStatementExecuteQueryAsync, 5},

From 2eeae3f14a4b3342f501f4c8c5e4b20fb2e9451f Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sat, 4 May 2024 22:48:47 -0300
Subject: [PATCH 13/36] test initial infrastructure

---
 r/adbcdrivermanager/NAMESPACE                 |  3 ++
 r/adbcdrivermanager/R/async.R                 | 24 ++++++++++++++++
 r/adbcdrivermanager/src/async.cc              | 16 +++++++----
 .../tests/testthat/test-async.R               | 28 +++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 5c671a2b3d..2c1466ffcd 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -1,8 +1,10 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method("$",adbc_async_task)
 S3method("$",adbc_error)
 S3method("$",adbc_xptr)
 S3method("$<-",adbc_xptr)
+S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
@@ -21,6 +23,7 @@ S3method(execute_adbc,default)
 S3method(format,adbc_xptr)
 S3method(length,adbc_error)
 S3method(length,adbc_xptr)
+S3method(names,adbc_async_task)
 S3method(names,adbc_error)
 S3method(names,adbc_xptr)
 S3method(print,adbc_driver)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index ba95336adf..eb9e16c111 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -15,6 +15,30 @@
 # specific language governing permissions and limitations
 # under the License.
 
+
+adbc_async_task <- function() {
+  .Call(RAdbcAsyncTaskNew, adbc_allocate_error())
+}
+
+adbc_async_task_wait <- function(task, duration_ms) {
+  .Call(RAdbcAsyncTaskWait, task, duration_ms)
+}
+
+#' @export
+names.adbc_async_task <- function(x) {
+  names(.Call(RAdbcAsyncTaskData, x))
+}
+
+#' @export
+`[[.adbc_async_task` <- function(x, i) {
+  .Call(RAdbcAsyncTaskData, x)[[i]]
+}
+
+#' @export
+`$.adbc_async_task` <- function(x, name) {
+  .Call(RAdbcAsyncTaskData, x)[[name]]
+}
+
 adbc_callback_queue <- function() {
   .Call(RAdbcNewCallbackQueue)
 }
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 7404a7042f..cea812061e 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -69,15 +69,15 @@ extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
   task->return_code = INTEGER(VECTOR_ELT(task_prot, 1));
   task->rows_affected = REAL(VECTOR_ELT(task_prot, 2));
 
-  UNPROTECT(1);
-  return R_NilValue;
+  *(task->return_code) = NA_INTEGER;
+  *(task->rows_affected) = NA_REAL;
+
+  UNPROTECT(2);
+  return task_xptr;
 }
 
 extern "C" SEXP RAdbcAsyncTaskData(SEXP task_xptr) {
-  if (!Rf_inherits(task_xptr, adbc_xptr_class<RAdbcAsyncTask>())) {
-    Rf_error("task must inherit from 'adbc_async_task'");
-  }
-
+  adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
   return R_ExternalPtrProtected(task_xptr);
 }
 
@@ -85,6 +85,10 @@ extern "C" SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp) {
   auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
   int duration_ms = adbc_as_int(duration_ms_sexp);
 
+  if (duration_ms < 0) {
+    Rf_error("duration_ms must be >= 0");
+  }
+
   if (task->result == nullptr) {
     return Rf_mkString("not_started");
   }
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index b32b3acc7f..9c12b6d977 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -15,6 +15,34 @@
 # specific language governing permissions and limitations
 # under the License.
 
+test_that("async tasks can be created and inspected", {
+  task <- adbc_async_task()
+
+  expect_identical(
+    names(task),
+    c("error_xptr", "return_code", "rows_affected", "result_xptr")
+  )
+
+  expect_s3_class(task$error_xptr, "adbc_error")
+  expect_identical(task$return_code, NA_integer_)
+  expect_identical(task$rows_affected, NA_real_)
+  expect_identical(task$result_xptr, NULL)
+  expect_identical(adbc_async_task_wait(task, 0), "not_started")
+})
+
+test_that("async task methods error for invalid input", {
+  task <- unserialize(serialize(adbc_async_task(), NULL))
+  expect_error(
+    names(task),
+    "Can't convert external pointer to NULL"
+  )
+
+  expect_error(
+    adbc_async_task_wait(adbc_async_task(), -1),
+    "duration_ms must be >= 0"
+  )
+})
+
 test_that("async array_stream$get_next() works", {
   stream <- nanoarrow::basic_array_stream(list(1:5))
   async_called <- FALSE

From 0e86f354525adddb43adc833e9644ee44775ab12 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 00:09:30 -0300
Subject: [PATCH 14/36] dummy task

---
 r/adbcdrivermanager/R/async.R                 | 18 ++++++-
 r/adbcdrivermanager/src/async.cc              | 49 ++++++++++++++++---
 r/adbcdrivermanager/src/init.c                |  2 +
 r/adbcdrivermanager/src/radbc.h               | 19 ++++---
 .../tests/testthat/test-async.R               | 32 +++++++++++-
 5 files changed, 103 insertions(+), 17 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index eb9e16c111..f45f71783c 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -16,8 +16,11 @@
 # under the License.
 
 
-adbc_async_task <- function() {
-  .Call(RAdbcAsyncTaskNew, adbc_allocate_error())
+adbc_async_task <- function(subclass = character()) {
+  structure(
+    .Call(RAdbcAsyncTaskNew, adbc_allocate_error()),
+    class = union(subclass, "adbc_async_task")
+  )
 }
 
 adbc_async_task_wait <- function(task, duration_ms) {
@@ -39,6 +42,17 @@ names.adbc_async_task <- function(x) {
   .Call(RAdbcAsyncTaskData, x)[[name]]
 }
 
+adbc_async_sleep <- function(duration_ms) {
+  task <- adbc_async_task("adbc_async_sleep")
+  .Call(RAdbcAsyncTaskLaunchSleep, task, duration_ms)
+
+  user_data <- task$user_data
+  user_data$duration_ms <- duration_ms
+
+  task
+}
+
+
 adbc_callback_queue <- function() {
   .Call(RAdbcNewCallbackQueue)
 }
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index cea812061e..6ab63c7ca8 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -31,6 +31,8 @@
 
 #include "radbc.h"
 
+enum class RAdbcAsyncTaskStatus { NOT_STARTED, STARTED, READY };
+
 struct RAdbcAsyncTask {
   AdbcError* return_error{nullptr};
   int* return_code{nullptr};
@@ -38,7 +40,8 @@ struct RAdbcAsyncTask {
   void* return_value_ptr{nullptr};
 
   std::string error_message;
-  std::unique_ptr<std::future<void>> result;
+  RAdbcAsyncTaskStatus status;
+  std::future<void> result;
 };
 
 template <>
@@ -53,16 +56,26 @@ static void FinalizeTaskXptr(SEXP task_xptr) {
   }
 }
 
+static void error_for_started_task(RAdbcAsyncTask* task) {
+  if (task->result.valid()) {
+    Rf_error("adbc_async_task is already in use");
+  }
+}
+
 extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
-  const char* names[] = {"error_xptr", "return_code", "rows_affected", "result_xptr", ""};
+  const char* names[] = {"error_xptr",  "return_code", "rows_affected",
+                         "result_xptr", "user_data",   ""};
   SEXP task_prot = PROTECT(Rf_mkNamed(VECSXP, names));
   SET_VECTOR_ELT(task_prot, 0, error_xptr);
   SET_VECTOR_ELT(task_prot, 1, Rf_allocVector(INTSXP, 1));
   SET_VECTOR_ELT(task_prot, 2, Rf_allocVector(REALSXP, 1));
 
+  SEXP new_env = PROTECT(adbc_new_env());
+  SET_VECTOR_ELT(task_prot, 4, new_env);
+  UNPROTECT(1);
+
   auto task = new RAdbcAsyncTask();
   SEXP task_xptr = PROTECT(R_MakeExternalPtr(task, R_NilValue, task_prot));
-  Rf_setAttrib(task_xptr, R_ClassSymbol, Rf_mkString("adbc_async_task"));
   R_RegisterCFinalizer(task_xptr, &FinalizeTaskXptr);
 
   task->return_error = adbc_from_xptr<AdbcError>(error_xptr);
@@ -71,6 +84,7 @@ extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
 
   *(task->return_code) = NA_INTEGER;
   *(task->rows_affected) = NA_REAL;
+  task->status = RAdbcAsyncTaskStatus::NOT_STARTED;
 
   UNPROTECT(2);
   return task_xptr;
@@ -89,22 +103,43 @@ extern "C" SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp) {
     Rf_error("duration_ms must be >= 0");
   }
 
-  if (task->result == nullptr) {
-    return Rf_mkString("not_started");
+  switch (task->status) {
+    case RAdbcAsyncTaskStatus::NOT_STARTED:
+      return Rf_mkString("not_started");
+    case RAdbcAsyncTaskStatus::READY:
+      return Rf_mkString("ready");
+    default:
+      break;
   }
 
   std::future_status status =
-      task->result->wait_for(std::chrono::milliseconds(duration_ms));
+      task->result.wait_for(std::chrono::milliseconds(duration_ms));
   switch (status) {
     case std::future_status::timeout:
       return Rf_mkString("timeout");
     case std::future_status::ready:
+
       return Rf_mkString("ready");
     default:
-      return Rf_mkString("unknown");
+      Rf_error("Unknown status returned from future::wait_for()");
   }
 }
 
+extern "C" SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  error_for_started_task(task);
+
+  int duration_ms = adbc_as_int(duration_ms_sexp);
+
+  task->result = std::async(std::launch::async, [task, duration_ms] {
+    std::this_thread::sleep_for(std::chrono::milliseconds(duration_ms));
+    *(task->return_code) = ADBC_STATUS_OK;
+  });
+
+  task->status = RAdbcAsyncTaskStatus::STARTED;
+  return R_NilValue;
+}
+
 // A thin wrapper around a std::thread() that ensures that the thread
 // does not leak. This could also maybe just be an external pointer to
 // a std::thread*. The external pointer to the Task holds a strong
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 0dbbc2fa5a..178c03bd56 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -23,6 +23,7 @@
 SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp);
+SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcNewCallbackQueue(void);
 SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
 SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr, SEXP statement_xptr,
@@ -115,6 +116,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskNew", (DL_FUNC)&RAdbcAsyncTaskNew, 1},
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
     {"RAdbcAsyncTaskWait", (DL_FUNC)&RAdbcAsyncTaskWait, 2},
+    {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
     {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
     {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
     {"RAdbcStatementExecuteQueryAsync", (DL_FUNC)&RAdbcStatementExecuteQueryAsync, 5},
diff --git a/r/adbcdrivermanager/src/radbc.h b/r/adbcdrivermanager/src/radbc.h
index 4f1ec28317..72cf10c1a0 100644
--- a/r/adbcdrivermanager/src/radbc.h
+++ b/r/adbcdrivermanager/src/radbc.h
@@ -24,6 +24,17 @@
 
 #include <adbc.h>
 
+static inline SEXP adbc_new_env() {
+  SEXP new_env_sym = PROTECT(Rf_install("new_env"));
+  SEXP new_env_call = PROTECT(Rf_lang1(new_env_sym));
+  SEXP pkg_chr = PROTECT(Rf_mkString("adbcdrivermanager"));
+  SEXP pkg_ns = PROTECT(R_FindNamespace(pkg_chr));
+  SEXP new_env = PROTECT(Rf_eval(new_env_call, pkg_ns));
+  UNPROTECT(5);
+
+  return new_env;
+}
+
 template <typename T>
 static inline const char* adbc_xptr_class();
 
@@ -89,13 +100,9 @@ static inline SEXP adbc_borrow_xptr(T* ptr, SEXP shelter_sexp = R_NilValue) {
   Rf_setAttrib(xptr, R_ClassSymbol, xptr_class);
   UNPROTECT(1);
 
-  SEXP new_env_sym = PROTECT(Rf_install("new_env"));
-  SEXP new_env_call = PROTECT(Rf_lang1(new_env_sym));
-  SEXP pkg_chr = PROTECT(Rf_mkString("adbcdrivermanager"));
-  SEXP pkg_ns = PROTECT(R_FindNamespace(pkg_chr));
-  SEXP new_env = PROTECT(Rf_eval(new_env_call, pkg_ns));
+  SEXP new_env = PROTECT(adbc_new_env());
   R_SetExternalPtrTag(xptr, new_env);
-  UNPROTECT(5);
+  UNPROTECT(1);
 
   UNPROTECT(1);
   return xptr;
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 9c12b6d977..f130a417a6 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -16,20 +16,32 @@
 # under the License.
 
 test_that("async tasks can be created and inspected", {
-  task <- adbc_async_task()
+  task <- adbc_async_task(subclass = "specific_cls")
+  expect_s3_class(task, "adbc_async_task")
+  expect_s3_class(task, "specific_cls")
 
   expect_identical(
     names(task),
-    c("error_xptr", "return_code", "rows_affected", "result_xptr")
+    c("error_xptr", "return_code", "rows_affected", "result_xptr", "user_data")
   )
 
   expect_s3_class(task$error_xptr, "adbc_error")
   expect_identical(task$return_code, NA_integer_)
   expect_identical(task$rows_affected, NA_real_)
   expect_identical(task$result_xptr, NULL)
+
   expect_identical(adbc_async_task_wait(task, 0), "not_started")
 })
 
+test_that("async tasks can update R-level user data", {
+  task <- adbc_async_task()
+  expect_identical(as.list(task$user_data), list())
+
+  user_data <- task$user_data
+  user_data$some_field <- "some_value"
+  expect_identical(task$user_data$some_field, "some_value")
+})
+
 test_that("async task methods error for invalid input", {
   task <- unserialize(serialize(adbc_async_task(), NULL))
   expect_error(
@@ -41,8 +53,24 @@ test_that("async task methods error for invalid input", {
     adbc_async_task_wait(adbc_async_task(), -1),
     "duration_ms must be >= 0"
   )
+
+  task <- adbc_async_task()
+  expect_error(
+    task$result_xptr <- NULL,
+    "Can't update field"
+  )
 })
 
+test_that("async sleeper test works", {
+  sleep_task <- adbc_async_sleep(500)
+  expect_identical(adbc_async_task_wait(sleep_task, 0), "timeout")
+  expect_identical(adbc_async_task_wait(sleep_task, 1000), "ready")
+  expect_identical(adbc_async_task_wait(sleep_task, 0), "ready")
+  expect_identical(sleep_task$return_code, 0L)
+})
+
+
+
 test_that("async array_stream$get_next() works", {
   stream <- nanoarrow::basic_array_stream(list(1:5))
   async_called <- FALSE

From cd11f9373d05e9fd2711db13ec5b2cf515514246 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 00:31:03 -0300
Subject: [PATCH 15/36] better

---
 r/adbcdrivermanager/tests/testthat/test-async.R | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index f130a417a6..2aa2e0d96a 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -53,12 +53,6 @@ test_that("async task methods error for invalid input", {
     adbc_async_task_wait(adbc_async_task(), -1),
     "duration_ms must be >= 0"
   )
-
-  task <- adbc_async_task()
-  expect_error(
-    task$result_xptr <- NULL,
-    "Can't update field"
-  )
 })
 
 test_that("async sleeper test works", {

From d63942808ff77987554bb26ddfc8027990fb3d85 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 15:35:07 -0300
Subject: [PATCH 16/36] rename wait

---
 r/adbcdrivermanager/R/async.R                   |  4 ++--
 r/adbcdrivermanager/src/async.cc                |  2 +-
 r/adbcdrivermanager/src/init.c                  |  4 ++--
 r/adbcdrivermanager/tests/testthat/test-async.R | 10 +++++-----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index f45f71783c..4ba932c9a4 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -23,8 +23,8 @@ adbc_async_task <- function(subclass = character()) {
   )
 }
 
-adbc_async_task_wait <- function(task, duration_ms) {
-  .Call(RAdbcAsyncTaskWait, task, duration_ms)
+adbc_async_task_wait_for <- function(task, duration_ms) {
+  .Call(RAdbcAsyncTaskWaitFor, task, duration_ms)
 }
 
 #' @export
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 6ab63c7ca8..f16fd7d281 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -95,7 +95,7 @@ extern "C" SEXP RAdbcAsyncTaskData(SEXP task_xptr) {
   return R_ExternalPtrProtected(task_xptr);
 }
 
-extern "C" SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp) {
+extern "C" SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp) {
   auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
   int duration_ms = adbc_as_int(duration_ms_sexp);
 
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 178c03bd56..acc1e5d038 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -22,7 +22,7 @@
 /* generated by tools/make-callentries.R */
 SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
-SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP duration_ms_sexp);
+SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcNewCallbackQueue(void);
 SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
@@ -115,7 +115,7 @@ SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot);
 static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskNew", (DL_FUNC)&RAdbcAsyncTaskNew, 1},
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
-    {"RAdbcAsyncTaskWait", (DL_FUNC)&RAdbcAsyncTaskWait, 2},
+    {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
     {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
     {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 2aa2e0d96a..287bd4037b 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -30,7 +30,7 @@ test_that("async tasks can be created and inspected", {
   expect_identical(task$rows_affected, NA_real_)
   expect_identical(task$result_xptr, NULL)
 
-  expect_identical(adbc_async_task_wait(task, 0), "not_started")
+  expect_identical(adbc_async_task_wait_for(task, 0), "not_started")
 })
 
 test_that("async tasks can update R-level user data", {
@@ -50,16 +50,16 @@ test_that("async task methods error for invalid input", {
   )
 
   expect_error(
-    adbc_async_task_wait(adbc_async_task(), -1),
+    adbc_async_task_wait_for(adbc_async_task(), -1),
     "duration_ms must be >= 0"
   )
 })
 
 test_that("async sleeper test works", {
   sleep_task <- adbc_async_sleep(500)
-  expect_identical(adbc_async_task_wait(sleep_task, 0), "timeout")
-  expect_identical(adbc_async_task_wait(sleep_task, 1000), "ready")
-  expect_identical(adbc_async_task_wait(sleep_task, 0), "ready")
+  expect_identical(adbc_async_task_wait_for(sleep_task, 0), "timeout")
+  expect_identical(adbc_async_task_wait_for(sleep_task, 1000), "ready")
+  expect_identical(adbc_async_task_wait_for(sleep_task, 0), "ready")
   expect_identical(sleep_task$return_code, 0L)
 })
 

From 4282d124ad4716588224842e9e7b25f0306c5d90 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 15:48:16 -0300
Subject: [PATCH 17/36] waiter

---
 r/adbcdrivermanager/NAMESPACE                  |  1 +
 r/adbcdrivermanager/R/async.R                  | 18 ++++++++++++++++++
 .../tests/testthat/test-async.R                |  6 +++++-
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 2c1466ffcd..3833516b90 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -8,6 +8,7 @@ S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
+S3method(adbc_async_task_result,adbc_async_sleep)
 S3method(adbc_connection_init,adbc_database_log)
 S3method(adbc_connection_init,adbc_database_monkey)
 S3method(adbc_connection_init,default)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 4ba932c9a4..1831de1492 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -27,6 +27,19 @@ adbc_async_task_wait_for <- function(task, duration_ms) {
   .Call(RAdbcAsyncTaskWaitFor, task, duration_ms)
 }
 
+adbc_async_task_wait <- function(task, resolution_ms = 100) {
+  status <- "started"
+  while (status != "ready") {
+    status <- adbc_async_task_wait_for(task, resolution_ms)
+  }
+
+  adbc_async_task_result(task)
+}
+
+adbc_async_task_result <- function(task) {
+  UseMethod("adbc_async_task_result")
+}
+
 #' @export
 names.adbc_async_task <- function(x) {
   names(.Call(RAdbcAsyncTaskData, x))
@@ -52,6 +65,11 @@ adbc_async_sleep <- function(duration_ms) {
   task
 }
 
+#' @export
+adbc_async_task_result.adbc_async_sleep <- function(task) {
+  task$user_data$duration_ms
+}
+
 
 adbc_callback_queue <- function() {
   .Call(RAdbcNewCallbackQueue)
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 287bd4037b..44050b4c3b 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -61,9 +61,13 @@ test_that("async sleeper test works", {
   expect_identical(adbc_async_task_wait_for(sleep_task, 1000), "ready")
   expect_identical(adbc_async_task_wait_for(sleep_task, 0), "ready")
   expect_identical(sleep_task$return_code, 0L)
+  expect_identical(adbc_async_task_result(sleep_task), 500)
 })
 
-
+test_that("async task waiter works", {
+  sleep_task <- adbc_async_sleep(500)
+  expect_identical(adbc_async_task_wait(sleep_task), 500)
+})
 
 test_that("async array_stream$get_next() works", {
   stream <- nanoarrow::basic_array_stream(list(1:5))

From 56c3903a637b02f79074cb68a02dcdb1413008b5 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 16:02:08 -0300
Subject: [PATCH 18/36] scheduler

---
 r/adbcdrivermanager/R/async.R | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 1831de1492..e6ab2caa05 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -36,6 +36,22 @@ adbc_async_task_wait <- function(task, resolution_ms = 100) {
   adbc_async_task_result(task)
 }
 
+later_loop_schedule_task_callback <- function(task, callback,
+                                              loop = later::current_loop(),
+                                              delay = 0) {
+  force(task)
+  force(callback)
+
+  later::later(function() {
+    status <- adbc_async_task_wait_for(task, 0)
+    if (status == "timeout") {
+      later_loop_schedule_task_callback(task, callback, loop = loop, delay = delay)
+    } else {
+      callback(adbc_async_task_result(task))
+    }
+  }, delay = delay, loop = loop)
+}
+
 adbc_async_task_result <- function(task) {
   UseMethod("adbc_async_task_result")
 }

From bf11de42289942a61fc5b832e15216868680107a Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 21:59:19 -0300
Subject: [PATCH 19/36] maybe with error support

---
 r/adbcdrivermanager/R/async.R                 | 34 ++++++++--
 r/adbcdrivermanager/R/zzz.R                   | 67 +++++++++++++++++++
 .../tests/testthat/test-async.R               | 38 +++++++++++
 3 files changed, 134 insertions(+), 5 deletions(-)
 create mode 100644 r/adbcdrivermanager/R/zzz.R

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index e6ab2caa05..0f83f8c845 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -36,22 +36,39 @@ adbc_async_task_wait <- function(task, resolution_ms = 100) {
   adbc_async_task_result(task)
 }
 
-later_loop_schedule_task_callback <- function(task, callback,
+later_loop_schedule_task_callback <- function(task, resolve, reject,
                                               loop = later::current_loop(),
                                               delay = 0) {
   force(task)
-  force(callback)
+  force(resolve)
+  force(reject)
 
   later::later(function() {
     status <- adbc_async_task_wait_for(task, 0)
     if (status == "timeout") {
-      later_loop_schedule_task_callback(task, callback, loop = loop, delay = delay)
+      later_loop_schedule_task_callback(
+        task,
+        resolve,
+        reject,
+        loop = loop,
+        delay = delay
+      )
     } else {
-      callback(adbc_async_task_result(task))
+      tryCatch(
+        resolve(adbc_async_task_result(task)),
+        error = function(e) reject(e)
+      )
     }
   }, delay = delay, loop = loop)
 }
 
+as.promise.adbc_async_task <- function(task) {
+  force(task)
+  promises::promise(function(resolve, reject) {
+    later_loop_schedule_task_callback(task, resolve, reject)
+  })
+}
+
 adbc_async_task_result <- function(task) {
   UseMethod("adbc_async_task_result")
 }
@@ -71,18 +88,25 @@ names.adbc_async_task <- function(x) {
   .Call(RAdbcAsyncTaskData, x)[[name]]
 }
 
-adbc_async_sleep <- function(duration_ms) {
+adbc_async_sleep <- function(duration_ms, error_message = NULL) {
   task <- adbc_async_task("adbc_async_sleep")
   .Call(RAdbcAsyncTaskLaunchSleep, task, duration_ms)
 
   user_data <- task$user_data
   user_data$duration_ms <- duration_ms
+  user_data$error_message <- error_message
 
   task
 }
 
 #' @export
 adbc_async_task_result.adbc_async_sleep <- function(task) {
+  if (!is.null(task$user_data$error_message)) {
+    cnd <- simpleError(task$user_data$error_message)
+    class(cnd) <- c("adbc_async_sleep_error", class(cnd))
+    stop(cnd)
+  }
+
   task$user_data$duration_ms
 }
 
diff --git a/r/adbcdrivermanager/R/zzz.R b/r/adbcdrivermanager/R/zzz.R
new file mode 100644
index 0000000000..82901294fb
--- /dev/null
+++ b/r/adbcdrivermanager/R/zzz.R
@@ -0,0 +1,67 @@
+
+.onLoad <- function(...) {
+  s3_register("promises::as.promise", "adbc_async_task")
+}
+
+# From the `vctrs` package (this function is intended to be copied
+# without attribution or license requirements to avoid a hard dependency on
+# vctrs:
+# https://github.com/r-lib/vctrs/blob/c2a7710fe55e3a2249c4fdfe75bbccbafcf38804/R/register-s3.R#L25-L31
+s3_register <- function(generic, class, method = NULL) {
+  stopifnot(is.character(generic), length(generic) == 1)
+  stopifnot(is.character(class), length(class) == 1)
+
+  pieces <- strsplit(generic, "::")[[1]]
+  stopifnot(length(pieces) == 2)
+  package <- pieces[[1]]
+  generic <- pieces[[2]]
+
+  caller <- parent.frame()
+
+  get_method_env <- function() {
+    top <- topenv(caller)
+    if (isNamespace(top)) {
+      asNamespace(environmentName(top))
+    } else {
+      caller
+    }
+  }
+  get_method <- function(method, env) {
+    if (is.null(method)) {
+      get(paste0(generic, ".", class), envir = get_method_env())
+    } else {
+      method
+    }
+  }
+
+  register <- function(...) {
+    envir <- asNamespace(package)
+
+    # Refresh the method each time, it might have been updated by
+    # `devtools::load_all()`
+    method_fn <- get_method(method)
+    stopifnot(is.function(method_fn))
+
+
+    # Only register if generic can be accessed
+    if (exists(generic, envir)) {
+      registerS3method(generic, class, method_fn, envir = envir)
+    } else if (identical(Sys.getenv("NOT_CRAN"), "true")) {
+      warning(sprintf(
+        "Can't find generic `%s` in package %s to register S3 method.",
+        generic,
+        package
+      ))
+    }
+  }
+
+  # Always register hook in case package is later unloaded & reloaded
+  setHook(packageEvent(package, "onLoad"), register)
+
+  # Avoid registration failures during loading (pkgload or regular)
+  if (isNamespaceLoaded(package)) {
+    register()
+  }
+
+  invisible()
+}
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 44050b4c3b..159882b3db 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -67,6 +67,44 @@ test_that("async sleeper test works", {
 test_that("async task waiter works", {
   sleep_task <- adbc_async_sleep(500)
   expect_identical(adbc_async_task_wait(sleep_task), 500)
+
+  erroring_sleep_task <- adbc_async_sleep(500, error_message = "some error")
+  expect_error(
+    adbc_async_task_wait(erroring_sleep_task),
+    "some error",
+    class = "adbc_async_sleep_error"
+  )
+})
+
+test_that("async task can be converted to a promise", {
+  skip_if_not_installed("promises")
+
+  expect_output(
+    {
+      adbc_async_sleep(100) |>
+        promises::as.promise() |>
+        promises::then(~print(sprintf("waited for %s ms", .x)))
+      Sys.sleep(0.2)
+      later::run_now()
+      later::run_now()
+    },
+    "waited for 100 ms"
+  )
+
+  expect_output(
+    {
+      adbc_async_sleep(100, error_message = "errored after 100 ms") |>
+        promises::as.promise() |>
+        promises::then(
+          onFulfilled = ~print(sprintf("waited for %s ms", .x)),
+          onRejected = ~print(.x)
+        )
+      Sys.sleep(0.2)
+      later::run_now()
+      later::run_now()
+    },
+    "errored after 100 ms"
+  )
 })
 
 test_that("async array_stream$get_next() works", {

From 69590f97999d93cb00eba1f77a08a084e6639210 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 22:12:50 -0300
Subject: [PATCH 20/36] remove previous

---
 .../tests/testthat/test-async.R               | 155 +-----------------
 1 file changed, 5 insertions(+), 150 deletions(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 159882b3db..d7076ba33b 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -81,9 +81,10 @@ test_that("async task can be converted to a promise", {
 
   expect_output(
     {
-      adbc_async_sleep(100) |>
-        promises::as.promise() |>
+      adbc_async_sleep(100) %>%
+        promises::as.promise() %>%
         promises::then(~print(sprintf("waited for %s ms", .x)))
+
       Sys.sleep(0.2)
       later::run_now()
       later::run_now()
@@ -93,8 +94,8 @@ test_that("async task can be converted to a promise", {
 
   expect_output(
     {
-      adbc_async_sleep(100, error_message = "errored after 100 ms") |>
-        promises::as.promise() |>
+      adbc_async_sleep(100, error_message = "errored after 100 ms") %>%
+        promises::as.promise() %>%
         promises::then(
           onFulfilled = ~print(sprintf("waited for %s ms", .x)),
           onRejected = ~print(.x)
@@ -106,149 +107,3 @@ test_that("async task can be converted to a promise", {
     "errored after 100 ms"
   )
 })
-
-test_that("async array_stream$get_next() works", {
-  stream <- nanoarrow::basic_array_stream(list(1:5))
-  async_called <- FALSE
-  queue <- adbc_array_stream_get_next_async(stream, function(array) {
-    async_called <<- TRUE
-    expect_identical(nanoarrow::convert_array(array), 1:5)
-  })
-
-  expect_false(async_called)
-  expect_identical(adbc_callback_queue_run_pending(queue), 1)
-  expect_true(async_called)
-})
-
-test_that("async adbc_statement_execute_query() can return a value", {
-  db <- adbc_database_init(adbc_driver_monkey())
-  con <- adbc_connection_init(db)
-  input <- data.frame(x = 1:5)
-  stmt <- adbc_statement_init(con, input)
-
-  async_called <- FALSE
-  queue <- adbc_statement_execute_query_async(stmt, function(stream) {
-    async_called <<- TRUE
-    expect_identical(nanoarrow::convert_array_stream(stream), input)
-  })
-
-  expect_false(async_called)
-  expect_identical(adbc_callback_queue_run_pending(queue), 1)
-  expect_true(async_called)
-})
-
-test_that("async adbc_statement_execute_query() can return an error", {
-  db <- adbc_database_init(adbc_driver_void())
-  con <- adbc_connection_init(db)
-  stmt <- adbc_statement_init(con)
-
-  async_called <- FALSE
-  queue <- adbc_statement_execute_query_async(
-    stmt,
-    adbc_callback(
-      on_success = function(stream) {
-        stop("This should not be called")
-      },
-      on_error = function(status, error) {
-        expect_false(status == 0)
-        expect_s3_class(error, "adbc_error")
-        async_called <<- TRUE
-        stop_for_error(status, error)
-      }
-    )
-  )
-
-  expect_false(async_called)
-  expect_error(
-    adbc_callback_queue_run_pending(queue),
-    class = "adbc_status_not_implemented"
-  )
-  expect_true(async_called)
-  expect_identical(adbc_callback_queue_run_pending(queue), 0)
-})
-
-test_that("async array_stream$get_next() promises/later integration works", {
-  skip_if_not_installed("later")
-  skip_if_not_installed("promises")
-
-  stream <- nanoarrow::basic_array_stream(list(1:5))
-  loop <- later::create_loop()
-  async_called <- FALSE
-
-  promise <- adbc_array_stream_get_next_promise(stream, loop = loop)
-  promise$then(function(x) {
-    async_called <<- TRUE
-    expect_identical(nanoarrow::convert_array(x), 1:5)
-  })
-
-  later_loop_wait(loop)
-  expect_true(async_called)
-})
-
-test_that("async adbc_execute_query() promises/later integration works", {
-  skip_if_not_installed("later")
-  skip_if_not_installed("promises")
-
-  db <- adbc_database_init(adbc_driver_monkey())
-  con <- adbc_connection_init(db)
-  input <- data.frame(x = 1:5)
-  stmt <- adbc_statement_init(con, input)
-
-  loop <- later::create_loop()
-  async_called <- FALSE
-
-  adbc_statement_execute_query_promise(stmt, loop = loop)$then(function(x) {
-    async_called <<- TRUE
-    expect_identical(nanoarrow::convert_array_stream(x), input)
-  })
-
-  later_loop_wait(loop)
-  expect_true(async_called)
-})
-
-test_that("async adbc_execute_query() promises/later integration can error", {
-  skip_if_not_installed("later")
-  skip_if_not_installed("promises")
-
-  db <- adbc_database_init(adbc_driver_void())
-  con <- adbc_connection_init(db)
-  stmt <- adbc_statement_init(con)
-
-  loop <- later::create_loop()
-  async_called <- FALSE
-
-  adbc_statement_execute_query_promise(stmt, loop = loop)$then(
-    onFulfilled = function(x) stop("should not be called"),
-    onRejected = function(x) {
-      async_called <<- TRUE
-    }
-  )
-
-  later_loop_wait(loop)
-  expect_true(async_called)
-})
-
-test_that("execute_query and get_next promises can be chained", {
-  skip_if_not_installed("later")
-  skip_if_not_installed("promises")
-
-  db <- adbc_database_init(adbc_driver_monkey())
-  con <- adbc_connection_init(db)
-  input <- data.frame(x = 1:5)
-  stmt <- adbc_statement_init(con, input)
-
-  loop <- later::create_loop()
-  async_called <- FALSE
-
-  later::with_loop(loop, {
-    adbc_statement_execute_query_promise(stmt)$then(function(x) {
-      adbc_array_stream_get_next_promise(x)
-    })$then(function(x) {
-      async_called <<- TRUE
-      expect_identical(nanoarrow::convert_array(x), input)
-    })
-
-    later_loop_wait()
-    expect_true(async_called)
-  })
-})

From 48d53ec27bae11f6b6dce9009c521d024f9be1a3 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 22:13:54 -0300
Subject: [PATCH 21/36] remove previous

---
 r/adbcdrivermanager/R/async.R | 154 ----------------------------------
 1 file changed, 154 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 0f83f8c845..670ee20747 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -109,157 +109,3 @@ adbc_async_task_result.adbc_async_sleep <- function(task) {
 
   task$user_data$duration_ms
 }
-
-
-adbc_callback_queue <- function() {
-  .Call(RAdbcNewCallbackQueue)
-}
-
-adbc_callback_queue_run_pending <- function(queue) {
-  .Call(RAdbcCallbackQueueRunPending, queue)
-}
-
-adbc_array_stream_get_next_async <- function(stream, callback,
-                                             schema = stream$get_schema(),
-                                             queue = adbc_callback_queue()) {
-  callback <- as_adbc_callback(callback)
-  callback$args <- list(stream)
-  array <- nanoarrow::nanoarrow_allocate_array()
-  nanoarrow::nanoarrow_array_set_schema(array, schema, validate = FALSE)
-  .Call(RAdbcArrayStreamGetNextAsync, queue, stream, array, callback)
-
-  invisible(queue)
-}
-
-adbc_statement_execute_query_async <- function(statement, callback,
-                                               queue = adbc_callback_queue()) {
-  callback <- as_adbc_callback(callback)
-  stream_out <- nanoarrow::nanoarrow_allocate_array_stream()
-  callback$args <- list(statement, stream_out)
-
-  error <- adbc_allocate_error()
-  .Call(RAdbcStatementExecuteQueryAsync, queue, statement, stream_out, error, callback)
-
-  invisible(queue)
-}
-
-adbc_array_stream_get_next_promise <- function(stream,
-                                               schema = stream$get_schema(),
-                                               loop = NULL, delay = 0) {
-  if (is.null(loop)) {
-    loop <- later::current_loop()
-  }
-
-  queue <- callback_queue_from_later_loop(loop)
-
-  later_loop_schedule_run_pending(loop, delay = delay)
-
-  promises::promise(function(resolve, reject) {
-    callback <- adbc_callback(
-      on_success = resolve,
-      on_error = function(status, error) {
-        reject(adbc_error_message(status, error))
-      }
-    )
-
-    adbc_array_stream_get_next_async(
-      stream,
-      callback,
-      schema = schema,
-      queue = queue
-    )
-  })
-}
-
-adbc_statement_execute_query_promise <- function(statement, loop = NULL, delay = 0) {
-  if (is.null(loop)) {
-    loop <- later::current_loop()
-  }
-
-  queue <- callback_queue_from_later_loop(loop)
-
-  later_loop_schedule_run_pending(loop, delay = delay)
-
-  promises::promise(function(resolve, reject) {
-    callback <- adbc_callback(
-      on_success = resolve,
-      on_error = function(status, error) {
-        reject(adbc_error_message(status, error))
-      }
-    )
-
-    adbc_statement_execute_query_async(statement, callback, queue = queue)
-  })
-}
-
-later_loop_schedule_run_pending <- function(loop, delay = 0) {
-  later::later(function() {
-    queue <- callback_queue_from_later_loop(loop)
-    n_run <- adbc_callback_queue_run_pending(queue)
-    if (n_run > 0) {
-      later_loop_schedule_run_pending(loop, delay = delay)
-    } else {
-      string_id <- as.character(loop$id)
-      callback_queue_cache[[string_id]] <- adbc_callback_queue()
-    }
-  }, delay = delay, loop = loop)
-}
-
-callback_queue_from_later_loop <- function(loop) {
-  string_id <- as.character(loop$id)
-  if (is.null(callback_queue_cache[[string_id]])) {
-    callback_queue_cache[[string_id]] <- adbc_callback_queue()
-  }
-
-  callback_queue_cache[[string_id]]
-}
-
-callback_queue_cache <- new.env(parent = emptyenv())
-
-# test helper to force a promise to resolve
-later_loop_wait <- function(loop = NULL, timeout = 10, delay = 0.01) {
-  if (is.null(loop)) {
-    loop <- later::current_loop()
-  }
-
-  start <- Sys.time()
-  while (!later::loop_empty(loop)) {
-    elapsed <- difftime(Sys.time(), start, units = "secs")
-    if (elapsed > timeout) {
-      stop(sprintf("later_loop_wait() timed out at %s seconds", elapsed))
-    }
-
-    later::run_now(loop = loop)
-    Sys.sleep(delay)
-  }
-
-  invisible(loop)
-}
-
-as_adbc_callback <- function(x) {
-  if (inherits(x, "adbc_callback")) {
-    x
-  } else if (is.function(x)) {
-    adbc_callback(x)
-  } else {
-    stop(sprintf("Can't create adbc_callback() from object of type '%s'", class(x)[1]))
-  }
-}
-
-adbc_callback <- function(on_success, ..., on_error = stop_for_error) {
-  callback_env <- new.env(parent = emptyenv())
-  callback_env$sheltered_objects = list(...)
-
-  force(on_success)
-  force(on_error)
-  callback_env$callback <- function(status, error, return_value_xptr) {
-    if (!identical(status, 0L)) {
-      on_error(status, error)
-    } else {
-      on_success(return_value_xptr)
-    }
-  }
-
-  class(callback_env) <- "adbc_callback"
-  callback_env
-}

From 8ecfb52f07a84c0be599369f26d8b5aae9fe91ef Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 23:31:58 -0300
Subject: [PATCH 22/36] with one working method

---
 r/adbcdrivermanager/NAMESPACE    |  1 +
 r/adbcdrivermanager/R/async.R    | 20 ++++++++++++++++++++
 r/adbcdrivermanager/src/async.cc | 22 +++++++++++++++++++++-
 r/adbcdrivermanager/src/init.c   |  3 +++
 4 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 3833516b90..8bec2e9b10 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -8,6 +8,7 @@ S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
+S3method(adbc_async_task_result,adbc_async_execute_query_rows_affected)
 S3method(adbc_async_task_result,adbc_async_sleep)
 S3method(adbc_connection_init,adbc_database_log)
 S3method(adbc_connection_init,adbc_database_monkey)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 670ee20747..49df87deea 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -109,3 +109,23 @@ adbc_async_task_result.adbc_async_sleep <- function(task) {
 
   task$user_data$duration_ms
 }
+
+adbc_statement_execute_query_rows_affected_async <- function(statement) {
+  task <- adbc_async_task("adbc_async_execute_query_rows_affected")
+
+  user_data <- task$user_data
+  user_data$statement <- statement
+
+  .Call(RAdbcAsyncTaskLaunchExecuteQuery, task, statement, NULL)
+
+  task
+}
+
+#' @export
+adbc_async_task_result.adbc_async_execute_query_rows_affected <- function(task) {
+  if (!identical(task$return_code, 0L)) {
+    stop_for_error(task$return_code, task$error_xptr)
+  }
+
+  task$rows_affected
+}
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index f16fd7d281..1c198b1704 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -37,7 +37,8 @@ struct RAdbcAsyncTask {
   AdbcError* return_error{nullptr};
   int* return_code{nullptr};
   double* rows_affected{nullptr};
-  void* return_value_ptr{nullptr};
+  AdbcStatement* statement{nullptr};
+  ArrowArrayStream* stream{nullptr};
 
   std::string error_message;
   RAdbcAsyncTaskStatus status;
@@ -140,6 +141,25 @@ extern "C" SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp)
   return R_NilValue;
 }
 
+extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
+                                                 SEXP stream_xptr) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  error_for_started_task(task);
+
+  auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
+  //auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr, /*null_ok*/ true);
+
+  task->result = std::async(std::launch::async, [task, statement] {
+    int64_t rows_affected = -1;
+    *(task->return_code) =
+        AdbcStatementExecuteQuery(statement, nullptr, &rows_affected, task->return_error);
+    *(task->rows_affected) = static_cast<double>(rows_affected);
+  });
+
+  task->status = RAdbcAsyncTaskStatus::STARTED;
+  return R_NilValue;
+}
+
 // A thin wrapper around a std::thread() that ensures that the thread
 // does not leak. This could also maybe just be an external pointer to
 // a std::thread*. The external pointer to the Task holds a strong
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index acc1e5d038..c3dbf45a31 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -24,6 +24,8 @@ SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
+SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
+                                      SEXP stream_xptr);
 SEXP RAdbcNewCallbackQueue(void);
 SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
 SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr, SEXP statement_xptr,
@@ -117,6 +119,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
+    {"RAdbcAsyncTaskLaunchExecuteQuery", (DL_FUNC)&RAdbcAsyncTaskLaunchExecuteQuery, 3},
     {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
     {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
     {"RAdbcStatementExecuteQueryAsync", (DL_FUNC)&RAdbcStatementExecuteQueryAsync, 5},

From add8551a1f698f4e71460a79c94c792f8b6d214a Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 23:44:37 -0300
Subject: [PATCH 23/36] better separation of tasks

---
 r/adbcdrivermanager/R/async.R                 |  9 ++++--
 r/adbcdrivermanager/src/async.cc              | 30 +++++++++----------
 .../tests/testthat/test-async.R               |  4 +--
 3 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 49df87deea..6461da319b 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -116,7 +116,12 @@ adbc_statement_execute_query_rows_affected_async <- function(statement) {
   user_data <- task$user_data
   user_data$statement <- statement
 
-  .Call(RAdbcAsyncTaskLaunchExecuteQuery, task, statement, NULL)
+  user_data$rows_affected <- .Call(
+    RAdbcAsyncTaskLaunchExecuteQuery,
+    task,
+    statement,
+    NULL
+  )
 
   task
 }
@@ -127,5 +132,5 @@ adbc_async_task_result.adbc_async_execute_query_rows_affected <- function(task)
     stop_for_error(task$return_code, task$error_xptr)
   }
 
-  task$rows_affected
+  task$user_data$rows_affected
 }
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 1c198b1704..4713443eb5 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -36,11 +36,7 @@ enum class RAdbcAsyncTaskStatus { NOT_STARTED, STARTED, READY };
 struct RAdbcAsyncTask {
   AdbcError* return_error{nullptr};
   int* return_code{nullptr};
-  double* rows_affected{nullptr};
-  AdbcStatement* statement{nullptr};
-  ArrowArrayStream* stream{nullptr};
 
-  std::string error_message;
   RAdbcAsyncTaskStatus status;
   std::future<void> result;
 };
@@ -64,15 +60,17 @@ static void error_for_started_task(RAdbcAsyncTask* task) {
 }
 
 extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
-  const char* names[] = {"error_xptr",  "return_code", "rows_affected",
-                         "result_xptr", "user_data",   ""};
+  const char* names[] = {"error_xptr", "return_code", "user_data", ""};
   SEXP task_prot = PROTECT(Rf_mkNamed(VECSXP, names));
+
   SET_VECTOR_ELT(task_prot, 0, error_xptr);
-  SET_VECTOR_ELT(task_prot, 1, Rf_allocVector(INTSXP, 1));
-  SET_VECTOR_ELT(task_prot, 2, Rf_allocVector(REALSXP, 1));
+
+  SEXP return_code_sexp = PROTECT(Rf_allocVector(INTSXP, 1));
+  SET_VECTOR_ELT(task_prot, 1, return_code_sexp);
+  UNPROTECT(1);
 
   SEXP new_env = PROTECT(adbc_new_env());
-  SET_VECTOR_ELT(task_prot, 4, new_env);
+  SET_VECTOR_ELT(task_prot, 2, new_env);
   UNPROTECT(1);
 
   auto task = new RAdbcAsyncTask();
@@ -81,10 +79,8 @@ extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
 
   task->return_error = adbc_from_xptr<AdbcError>(error_xptr);
   task->return_code = INTEGER(VECTOR_ELT(task_prot, 1));
-  task->rows_affected = REAL(VECTOR_ELT(task_prot, 2));
 
   *(task->return_code) = NA_INTEGER;
-  *(task->rows_affected) = NA_REAL;
   task->status = RAdbcAsyncTaskStatus::NOT_STARTED;
 
   UNPROTECT(2);
@@ -147,17 +143,21 @@ extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_
   error_for_started_task(task);
 
   auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
-  //auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr, /*null_ok*/ true);
+  // auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr, /*null_ok*/ true);
 
-  task->result = std::async(std::launch::async, [task, statement] {
+  SEXP rows_affected_sexp = PROTECT(Rf_allocVector(REALSXP, 1));
+  double* rows_affected_dbl = REAL(rows_affected_sexp);
+
+  task->result = std::async(std::launch::async, [task, statement, rows_affected_dbl] {
     int64_t rows_affected = -1;
     *(task->return_code) =
         AdbcStatementExecuteQuery(statement, nullptr, &rows_affected, task->return_error);
-    *(task->rows_affected) = static_cast<double>(rows_affected);
+    *rows_affected_dbl = static_cast<double>(rows_affected);
   });
 
   task->status = RAdbcAsyncTaskStatus::STARTED;
-  return R_NilValue;
+  UNPROTECT(1);
+  return rows_affected_sexp;
 }
 
 // A thin wrapper around a std::thread() that ensures that the thread
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index d7076ba33b..5c6f248413 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -22,13 +22,11 @@ test_that("async tasks can be created and inspected", {
 
   expect_identical(
     names(task),
-    c("error_xptr", "return_code", "rows_affected", "result_xptr", "user_data")
+    c("error_xptr", "return_code", "user_data")
   )
 
   expect_s3_class(task$error_xptr, "adbc_error")
   expect_identical(task$return_code, NA_integer_)
-  expect_identical(task$rows_affected, NA_real_)
-  expect_identical(task$result_xptr, NULL)
 
   expect_identical(adbc_async_task_wait_for(task, 0), "not_started")
 })

From 473d053cd93e5eb0e3b1588bc2bba195f9fe5312 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 5 May 2024 23:46:07 -0300
Subject: [PATCH 24/36] remove old

---
 r/adbcdrivermanager/src/async.cc | 231 -------------------------------
 r/adbcdrivermanager/src/init.c   |  11 --
 2 files changed, 242 deletions(-)

diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 4713443eb5..6fddf02fe8 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -159,234 +159,3 @@ extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_
   UNPROTECT(1);
   return rows_affected_sexp;
 }
-
-// A thin wrapper around a std::thread() that ensures that the thread
-// does not leak. This could also maybe just be an external pointer to
-// a std::thread*. The external pointer to the Task holds a strong
-// reference to the external pointer to the CallbackQueue that is
-// released just before the callback is run.
-class Task {
- public:
-  Task() : worker(nullptr) {}
-
-  std::thread* worker;
-
-  static void FinalizeXptr(SEXP xptr) {
-    Task* task = reinterpret_cast<Task*>(R_ExternalPtrAddr(xptr));
-    if (task->worker != nullptr) {
-      // TODO: check task->worker->joinable()?
-      task->worker->join();
-      delete task->worker;
-    }
-
-    delete task;
-  }
-
-  static SEXP MakeXptr(SEXP shelter = R_NilValue) {
-    SEXP xptr = PROTECT(R_MakeExternalPtr(new Task(), R_NilValue, shelter));
-    R_RegisterCFinalizer(xptr, &FinalizeXptr);
-    UNPROTECT(1);
-    return xptr;
-  }
-};
-
-// A thread-safe queue of callbacks to execute.
-class CallbackQueue {
- public:
-  // Because the RCallback is used in C++ frames where a longjmp may occur,
-  // the members of this struct must be trivially destructible:
-  // Any destruction that needs to occur must occur via an SEXP
-  // finalizer set on the return_value_xptr. This struct is intentionally
-  // copyable and is passed by value.
-  struct RCallback {
-    // An environment containing a function named "callback". This callback
-    // is executed as callback(return_code, error_xptr, return_value_xptr). The
-    // environment may contain other items that need to stay valid for the lifetime of the
-    // task (e.g., inputs).
-    SEXP env_sexp;
-
-    // An external pointer to an AdbcError*
-    SEXP error_xptr;
-
-    // An external pointer to return_value_ptr (with proper finalizer set as applicable)
-    SEXP return_value_xptr;
-
-    // A return code (e.g., AdbcStatusCode). A return code of 0 indicates success.
-    int return_code;
-
-    // The external pointer address of error_xptr
-    AdbcError* return_error;
-
-    // The external pointer address of return_value_xptr
-    void* return_value_ptr;
-  };
-
-  // Initialize a callback and preserve its SEXP members. This must
-  // be called from the main R thread.
-  RCallback InitCallback(SEXP callback_env, SEXP return_value_xptr = R_NilValue,
-                         SEXP error_xptr = R_NilValue) {
-    RCallback out{callback_env, error_xptr, return_value_xptr,
-                  NA_INTEGER,   nullptr,    nullptr};
-    if (error_xptr != R_NilValue) {
-      out.return_error = reinterpret_cast<AdbcError*>(R_ExternalPtrAddr(error_xptr));
-    }
-
-    if (return_value_xptr != R_NilValue) {
-      out.return_value_ptr = R_ExternalPtrAddr(return_value_xptr);
-    }
-
-    R_PreserveObject(out.env_sexp);
-    R_PreserveObject(out.error_xptr);
-    R_PreserveObject(out.return_value_xptr);
-
-    return out;
-  }
-
-  // Add a callback to the queue with thread safety. This can
-  // (should) be called from another thread.
-  void AddCallback(RCallback callback) {
-    std::lock_guard<std::mutex> lock(callbacks_lock_);
-    pending_callbacks_.push_back(callback);
-  }
-
-  // Try to run all callbacks. Callbacks should be written such that
-  // they do not error; however, this function is written such that
-  // an erroring callback will simply result in a (potentially)
-  // incompletely executed callback queue.
-  int64_t RunPending() {
-    int64_t n_run = 0;
-    while (!pending_callbacks_.empty()) {
-      // RunCallback() may may longjmp
-      RCallback callback = GetNextCallback();
-      RunCallback(callback);
-      n_run++;
-    }
-
-    return n_run;
-  }
-
-  // Pop a callback from the end of the queue with thread safety.
-  RCallback GetNextCallback() {
-    std::lock_guard<std::mutex> lock(callbacks_lock_);
-    RCallback callback = pending_callbacks_.front();
-    pending_callbacks_.pop_front();
-    return callback;
-  }
-
-  // Transfer SEXP ownership of callback members to the stack and run
-  // the callback. This must be called from the main R thread.
-  void RunCallback(RCallback callback) {
-    // Transfer responsibility of releasing SEXPs to the stack such that
-    // they will be garbage collected if any of the R calls below error
-    SEXP env_sexp = PROTECT(callback.env_sexp);
-    SEXP error_xptr = PROTECT(callback.error_xptr);
-    SEXP return_value_xptr = PROTECT(callback.return_value_xptr);
-    R_ReleaseObject(env_sexp);
-    R_ReleaseObject(error_xptr);
-    R_ReleaseObject(return_value_xptr);
-
-    // Release the dependence of the task on this callback queue
-    SEXP task_sym = PROTECT(Rf_install("task"));
-    SEXP task_xptr = PROTECT(Rf_findVarInFrame(env_sexp, task_sym));
-    R_SetExternalPtrProtected(task_xptr, R_NilValue);
-    UNPROTECT(2);
-
-    // Set up the call to run the callback
-    SEXP callback_sym = PROTECT(Rf_install("callback"));
-    SEXP return_code_sexp = PROTECT(Rf_ScalarInteger(callback.return_code));
-    SEXP callback_call =
-        PROTECT(Rf_lang4(callback_sym, return_code_sexp, error_xptr, return_value_xptr));
-
-    // Run the callback
-    Rf_eval(callback_call, env_sexp);
-
-    UNPROTECT(6);
-  }
-
-  static void FinalizeXptr(SEXP xptr) {
-    CallbackQueue* queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(xptr));
-    delete queue;
-  }
-
-  static SEXP MakeXptr(SEXP shelter = R_NilValue) {
-    SEXP xptr = PROTECT(R_MakeExternalPtr(new CallbackQueue(), R_NilValue, shelter));
-    R_RegisterCFinalizer(xptr, &FinalizeXptr);
-    UNPROTECT(1);
-    return xptr;
-  }
-
- private:
-  std::deque<RCallback> pending_callbacks_;
-  std::mutex callbacks_lock_;
-};
-
-// TODO: unlike other parts of this package, these functions make use of
-// C++ standard library functions that might throw exceptions in functions
-// that are directly called from R. These will segfault if an exception is
-// thrown. cpp11 handles this using BEGIN_CPP11 an END_CPP11...we would need
-// similar for safety here.
-
-extern "C" SEXP RAdbcNewCallbackQueue(void) { return CallbackQueue::MakeXptr(); }
-
-extern "C" SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr) {
-  // TODO: Check callback_queue_xptr class
-  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
-  return Rf_ScalarReal(queue->RunPending());
-}
-
-extern "C" SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr,
-                                                SEXP statement_xptr, SEXP out_stream_xptr,
-                                                SEXP error_xptr, SEXP callback_env) {
-  // TODO: check array_stream/array/callback queue classes using utils in radbc.h
-  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
-  auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
-  auto out_stream = adbc_from_xptr<ArrowArrayStream>(out_stream_xptr);
-  auto error = adbc_from_xptr<AdbcError>(error_xptr);
-
-  // Task handle to ensure the thread pointer is cleaned up
-  SEXP task_xptr = PROTECT(Task::MakeXptr(callback_queue_xptr));
-  SEXP task_symbol = PROTECT(Rf_install("task"));
-  Rf_setVar(task_symbol, task_xptr, callback_env);
-  UNPROTECT(1);
-
-  auto task = reinterpret_cast<Task*>(R_ExternalPtrAddr(task_xptr));
-  CallbackQueue::RCallback callback =
-      queue->InitCallback(callback_env, out_stream_xptr, error_xptr);
-  task->worker = new std::thread([statement, out_stream, error, callback, queue] {
-    CallbackQueue::RCallback callback_out = callback;
-    int64_t rows_affected = -1;
-    callback_out.return_code =
-        AdbcStatementExecuteQuery(statement, out_stream, &rows_affected, error);
-    queue->AddCallback(callback_out);
-  });
-
-  UNPROTECT(1);
-  return R_NilValue;
-}
-
-extern "C" SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr,
-                                             SEXP array_stream_xptr, SEXP array_xptr,
-                                             SEXP callback_env) {
-  // TODO: check array_stream/array/callback queue classes using utils in radbc.h
-  auto queue = reinterpret_cast<CallbackQueue*>(R_ExternalPtrAddr(callback_queue_xptr));
-  auto array_stream =
-      reinterpret_cast<struct ArrowArrayStream*>(R_ExternalPtrAddr(array_stream_xptr));
-  auto array = reinterpret_cast<struct ArrowArray*>(R_ExternalPtrAddr(array_xptr));
-
-  // Task handle to ensure the thread pointer is cleaned up
-  SEXP task_xptr = PROTECT(Task::MakeXptr(callback_queue_xptr));
-  SEXP task_symbol = PROTECT(Rf_install("task"));
-  Rf_setVar(task_symbol, task_xptr, callback_env);
-  UNPROTECT(1);
-
-  auto task = reinterpret_cast<Task*>(R_ExternalPtrAddr(task_xptr));
-  CallbackQueue::RCallback callback = queue->InitCallback(callback_env, array_xptr);
-  task->worker = new std::thread([array_stream, array, callback, queue] {
-    CallbackQueue::RCallback callback_out = callback;
-    callback_out.return_code = array_stream->get_next(array_stream, array);
-    queue->AddCallback(callback_out);
-  });
-
-  UNPROTECT(1);
-  return R_NilValue;
-}
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index c3dbf45a31..14362e0bcd 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -26,13 +26,6 @@ SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
                                       SEXP stream_xptr);
-SEXP RAdbcNewCallbackQueue(void);
-SEXP RAdbcCallbackQueueRunPending(SEXP callback_queue_xptr);
-SEXP RAdbcStatementExecuteQueryAsync(SEXP callback_queue_xptr, SEXP statement_xptr,
-                                     SEXP out_stream_xptr, SEXP error_xptr,
-                                     SEXP callback_env);
-SEXP RAdbcArrayStreamGetNextAsync(SEXP callback_queue_xptr, SEXP array_stream_xptr,
-                                  SEXP array_xptr, SEXP callback_env);
 SEXP RAdbcVoidDriverInitFunc(void);
 SEXP RAdbcMonkeyDriverInitFunc(void);
 SEXP RAdbcLogDriverInitFunc(void);
@@ -120,10 +113,6 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
     {"RAdbcAsyncTaskLaunchExecuteQuery", (DL_FUNC)&RAdbcAsyncTaskLaunchExecuteQuery, 3},
-    {"RAdbcNewCallbackQueue", (DL_FUNC)&RAdbcNewCallbackQueue, 0},
-    {"RAdbcCallbackQueueRunPending", (DL_FUNC)&RAdbcCallbackQueueRunPending, 1},
-    {"RAdbcStatementExecuteQueryAsync", (DL_FUNC)&RAdbcStatementExecuteQueryAsync, 5},
-    {"RAdbcArrayStreamGetNextAsync", (DL_FUNC)&RAdbcArrayStreamGetNextAsync, 4},
     {"RAdbcVoidDriverInitFunc", (DL_FUNC)&RAdbcVoidDriverInitFunc, 0},
     {"RAdbcMonkeyDriverInitFunc", (DL_FUNC)&RAdbcMonkeyDriverInitFunc, 0},
     {"RAdbcLogDriverInitFunc", (DL_FUNC)&RAdbcLogDriverInitFunc, 0},

From 93eeb235acab46a0a5b2e79adf5971867b35cfc8 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Mon, 6 May 2024 00:25:33 -0300
Subject: [PATCH 25/36] get next

---
 r/adbcdrivermanager/NAMESPACE    |  3 +-
 r/adbcdrivermanager/R/async.R    | 48 ++++++++++++++++++++++++++++----
 r/adbcdrivermanager/src/async.cc | 34 +++++++++++++++++-----
 r/adbcdrivermanager/src/init.c   |  2 ++
 4 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 8bec2e9b10..91b8cd334a 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -8,8 +8,9 @@ S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
-S3method(adbc_async_task_result,adbc_async_execute_query_rows_affected)
+S3method(adbc_async_task_result,adbc_async_execute_query)
 S3method(adbc_async_task_result,adbc_async_sleep)
+S3method(adbc_async_task_result,adbc_async_statement_stream_get_next)
 S3method(adbc_connection_init,adbc_database_log)
 S3method(adbc_connection_init,adbc_database_monkey)
 S3method(adbc_connection_init,default)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 6461da319b..bbc2f4627f 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -27,7 +27,7 @@ adbc_async_task_wait_for <- function(task, duration_ms) {
   .Call(RAdbcAsyncTaskWaitFor, task, duration_ms)
 }
 
-adbc_async_task_wait <- function(task, resolution_ms = 100) {
+adbc_async_task_wait <- function(task, resolution_ms = 50) {
   status <- "started"
   while (status != "ready") {
     status <- adbc_async_task_wait_for(task, resolution_ms)
@@ -110,27 +110,63 @@ adbc_async_task_result.adbc_async_sleep <- function(task) {
   task$user_data$duration_ms
 }
 
-adbc_statement_execute_query_rows_affected_async <- function(statement) {
-  task <- adbc_async_task("adbc_async_execute_query_rows_affected")
+adbc_statement_execute_query_async <- function(statement, stream = NULL) {
+  task <- adbc_async_task("adbc_async_execute_query")
 
   user_data <- task$user_data
   user_data$statement <- statement
+  user_data$stream <- stream
 
   user_data$rows_affected <- .Call(
     RAdbcAsyncTaskLaunchExecuteQuery,
     task,
     statement,
-    NULL
+    stream
   )
 
   task
 }
 
 #' @export
-adbc_async_task_result.adbc_async_execute_query_rows_affected <- function(task) {
+adbc_async_task_result.adbc_async_execute_query <- function(task) {
   if (!identical(task$return_code, 0L)) {
     stop_for_error(task$return_code, task$error_xptr)
   }
 
-  task$user_data$rows_affected
+  list(
+    statement = task$user_data$statement,
+    stream = task$user_data$stream,
+    rows_affected = task$user_data$rows_affected
+  )
+}
+
+adbc_statement_stream_get_next_async <- function(statement, stream) {
+  task <- adbc_async_task("adbc_async_statement_stream_get_next")
+
+  user_data <- task$user_data
+  user_data$statement <- statement
+  user_data$stream <- stream
+  user_data$array <- nanoarrow::nanoarrow_allocate_array()
+
+  user_data$rows_affected <- .Call(
+    RAdbcAsyncTaskLaunchStreamGetNext,
+    task,
+    stream,
+    user_data$array
+  )
+
+  task
+}
+
+#' @export
+adbc_async_task_result.adbc_async_statement_stream_get_next <- function(task) {
+  if (!identical(task$return_code, 0L)) {
+    adbc_statement_release(task$user_data$statement)
+    stop(task$user_data$stream$get_last_error())
+  }
+
+  list(
+    statement = task$user_data$statement,
+    array = task$user_data$array
+  )
 }
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 6fddf02fe8..ab91c6a50a 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -143,19 +143,39 @@ extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_
   error_for_started_task(task);
 
   auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
-  // auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr, /*null_ok*/ true);
+  ArrowArrayStream* stream = nullptr;
+  if (stream_xptr != R_NilValue) {
+    stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr);
+  }
 
   SEXP rows_affected_sexp = PROTECT(Rf_allocVector(REALSXP, 1));
   double* rows_affected_dbl = REAL(rows_affected_sexp);
 
-  task->result = std::async(std::launch::async, [task, statement, rows_affected_dbl] {
-    int64_t rows_affected = -1;
-    *(task->return_code) =
-        AdbcStatementExecuteQuery(statement, nullptr, &rows_affected, task->return_error);
-    *rows_affected_dbl = static_cast<double>(rows_affected);
-  });
+  task->result =
+      std::async(std::launch::async, [task, statement, stream, rows_affected_dbl] {
+        int64_t rows_affected = -1;
+        *(task->return_code) = AdbcStatementExecuteQuery(
+            statement, stream, &rows_affected, task->return_error);
+        *rows_affected_dbl = static_cast<double>(rows_affected);
+      });
 
   task->status = RAdbcAsyncTaskStatus::STARTED;
   UNPROTECT(1);
   return rows_affected_sexp;
 }
+
+extern "C" SEXP RAdbcAsyncTaskLaunchStreamGetNext(SEXP task_xptr, SEXP stream_xptr,
+                                                  SEXP array_xptr) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  error_for_started_task(task);
+
+  auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr);
+  auto array = adbc_from_xptr<ArrowArray>(array_xptr);
+
+  task->result = std::async(std::launch::async, [task, stream, array] {
+    *(task->return_code) = stream->get_next(stream, array);
+  });
+
+  task->status = RAdbcAsyncTaskStatus::STARTED;
+  return R_NilValue;
+}
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 14362e0bcd..819f4b4b6d 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -26,6 +26,7 @@ SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
                                       SEXP stream_xptr);
+SEXP RAdbcAsyncTaskLaunchStreamGetNext(SEXP task_xptr, SEXP stream_xptr, SEXP array_xptr);
 SEXP RAdbcVoidDriverInitFunc(void);
 SEXP RAdbcMonkeyDriverInitFunc(void);
 SEXP RAdbcLogDriverInitFunc(void);
@@ -113,6 +114,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
     {"RAdbcAsyncTaskLaunchExecuteQuery", (DL_FUNC)&RAdbcAsyncTaskLaunchExecuteQuery, 3},
+    {"RAdbcAsyncTaskLaunchStreamGetNext", (DL_FUNC)&RAdbcAsyncTaskLaunchStreamGetNext, 3},
     {"RAdbcVoidDriverInitFunc", (DL_FUNC)&RAdbcVoidDriverInitFunc, 0},
     {"RAdbcMonkeyDriverInitFunc", (DL_FUNC)&RAdbcMonkeyDriverInitFunc, 0},
     {"RAdbcLogDriverInitFunc", (DL_FUNC)&RAdbcLogDriverInitFunc, 0},

From cb677a3d386c58b193347e519c75e2270a3ab567 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Mon, 6 May 2024 22:47:03 -0300
Subject: [PATCH 26/36] add license

---
 r/adbcdrivermanager/R/zzz.R | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/r/adbcdrivermanager/R/zzz.R b/r/adbcdrivermanager/R/zzz.R
index 82901294fb..e5ba51804b 100644
--- a/r/adbcdrivermanager/R/zzz.R
+++ b/r/adbcdrivermanager/R/zzz.R
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 .onLoad <- function(...) {
   s3_register("promises::as.promise", "adbc_async_task")

From 25fa70f3661eeb25a0e3ce17944dbd3f9df4aaf5 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Mon, 6 May 2024 23:01:29 -0300
Subject: [PATCH 27/36] try more robust test for promise call

---
 .../tests/testthat/test-async.R               | 72 ++++++++++++-------
 1 file changed, 46 insertions(+), 26 deletions(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 5c6f248413..544fbf851a 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -77,31 +77,51 @@ test_that("async task waiter works", {
 test_that("async task can be converted to a promise", {
   skip_if_not_installed("promises")
 
-  expect_output(
-    {
-      adbc_async_sleep(100) %>%
-        promises::as.promise() %>%
-        promises::then(~print(sprintf("waited for %s ms", .x)))
-
-      Sys.sleep(0.2)
-      later::run_now()
-      later::run_now()
-    },
-    "waited for 100 ms"
-  )
+  # Enough time for most CI runners to handle this
+  max_wait_s <- 5
+
+  # Check successful call
+  async_called <- FALSE
+  adbc_async_sleep(100) %>%
+    promises::as.promise() %>%
+    promises::then(
+      onFulfilled = function(duration_ms) {
+        expect_identical(duration_ms, 100)
+        async_called <<- TRUE
+      }
+    )
+
+  # Only wait for so long before bailing on this test
+  for (i in seq_len(max_wait_s * 100)) {
+    later::run_now()
+    if (async_called) {
+      break
+    }
+
+    Sys.sleep(max_wait_s / 100)
+  }
+
+  expect_true(async_called)
+
+  # Check erroring call
+  async_called <- FALSE
+  adbc_async_sleep(100, error_message = "errored after 100 ms") %>%
+    promises::as.promise() %>%
+    promises::then(
+      onRejected = function(reason) {
+        expect_s3_class(reason, "adbc_async_sleep_error")
+        async_called <<- TRUE
+      }
+    )
+
+  for (i in seq_len(max_wait_s * 100)) {
+    later::run_now()
+    if (async_called) {
+      break
+    }
+    Sys.sleep(max_wait_s / 100)
+  }
+
+  expect_true(async_called)
 
-  expect_output(
-    {
-      adbc_async_sleep(100, error_message = "errored after 100 ms") %>%
-        promises::as.promise() %>%
-        promises::then(
-          onFulfilled = ~print(sprintf("waited for %s ms", .x)),
-          onRejected = ~print(.x)
-        )
-      Sys.sleep(0.2)
-      later::run_now()
-      later::run_now()
-    },
-    "errored after 100 ms"
-  )
 })

From 8a18b6a8bd66b0840a3011b35bf991be5894b54b Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Tue, 7 May 2024 00:02:27 -0300
Subject: [PATCH 28/36] waiting with cancel

---
 r/adbcdrivermanager/R/async.R                 | 33 +++++++++++++++----
 .../tests/testthat/test-async.R               | 10 +++---
 2 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index bbc2f4627f..cd5ac5d763 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -23,14 +23,18 @@ adbc_async_task <- function(subclass = character()) {
   )
 }
 
-adbc_async_task_wait_for <- function(task, duration_ms) {
-  .Call(RAdbcAsyncTaskWaitFor, task, duration_ms)
+adbc_async_task_status <- function(task, duration_ms) {
+  .Call(RAdbcAsyncTaskWaitFor, task, 0)
 }
 
-adbc_async_task_wait <- function(task, resolution_ms = 50) {
-  status <- "started"
-  while (status != "ready") {
-    status <- adbc_async_task_wait_for(task, resolution_ms)
+adbc_async_task_wait <- function(task, resolution = 0.05) {
+  while (adbc_async_task_status(task, 0) != "ready") {
+    withCallingHandlers(
+      Sys.sleep(resolution),
+      interrupt = function(e) {
+        adbc_async_task_cancel(task)
+      }
+    )
   }
 
   adbc_async_task_result(task)
@@ -44,7 +48,7 @@ later_loop_schedule_task_callback <- function(task, resolve, reject,
   force(reject)
 
   later::later(function() {
-    status <- adbc_async_task_wait_for(task, 0)
+    status <- adbc_async_task_status(task, 0)
     if (status == "timeout") {
       later_loop_schedule_task_callback(
         task,
@@ -69,6 +73,15 @@ as.promise.adbc_async_task <- function(task) {
   })
 }
 
+adbc_async_task_cancel <- function(task) {
+  UseMethod("adbc_async_task_cancel")
+}
+
+#' @export
+adbc_async_task_cancel.default <- function(task) {
+  FALSE
+}
+
 adbc_async_task_result <- function(task) {
   UseMethod("adbc_async_task_result")
 }
@@ -127,6 +140,12 @@ adbc_statement_execute_query_async <- function(statement, stream = NULL) {
   task
 }
 
+#' @export
+adbc_async_task_cancel.adbc_async_execute_query <- function(task) {
+  adbc_statement_cancel(task$user_data$statement)
+  TRUE
+}
+
 #' @export
 adbc_async_task_result.adbc_async_execute_query <- function(task) {
   if (!identical(task$return_code, 0L)) {
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 544fbf851a..e0b6cc6269 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -28,7 +28,7 @@ test_that("async tasks can be created and inspected", {
   expect_s3_class(task$error_xptr, "adbc_error")
   expect_identical(task$return_code, NA_integer_)
 
-  expect_identical(adbc_async_task_wait_for(task, 0), "not_started")
+  expect_identical(adbc_async_task_status(task, 0), "not_started")
 })
 
 test_that("async tasks can update R-level user data", {
@@ -48,16 +48,16 @@ test_that("async task methods error for invalid input", {
   )
 
   expect_error(
-    adbc_async_task_wait_for(adbc_async_task(), -1),
+    adbc_async_task_status(adbc_async_task(), -1),
     "duration_ms must be >= 0"
   )
 })
 
 test_that("async sleeper test works", {
   sleep_task <- adbc_async_sleep(500)
-  expect_identical(adbc_async_task_wait_for(sleep_task, 0), "timeout")
-  expect_identical(adbc_async_task_wait_for(sleep_task, 1000), "ready")
-  expect_identical(adbc_async_task_wait_for(sleep_task, 0), "ready")
+  expect_identical(adbc_async_task_status(sleep_task, 0), "timeout")
+  expect_identical(adbc_async_task_status(sleep_task, 1000), "ready")
+  expect_identical(adbc_async_task_status(sleep_task, 0), "ready")
   expect_identical(sleep_task$return_code, 0L)
   expect_identical(adbc_async_task_result(sleep_task), 500)
 })

From 4b6aad77788834fbcde9beaae54b1d9f25beb54b Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Tue, 7 May 2024 00:19:27 -0300
Subject: [PATCH 29/36] fix status/wait

---
 r/adbcdrivermanager/R/async.R                   | 10 +++++++---
 r/adbcdrivermanager/tests/testthat/test-async.R | 10 +++++-----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index cd5ac5d763..386155cf80 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -23,12 +23,16 @@ adbc_async_task <- function(subclass = character()) {
   )
 }
 
-adbc_async_task_status <- function(task, duration_ms) {
+adbc_async_task_status <- function(task) {
   .Call(RAdbcAsyncTaskWaitFor, task, 0)
 }
 
+adbc_async_task_wait_non_cancellable <- function(task, resolution = 0.05) {
+  .Call(RAdbcAsyncTaskWaitFor, task, round(resolution * 1000))
+}
+
 adbc_async_task_wait <- function(task, resolution = 0.05) {
-  while (adbc_async_task_status(task, 0) != "ready") {
+  while (adbc_async_task_status(task) != "ready") {
     withCallingHandlers(
       Sys.sleep(resolution),
       interrupt = function(e) {
@@ -48,7 +52,7 @@ later_loop_schedule_task_callback <- function(task, resolve, reject,
   force(reject)
 
   later::later(function() {
-    status <- adbc_async_task_status(task, 0)
+    status <- adbc_async_task_status(task)
     if (status == "timeout") {
       later_loop_schedule_task_callback(
         task,
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index e0b6cc6269..22be8cdd9d 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -28,7 +28,7 @@ test_that("async tasks can be created and inspected", {
   expect_s3_class(task$error_xptr, "adbc_error")
   expect_identical(task$return_code, NA_integer_)
 
-  expect_identical(adbc_async_task_status(task, 0), "not_started")
+  expect_identical(adbc_async_task_status(task), "not_started")
 })
 
 test_that("async tasks can update R-level user data", {
@@ -48,16 +48,16 @@ test_that("async task methods error for invalid input", {
   )
 
   expect_error(
-    adbc_async_task_status(adbc_async_task(), -1),
+    adbc_async_task_wait_non_cancellable(adbc_async_task(), -1),
     "duration_ms must be >= 0"
   )
 })
 
 test_that("async sleeper test works", {
   sleep_task <- adbc_async_sleep(500)
-  expect_identical(adbc_async_task_status(sleep_task, 0), "timeout")
-  expect_identical(adbc_async_task_status(sleep_task, 1000), "ready")
-  expect_identical(adbc_async_task_status(sleep_task, 0), "ready")
+  expect_identical(adbc_async_task_status(sleep_task), "timeout")
+  expect_identical(adbc_async_task_wait_non_cancellable(sleep_task, 1000), "ready")
+  expect_identical(adbc_async_task_status(sleep_task), "ready")
   expect_identical(sleep_task$return_code, 0L)
   expect_identical(adbc_async_task_result(sleep_task), 500)
 })

From f0cffa4e05350363c25bb23d9515db736bcb708f Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sat, 11 May 2024 22:57:33 -0300
Subject: [PATCH 30/36] prototype later integration

---
 r/adbcdrivermanager/src/async.cc              | 68 +++++++++++++++++--
 .../tests/testthat/test-async.R               |  2 +-
 2 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index ab91c6a50a..c7e1ee3b15 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -31,16 +31,63 @@
 
 #include "radbc.h"
 
+typedef void (*ExcecLaterNativeFn)(void (*func)(void*), void*, double, int);
+
+static ExcecLaterNativeFn later_execLaterNative2 = NULL;
+
+static inline void later_ensure_initialized() {
+  later_execLaterNative2 =
+      (ExcecLaterNativeFn)R_GetCCallable("later", "execLaterNative2");
+}
+
+static void later_task_callback_wrapper(void* data);
+
 enum class RAdbcAsyncTaskStatus { NOT_STARTED, STARTED, READY };
 
 struct RAdbcAsyncTask {
+  RAdbcAsyncTask() : callback_sexp(R_NilValue), callback_data_sexp(R_NilValue) {}
+
+  void SetCallback(SEXP callback, SEXP data, int loop_id) {
+    if (callback_sexp != R_NilValue) {
+      return;
+    }
+
+    callback_sexp = callback;
+    callback_data_sexp = data;
+    later_loop_id = loop_id;
+    later_ensure_initialized();
+  }
+
+  void ScheduleCallbackIfSet() {
+    if (callback_sexp != R_NilValue) {
+      later_execLaterNative2(&later_task_callback_wrapper, this, 0, later_loop_id);
+      callback_sexp = R_NilValue;
+    }
+  }
+
   AdbcError* return_error{nullptr};
   int* return_code{nullptr};
 
-  RAdbcAsyncTaskStatus status;
+  SEXP callback_sexp;
+  SEXP callback_data_sexp;
+  int later_loop_id{-1};
+
+  RAdbcAsyncTaskStatus status{RAdbcAsyncTaskStatus::NOT_STARTED};
   std::future<void> result;
 };
 
+static void later_task_callback_wrapper(void* data) {
+  auto task = reinterpret_cast<RAdbcAsyncTask*>(data);
+
+  SEXP func_sym = PROTECT(Rf_install("adbc_async_run_callback"));
+  SEXP func_call =
+      PROTECT(Rf_lang3(func_sym, task->callback_sexp, task->callback_data_sexp));
+  SEXP pkg_chr = PROTECT(Rf_mkString("adbcdrivermanager"));
+  SEXP pkg_ns = PROTECT(R_FindNamespace(pkg_chr));
+  Rf_eval(func_call, pkg_ns);
+  UNPROTECT(4);
+}
+
 template <>
 inline const char* adbc_xptr_class<RAdbcAsyncTask>() {
   return "adbc_async_task";
@@ -60,7 +107,7 @@ static void error_for_started_task(RAdbcAsyncTask* task) {
 }
 
 extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
-  const char* names[] = {"error_xptr", "return_code", "user_data", ""};
+  const char* names[] = {"error_xptr", "return_code", "user_data", "callback", ""};
   SEXP task_prot = PROTECT(Rf_mkNamed(VECSXP, names));
 
   SET_VECTOR_ELT(task_prot, 0, error_xptr);
@@ -81,12 +128,22 @@ extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
   task->return_code = INTEGER(VECTOR_ELT(task_prot, 1));
 
   *(task->return_code) = NA_INTEGER;
-  task->status = RAdbcAsyncTaskStatus::NOT_STARTED;
 
   UNPROTECT(2);
   return task_xptr;
 }
 
+extern "C" SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_sexp,
+                                          SEXP loop_id_sexp) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  SEXP task_prot = R_ExternalPtrProtected(task_xptr);
+  int loop_id = adbc_as_int(loop_id_sexp);
+
+  SET_VECTOR_ELT(task_prot, 3, callback_sexp);
+  task->SetCallback(callback_sexp, task_xptr, loop_id);
+  return R_NilValue;
+}
+
 extern "C" SEXP RAdbcAsyncTaskData(SEXP task_xptr) {
   adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
   return R_ExternalPtrProtected(task_xptr);
@@ -115,7 +172,7 @@ extern "C" SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp) {
     case std::future_status::timeout:
       return Rf_mkString("timeout");
     case std::future_status::ready:
-
+      task->status = RAdbcAsyncTaskStatus::READY;
       return Rf_mkString("ready");
     default:
       Rf_error("Unknown status returned from future::wait_for()");
@@ -131,6 +188,7 @@ extern "C" SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp)
   task->result = std::async(std::launch::async, [task, duration_ms] {
     std::this_thread::sleep_for(std::chrono::milliseconds(duration_ms));
     *(task->return_code) = ADBC_STATUS_OK;
+    task->ScheduleCallbackIfSet();
   });
 
   task->status = RAdbcAsyncTaskStatus::STARTED;
@@ -157,6 +215,7 @@ extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_
         *(task->return_code) = AdbcStatementExecuteQuery(
             statement, stream, &rows_affected, task->return_error);
         *rows_affected_dbl = static_cast<double>(rows_affected);
+        task->ScheduleCallbackIfSet();
       });
 
   task->status = RAdbcAsyncTaskStatus::STARTED;
@@ -174,6 +233,7 @@ extern "C" SEXP RAdbcAsyncTaskLaunchStreamGetNext(SEXP task_xptr, SEXP stream_xp
 
   task->result = std::async(std::launch::async, [task, stream, array] {
     *(task->return_code) = stream->get_next(stream, array);
+    task->ScheduleCallbackIfSet();
   });
 
   task->status = RAdbcAsyncTaskStatus::STARTED;
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 22be8cdd9d..795309ab3f 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -22,7 +22,7 @@ test_that("async tasks can be created and inspected", {
 
   expect_identical(
     names(task),
-    c("error_xptr", "return_code", "user_data")
+    c("error_xptr", "return_code", "user_data", "callback")
   )
 
   expect_s3_class(task$error_xptr, "adbc_error")

From c1d2313dcee9672546fa47e6419fd4b1767e785f Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sat, 11 May 2024 23:42:33 -0300
Subject: [PATCH 31/36] with test

---
 r/adbcdrivermanager/R/async.R                 | 19 ++++++++++++++++++
 r/adbcdrivermanager/src/async.cc              | 20 ++++++-------------
 r/adbcdrivermanager/src/init.c                |  2 ++
 .../tests/testthat/test-async.R               | 18 +++++++++++++++++
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 386155cf80..ff54f5cb37 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -27,6 +27,25 @@ adbc_async_task_status <- function(task) {
   .Call(RAdbcAsyncTaskWaitFor, task, 0)
 }
 
+adbc_async_task_set_callback <- function(task, callback, loop = later::current_loop()) {
+  # If the task is completed, run the callback (or else the callback
+  # will not run)
+  if (adbc_async_task_status(task) == "ready") {
+    result <- adbc_async_task_result(task)
+    callback(result)
+  } else {
+    .Call(RAdbcAsyncTaskSetCallback, task, callback, loop$id)
+  }
+
+  invisible(task)
+}
+
+adbc_async_task_run_callback <- function(task) {
+  callback <- task$callback
+  result <- adbc_async_task_result(task)
+  callback(result)
+}
+
 adbc_async_task_wait_non_cancellable <- function(task, resolution = 0.05) {
   .Call(RAdbcAsyncTaskWaitFor, task, round(resolution * 1000))
 }
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index c7e1ee3b15..19379d2b7e 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -45,30 +45,23 @@ static void later_task_callback_wrapper(void* data);
 enum class RAdbcAsyncTaskStatus { NOT_STARTED, STARTED, READY };
 
 struct RAdbcAsyncTask {
-  RAdbcAsyncTask() : callback_sexp(R_NilValue), callback_data_sexp(R_NilValue) {}
+  RAdbcAsyncTask() : callback_data_sexp(R_NilValue) {}
 
-  void SetCallback(SEXP callback, SEXP data, int loop_id) {
-    if (callback_sexp != R_NilValue) {
-      return;
-    }
-
-    callback_sexp = callback;
+  void SetCallback(SEXP data, int loop_id) {
     callback_data_sexp = data;
     later_loop_id = loop_id;
     later_ensure_initialized();
   }
 
   void ScheduleCallbackIfSet() {
-    if (callback_sexp != R_NilValue) {
+    if (callback_data_sexp != R_NilValue) {
       later_execLaterNative2(&later_task_callback_wrapper, this, 0, later_loop_id);
-      callback_sexp = R_NilValue;
     }
   }
 
   AdbcError* return_error{nullptr};
   int* return_code{nullptr};
 
-  SEXP callback_sexp;
   SEXP callback_data_sexp;
   int later_loop_id{-1};
 
@@ -79,9 +72,8 @@ struct RAdbcAsyncTask {
 static void later_task_callback_wrapper(void* data) {
   auto task = reinterpret_cast<RAdbcAsyncTask*>(data);
 
-  SEXP func_sym = PROTECT(Rf_install("adbc_async_run_callback"));
-  SEXP func_call =
-      PROTECT(Rf_lang3(func_sym, task->callback_sexp, task->callback_data_sexp));
+  SEXP func_sym = PROTECT(Rf_install("adbc_async_task_run_callback"));
+  SEXP func_call = PROTECT(Rf_lang2(func_sym, task->callback_data_sexp));
   SEXP pkg_chr = PROTECT(Rf_mkString("adbcdrivermanager"));
   SEXP pkg_ns = PROTECT(R_FindNamespace(pkg_chr));
   Rf_eval(func_call, pkg_ns);
@@ -140,7 +132,7 @@ extern "C" SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_sexp,
   int loop_id = adbc_as_int(loop_id_sexp);
 
   SET_VECTOR_ELT(task_prot, 3, callback_sexp);
-  task->SetCallback(callback_sexp, task_xptr, loop_id);
+  task->SetCallback(task_xptr, loop_id);
   return R_NilValue;
 }
 
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 819f4b4b6d..a2e5a59ac5 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -21,6 +21,7 @@
 
 /* generated by tools/make-callentries.R */
 SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
+SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_sexp, SEXP loop_id_sexp);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
@@ -110,6 +111,7 @@ SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot);
 
 static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskNew", (DL_FUNC)&RAdbcAsyncTaskNew, 1},
+    {"RAdbcAsyncTaskSetCallback", (DL_FUNC)&RAdbcAsyncTaskSetCallback, 3},
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 795309ab3f..60b22f82db 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -74,6 +74,24 @@ test_that("async task waiter works", {
   )
 })
 
+test_that("async tasks can set an R callback", {
+  skip_if_not_installed("later")
+
+  async_called <- FALSE
+  sleep_task <- adbc_async_sleep(200)
+  adbc_async_task_set_callback(sleep_task, function(x) { async_called <<- TRUE })
+  Sys.sleep(0.4)
+  later::run_now()
+  expect_true(async_called)
+
+  # Ensure the callback runs even if the task is already finished
+  async_called <- FALSE
+  sleep_task <- adbc_async_sleep(0)
+  adbc_async_task_set_callback(sleep_task, function(x) { async_called <<- TRUE })
+  Sys.sleep(0.1)
+  expect_true(async_called)
+})
+
 test_that("async task can be converted to a promise", {
   skip_if_not_installed("promises")
 

From d8bb000502f9037f2f32ed4a931af17f023e22a3 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Sun, 12 May 2024 00:03:51 -0300
Subject: [PATCH 32/36] use with promises

---
 r/adbcdrivermanager/R/async.R                 | 56 +++++++------------
 r/adbcdrivermanager/src/async.cc              | 10 ++--
 r/adbcdrivermanager/src/init.c                |  5 +-
 .../tests/testthat/test-async.R               | 13 ++++-
 4 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index ff54f5cb37..81eaa8b872 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -27,23 +27,35 @@ adbc_async_task_status <- function(task) {
   .Call(RAdbcAsyncTaskWaitFor, task, 0)
 }
 
-adbc_async_task_set_callback <- function(task, callback, loop = later::current_loop()) {
+adbc_async_task_set_callback <- function(task, resolve, reject = NULL,
+                                         loop = later::current_loop()) {
   # If the task is completed, run the callback (or else the callback
   # will not run)
   if (adbc_async_task_status(task) == "ready") {
-    result <- adbc_async_task_result(task)
-    callback(result)
+    adbc_async_task_run_callback(task, resolve, reject)
   } else {
-    .Call(RAdbcAsyncTaskSetCallback, task, callback, loop$id)
+    .Call(RAdbcAsyncTaskSetCallback, task, resolve, reject, loop$id)
   }
 
   invisible(task)
 }
 
-adbc_async_task_run_callback <- function(task) {
-  callback <- task$callback
-  result <- adbc_async_task_result(task)
-  callback(result)
+adbc_async_task_run_callback <- function(task, resolve = task$resolve,
+                                         reject = task$reject) {
+  tryCatch({
+      result <- adbc_async_task_result(task)
+      resolve(result)
+    },
+    error = function(e) {
+      if (is.null(reject)) {
+        stop(e)
+      } else {
+        reject(e)
+      }
+    }
+  )
+
+  invisible(task)
 }
 
 adbc_async_task_wait_non_cancellable <- function(task, resolution = 0.05) {
@@ -63,36 +75,10 @@ adbc_async_task_wait <- function(task, resolution = 0.05) {
   adbc_async_task_result(task)
 }
 
-later_loop_schedule_task_callback <- function(task, resolve, reject,
-                                              loop = later::current_loop(),
-                                              delay = 0) {
-  force(task)
-  force(resolve)
-  force(reject)
-
-  later::later(function() {
-    status <- adbc_async_task_status(task)
-    if (status == "timeout") {
-      later_loop_schedule_task_callback(
-        task,
-        resolve,
-        reject,
-        loop = loop,
-        delay = delay
-      )
-    } else {
-      tryCatch(
-        resolve(adbc_async_task_result(task)),
-        error = function(e) reject(e)
-      )
-    }
-  }, delay = delay, loop = loop)
-}
-
 as.promise.adbc_async_task <- function(task) {
   force(task)
   promises::promise(function(resolve, reject) {
-    later_loop_schedule_task_callback(task, resolve, reject)
+    adbc_async_task_set_callback(task, resolve, reject)
   })
 }
 
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index 19379d2b7e..c32afee4ef 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -99,7 +99,8 @@ static void error_for_started_task(RAdbcAsyncTask* task) {
 }
 
 extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
-  const char* names[] = {"error_xptr", "return_code", "user_data", "callback", ""};
+  const char* names[] = {"error_xptr", "return_code", "user_data",
+                         "resolve",    "reject",      ""};
   SEXP task_prot = PROTECT(Rf_mkNamed(VECSXP, names));
 
   SET_VECTOR_ELT(task_prot, 0, error_xptr);
@@ -125,13 +126,14 @@ extern "C" SEXP RAdbcAsyncTaskNew(SEXP error_xptr) {
   return task_xptr;
 }
 
-extern "C" SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_sexp,
-                                          SEXP loop_id_sexp) {
+extern "C" SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_resolve_sexp,
+                                          SEXP callback_reject_sexp, SEXP loop_id_sexp) {
   auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
   SEXP task_prot = R_ExternalPtrProtected(task_xptr);
   int loop_id = adbc_as_int(loop_id_sexp);
 
-  SET_VECTOR_ELT(task_prot, 3, callback_sexp);
+  SET_VECTOR_ELT(task_prot, 3, callback_resolve_sexp);
+  SET_VECTOR_ELT(task_prot, 4, callback_reject_sexp);
   task->SetCallback(task_xptr, loop_id);
   return R_NilValue;
 }
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index a2e5a59ac5..1ae9ab30d3 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -21,7 +21,8 @@
 
 /* generated by tools/make-callentries.R */
 SEXP RAdbcAsyncTaskNew(SEXP error_xptr);
-SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_sexp, SEXP loop_id_sexp);
+SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_resolve_sexp,
+                               SEXP callback_reject_sexp, SEXP loop_id_sexp);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
@@ -111,7 +112,7 @@ SEXP RAdbcXptrSetProtected(SEXP xptr, SEXP prot);
 
 static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskNew", (DL_FUNC)&RAdbcAsyncTaskNew, 1},
-    {"RAdbcAsyncTaskSetCallback", (DL_FUNC)&RAdbcAsyncTaskSetCallback, 3},
+    {"RAdbcAsyncTaskSetCallback", (DL_FUNC)&RAdbcAsyncTaskSetCallback, 4},
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 60b22f82db..1a48fe90bf 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -22,7 +22,7 @@ test_that("async tasks can be created and inspected", {
 
   expect_identical(
     names(task),
-    c("error_xptr", "return_code", "user_data", "callback")
+    c("error_xptr", "return_code", "user_data", "resolve", "reject")
   )
 
   expect_s3_class(task$error_xptr, "adbc_error")
@@ -90,6 +90,17 @@ test_that("async tasks can set an R callback", {
   adbc_async_task_set_callback(sleep_task, function(x) { async_called <<- TRUE })
   Sys.sleep(0.1)
   expect_true(async_called)
+
+  # Ensure this also works on error
+  async_called <- FALSE
+  sleep_task <- adbc_async_sleep(0, error_message = "some error")
+  adbc_async_task_set_callback(
+    sleep_task,
+    resolve = function(x) NULL,
+    reject = function(x) { async_called <<- TRUE }
+  )
+  Sys.sleep(0.1)
+  expect_true(async_called)
 })
 
 test_that("async task can be converted to a promise", {

From 702e5802799536d8a8f9e40248a38917c38006fc Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Fri, 31 May 2024 14:23:20 -0300
Subject: [PATCH 33/36] maybe better waiting

---
 r/adbcdrivermanager/NAMESPACE                 |  2 ++
 r/adbcdrivermanager/R/async.R                 | 20 +++++++------
 r/adbcdrivermanager/src/async.cc              | 30 ++++++++++++++++++-
 r/adbcdrivermanager/src/init.c                |  2 ++
 .../tests/testthat/test-async.R               |  2 +-
 5 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 91b8cd334a..63b2552224 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -8,6 +8,8 @@ S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
+S3method(adbc_async_task_cancel,adbc_async_execute_query)
+S3method(adbc_async_task_cancel,default)
 S3method(adbc_async_task_result,adbc_async_execute_query)
 S3method(adbc_async_task_result,adbc_async_sleep)
 S3method(adbc_async_task_result,adbc_async_statement_stream_get_next)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 81eaa8b872..1f39607ff0 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -58,18 +58,20 @@ adbc_async_task_run_callback <- function(task, resolve = task$resolve,
   invisible(task)
 }
 
-adbc_async_task_wait_non_cancellable <- function(task, resolution = 0.05) {
+adbc_async_task_wait_non_cancellable <- function(task, resolution = 0.1) {
   .Call(RAdbcAsyncTaskWaitFor, task, round(resolution * 1000))
 }
 
-adbc_async_task_wait <- function(task, resolution = 0.05) {
-  while (adbc_async_task_status(task) != "ready") {
-    withCallingHandlers(
-      Sys.sleep(resolution),
-      interrupt = function(e) {
-        adbc_async_task_cancel(task)
-      }
-    )
+adbc_async_task_wait <- function(task, resolution = 0.1) {
+  withCallingHandlers(
+    status <- .Call(RAdbcAsyncTaskWait, task, round(resolution * 1000)),
+    interrupt = function(e) {
+      adbc_async_task_cancel(task)
+    }
+  )
+
+  if (status != "ready") {
+    stop(sprintf("Expected status ready but got %s", status))
   }
 
   adbc_async_task_result(task)
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index c32afee4ef..bf436a6d69 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -164,7 +164,35 @@ extern "C" SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp) {
       task->result.wait_for(std::chrono::milliseconds(duration_ms));
   switch (status) {
     case std::future_status::timeout:
-      return Rf_mkString("timeout");
+      return Rf_mkString("started");
+    case std::future_status::ready:
+      task->status = RAdbcAsyncTaskStatus::READY;
+      return Rf_mkString("ready");
+    default:
+      Rf_error("Unknown status returned from future::wait_for()");
+  }
+}
+
+extern "C" SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP resolution_ms_sexp) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  int resolution_ms = adbc_as_int(resolution_ms_sexp);
+
+  switch (task->status) {
+    case RAdbcAsyncTaskStatus::NOT_STARTED:
+      return Rf_mkString("not_started");
+    case RAdbcAsyncTaskStatus::READY:
+      return Rf_mkString("ready");
+    default:
+      break;
+  }
+
+  std::future_status status;
+  do {
+    status = task->result.wait_for(std::chrono::milliseconds(resolution_ms));
+    R_CheckUserInterrupt();
+  } while (status == std::future_status::timeout);
+
+  switch (status) {
     case std::future_status::ready:
       task->status = RAdbcAsyncTaskStatus::READY;
       return Rf_mkString("ready");
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 1ae9ab30d3..743e5bb194 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -25,6 +25,7 @@ SEXP RAdbcAsyncTaskSetCallback(SEXP task_xptr, SEXP callback_resolve_sexp,
                                SEXP callback_reject_sexp, SEXP loop_id_sexp);
 SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
+SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP resolution_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
                                       SEXP stream_xptr);
@@ -115,6 +116,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskSetCallback", (DL_FUNC)&RAdbcAsyncTaskSetCallback, 4},
     {"RAdbcAsyncTaskData", (DL_FUNC)&RAdbcAsyncTaskData, 1},
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
+    {"RAdbcAsyncTaskWait", (DL_FUNC)&RAdbcAsyncTaskWait, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
     {"RAdbcAsyncTaskLaunchExecuteQuery", (DL_FUNC)&RAdbcAsyncTaskLaunchExecuteQuery, 3},
     {"RAdbcAsyncTaskLaunchStreamGetNext", (DL_FUNC)&RAdbcAsyncTaskLaunchStreamGetNext, 3},
diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 1a48fe90bf..5c190537bf 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -55,7 +55,7 @@ test_that("async task methods error for invalid input", {
 
 test_that("async sleeper test works", {
   sleep_task <- adbc_async_sleep(500)
-  expect_identical(adbc_async_task_status(sleep_task), "timeout")
+  expect_identical(adbc_async_task_status(sleep_task), "started")
   expect_identical(adbc_async_task_wait_non_cancellable(sleep_task, 1000), "ready")
   expect_identical(adbc_async_task_status(sleep_task), "ready")
   expect_identical(sleep_task$return_code, 0L)

From 01cfa543fe1200b0d6e256361f08b6e3dc3b4b36 Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Fri, 31 May 2024 14:27:43 -0300
Subject: [PATCH 34/36] try to fix tests

---
 r/adbcdrivermanager/tests/testthat/test-async.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 5c190537bf..80d1edc05c 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -86,14 +86,14 @@ test_that("async tasks can set an R callback", {
 
   # Ensure the callback runs even if the task is already finished
   async_called <- FALSE
-  sleep_task <- adbc_async_sleep(0)
+  sleep_task <- adbc_async_sleep(1)
   adbc_async_task_set_callback(sleep_task, function(x) { async_called <<- TRUE })
   Sys.sleep(0.1)
   expect_true(async_called)
 
   # Ensure this also works on error
   async_called <- FALSE
-  sleep_task <- adbc_async_sleep(0, error_message = "some error")
+  sleep_task <- adbc_async_sleep(1, error_message = "some error")
   adbc_async_task_set_callback(
     sleep_task,
     resolve = function(x) NULL,

From 50d6d3fabfd044f43ff52dcae06d2656b7a54e2c Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Fri, 31 May 2024 14:32:12 -0300
Subject: [PATCH 35/36] see if waiting a little more is more reliable on CI

---
 r/adbcdrivermanager/tests/testthat/test-async.R | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/r/adbcdrivermanager/tests/testthat/test-async.R b/r/adbcdrivermanager/tests/testthat/test-async.R
index 80d1edc05c..517b51a5ff 100644
--- a/r/adbcdrivermanager/tests/testthat/test-async.R
+++ b/r/adbcdrivermanager/tests/testthat/test-async.R
@@ -86,14 +86,16 @@ test_that("async tasks can set an R callback", {
 
   # Ensure the callback runs even if the task is already finished
   async_called <- FALSE
-  sleep_task <- adbc_async_sleep(1)
+  sleep_task <- adbc_async_sleep(0)
+  Sys.sleep(0.1)
   adbc_async_task_set_callback(sleep_task, function(x) { async_called <<- TRUE })
   Sys.sleep(0.1)
   expect_true(async_called)
 
   # Ensure this also works on error
   async_called <- FALSE
-  sleep_task <- adbc_async_sleep(1, error_message = "some error")
+  sleep_task <- adbc_async_sleep(0, error_message = "some error")
+  Sys.sleep(0.1)
   adbc_async_task_set_callback(
     sleep_task,
     resolve = function(x) NULL,

From a36f41e6b2a12d9b72c7cf9607c46c5ee2fa2bfa Mon Sep 17 00:00:00 2001
From: Dewey Dunnington <dewey@fishandwhistle.net>
Date: Fri, 31 May 2024 15:53:02 -0300
Subject: [PATCH 36/36] maybe prepare + schema

---
 r/adbcdrivermanager/NAMESPACE    |  4 +-
 r/adbcdrivermanager/R/async.R    | 75 ++++++++++++++++++++++++++++----
 r/adbcdrivermanager/src/async.cc | 45 +++++++++++++++++++
 r/adbcdrivermanager/src/init.c   |  6 +++
 4 files changed, 121 insertions(+), 9 deletions(-)

diff --git a/r/adbcdrivermanager/NAMESPACE b/r/adbcdrivermanager/NAMESPACE
index 63b2552224..a37824acb9 100644
--- a/r/adbcdrivermanager/NAMESPACE
+++ b/r/adbcdrivermanager/NAMESPACE
@@ -8,11 +8,13 @@ S3method("[[",adbc_async_task)
 S3method("[[",adbc_error)
 S3method("[[",adbc_xptr)
 S3method("[[<-",adbc_xptr)
-S3method(adbc_async_task_cancel,adbc_async_execute_query)
+S3method(adbc_async_task_cancel,adbc_async_statement_cancellable)
 S3method(adbc_async_task_cancel,default)
 S3method(adbc_async_task_result,adbc_async_execute_query)
+S3method(adbc_async_task_result,adbc_async_prepare)
 S3method(adbc_async_task_result,adbc_async_sleep)
 S3method(adbc_async_task_result,adbc_async_statement_stream_get_next)
+S3method(adbc_async_task_result,adbc_async_statement_stream_schema)
 S3method(adbc_connection_init,adbc_database_log)
 S3method(adbc_connection_init,adbc_database_monkey)
 S3method(adbc_connection_init,default)
diff --git a/r/adbcdrivermanager/R/async.R b/r/adbcdrivermanager/R/async.R
index 1f39607ff0..78bbe60eb5 100644
--- a/r/adbcdrivermanager/R/async.R
+++ b/r/adbcdrivermanager/R/async.R
@@ -134,8 +134,37 @@ adbc_async_task_result.adbc_async_sleep <- function(task) {
   task$user_data$duration_ms
 }
 
+#' @export
+adbc_async_task_cancel.adbc_async_statement_cancellable <- function(task) {
+  adbc_statement_cancel(task$user_data$statement)
+  TRUE
+}
+
+adbc_statement_prepare_async <- function(statement) {
+  task <- adbc_async_task(
+    c("adbc_async_prepare", "adbc_async_statement_cancellable")
+  )
+
+  user_data <- task$user_data
+  user_data$statement <- statement
+  .Call(RAdbcAsyncTaskLaunchPrepare, task, statement)
+
+  task
+}
+
+#' @export
+adbc_async_task_result.adbc_async_prepare <- function(task) {
+  if (!identical(task$return_code, 0L)) {
+    stop_for_error(task$return_code, task$error_xptr)
+  }
+
+  task$user_data$statement
+}
+
 adbc_statement_execute_query_async <- function(statement, stream = NULL) {
-  task <- adbc_async_task("adbc_async_execute_query")
+  task <- adbc_async_task(
+    c("adbc_async_execute_query", "adbc_async_statement_cancellable")
+  )
 
   user_data <- task$user_data
   user_data$statement <- statement
@@ -151,12 +180,6 @@ adbc_statement_execute_query_async <- function(statement, stream = NULL) {
   task
 }
 
-#' @export
-adbc_async_task_cancel.adbc_async_execute_query <- function(task) {
-  adbc_statement_cancel(task$user_data$statement)
-  TRUE
-}
-
 #' @export
 adbc_async_task_result.adbc_async_execute_query <- function(task) {
   if (!identical(task$return_code, 0L)) {
@@ -170,8 +193,44 @@ adbc_async_task_result.adbc_async_execute_query <- function(task) {
   )
 }
 
+adbc_statement_stream_get_schema_async <- function(statement, stream) {
+  task <- adbc_async_task(
+    c("adbc_async_statement_stream_get_next", "adbc_async_statement_cancellable")
+  )
+
+  user_data <- task$user_data
+  user_data$statement <- statement
+  user_data$stream <- stream
+  user_data$schema <- nanoarrow::nanoarrow_allocate_schema()
+
+  user_data$rows_affected <- .Call(
+    RAdbcAsyncTaskLaunchStreamGetSchema,
+    task,
+    stream,
+    user_data$schema
+  )
+
+  task
+}
+
+
+#' @export
+adbc_async_task_result.adbc_async_statement_stream_schema <- function(task) {
+  if (!identical(task$return_code, 0L)) {
+    adbc_statement_release(task$user_data$statement)
+    stop(task$user_data$stream$get_last_error())
+  }
+
+  list(
+    statement = task$user_data$statement,
+    array = task$user_data$schema
+  )
+}
+
 adbc_statement_stream_get_next_async <- function(statement, stream) {
-  task <- adbc_async_task("adbc_async_statement_stream_get_next")
+  task <- adbc_async_task(
+    c("adbc_async_statement_stream_get_next", "adbc_async_statement_cancellable")
+  )
 
   user_data <- task$user_data
   user_data$statement <- statement
diff --git a/r/adbcdrivermanager/src/async.cc b/r/adbcdrivermanager/src/async.cc
index bf436a6d69..6ef130192b 100644
--- a/r/adbcdrivermanager/src/async.cc
+++ b/r/adbcdrivermanager/src/async.cc
@@ -22,6 +22,7 @@
 
 #include <deque>
 #include <future>
+#include <memory>
 #include <mutex>
 #include <string>
 #include <thread>
@@ -42,6 +43,17 @@ static inline void later_ensure_initialized() {
 
 static void later_task_callback_wrapper(void* data);
 
+struct ArrowArrayCustomDeleter {
+  void operator()(ArrowArray* array) const {
+    if (array->release != nullptr) {
+      array->release(array);
+    }
+    delete array;
+  }
+};
+
+using UniqueArrowArrayPtr = std::unique_ptr<ArrowArray, ArrowArrayCustomDeleter>;
+
 enum class RAdbcAsyncTaskStatus { NOT_STARTED, STARTED, READY };
 
 struct RAdbcAsyncTask {
@@ -217,6 +229,22 @@ extern "C" SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp)
   return R_NilValue;
 }
 
+extern "C" SEXP RAdbcAsyncTaskLaunchPrepare(SEXP task_xptr, SEXP statement_xptr) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  error_for_started_task(task);
+
+  auto statement = adbc_from_xptr<AdbcStatement>(statement_xptr);
+
+  task->result = std::async(std::launch::async, [task, statement] {
+    *(task->return_code) = AdbcStatementPrepare(statement, task->return_error);
+    task->ScheduleCallbackIfSet();
+  });
+
+  task->status = RAdbcAsyncTaskStatus::STARTED;
+  UNPROTECT(1);
+  return R_NilValue;
+}
+
 extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
                                                  SEXP stream_xptr) {
   auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
@@ -245,6 +273,23 @@ extern "C" SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_
   return rows_affected_sexp;
 }
 
+extern "C" SEXP RAdbcAsyncTaskLaunchStreamGetSchema(SEXP task_xptr, SEXP stream_xptr,
+                                                    SEXP schema_xptr) {
+  auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
+  error_for_started_task(task);
+
+  auto stream = adbc_from_xptr<ArrowArrayStream>(stream_xptr);
+  auto schema = adbc_from_xptr<ArrowSchema>(schema_xptr);
+
+  task->result = std::async(std::launch::async, [task, stream, schema] {
+    *(task->return_code) = stream->get_schema(stream, schema);
+    task->ScheduleCallbackIfSet();
+  });
+
+  task->status = RAdbcAsyncTaskStatus::STARTED;
+  return R_NilValue;
+}
+
 extern "C" SEXP RAdbcAsyncTaskLaunchStreamGetNext(SEXP task_xptr, SEXP stream_xptr,
                                                   SEXP array_xptr) {
   auto task = adbc_from_xptr<RAdbcAsyncTask>(task_xptr);
diff --git a/r/adbcdrivermanager/src/init.c b/r/adbcdrivermanager/src/init.c
index 743e5bb194..2ea4c7dde3 100644
--- a/r/adbcdrivermanager/src/init.c
+++ b/r/adbcdrivermanager/src/init.c
@@ -27,8 +27,11 @@ SEXP RAdbcAsyncTaskData(SEXP task_xptr);
 SEXP RAdbcAsyncTaskWaitFor(SEXP task_xptr, SEXP duration_ms_sexp);
 SEXP RAdbcAsyncTaskWait(SEXP task_xptr, SEXP resolution_ms_sexp);
 SEXP RAdbcAsyncTaskLaunchSleep(SEXP task_xptr, SEXP duration_ms_sexp);
+SEXP RAdbcAsyncTaskLaunchPrepare(SEXP task_xptr, SEXP statement_xptr);
 SEXP RAdbcAsyncTaskLaunchExecuteQuery(SEXP task_xptr, SEXP statement_xptr,
                                       SEXP stream_xptr);
+SEXP RAdbcAsyncTaskLaunchStreamGetSchema(SEXP task_xptr, SEXP stream_xptr,
+                                         SEXP schema_xptr);
 SEXP RAdbcAsyncTaskLaunchStreamGetNext(SEXP task_xptr, SEXP stream_xptr, SEXP array_xptr);
 SEXP RAdbcVoidDriverInitFunc(void);
 SEXP RAdbcMonkeyDriverInitFunc(void);
@@ -118,7 +121,10 @@ static const R_CallMethodDef CallEntries[] = {
     {"RAdbcAsyncTaskWaitFor", (DL_FUNC)&RAdbcAsyncTaskWaitFor, 2},
     {"RAdbcAsyncTaskWait", (DL_FUNC)&RAdbcAsyncTaskWait, 2},
     {"RAdbcAsyncTaskLaunchSleep", (DL_FUNC)&RAdbcAsyncTaskLaunchSleep, 2},
+    {"RAdbcAsyncTaskLaunchPrepare", (DL_FUNC)&RAdbcAsyncTaskLaunchPrepare, 2},
     {"RAdbcAsyncTaskLaunchExecuteQuery", (DL_FUNC)&RAdbcAsyncTaskLaunchExecuteQuery, 3},
+    {"RAdbcAsyncTaskLaunchStreamGetSchema", (DL_FUNC)&RAdbcAsyncTaskLaunchStreamGetSchema,
+     3},
     {"RAdbcAsyncTaskLaunchStreamGetNext", (DL_FUNC)&RAdbcAsyncTaskLaunchStreamGetNext, 3},
     {"RAdbcVoidDriverInitFunc", (DL_FUNC)&RAdbcVoidDriverInitFunc, 0},
     {"RAdbcMonkeyDriverInitFunc", (DL_FUNC)&RAdbcMonkeyDriverInitFunc, 0},