Rewrite the aio module

The existing AIO implementation has some problems:
1) The in_progress field is checked at runtime, not compile time.
2) The mutable field is checked at runtime, not compile time.
3) A downstream lio_listio user must store extra state to track whether
   the whole operation is partially, completely, or not at all
   submitted.
4) Nix does heap allocation itself, rather than allowing the caller to
   choose it.  This can result in double (or triple, or quadruple)
   boxing.
5) There's no easy way to use lio_listio to submit multiple operations with
   a single syscall, but poll each individually.
6) The lio_listio usage is far from transparent and zero-cost.
7) No aio_readv or aio_writev support.
8) priority has type c_int; should be i32
9) aio_return should return a usize instead of an isize, since it only
   uses negative values to indicate errors, which Rust represents via
   the Result type.

This rewrite solves several problems:
1) Unsolved.  I don't think it can be solved without something like
   C++'s guaranteed type elision.  It might require changing the
   signature of Future::poll too.
2) Solved.
3) Solved, by the new in_progress method and by removing the complicated
   lio_listio resubmit code.
4) Solved.
5) Solved.
6) Solved, by removing the lio_listo resubmit code.  It can be
   reimplemented downstream if necessary.  Or even in Nix, but it
   doesn't fit Nix's theme of zero-cost abstractions.
7) Solved.
8) Solved.
9) Solved.

The rewrite includes functions that don't work on FreeBSD, so add CI
testing for FreeBSD 14 too.

By default only enable tests that will pass on FreeBSD 12.3.  But run a
CI job on FreeBSD 14 and set a flag that will enable such tests.

1 files changed, 551 insertions, 535 deletions

diff --git a/test/sys/test_aio.rs b/test/sys/test_aio.rs
index 80cd053f..ca35b5f8 100644
--- a/test/sys/test_aio.rs
+++ b/test/sys/test_aio.rs
@@ -1,415 +1,525 @@
-use libc::{c_int, c_void};
-use nix::Result;
-use nix::errno::*;
-use nix::sys::aio::*;
-use nix::sys::signal::{SaFlags, SigAction, sigaction, SigevNotify, SigHandler, Signal, SigSet};
-use nix::sys::time::{TimeSpec, TimeValLike};
-use std::io::{Write, Read, Seek, SeekFrom};
-use std::ops::Deref;
-use std::os::unix::io::AsRawFd;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::{thread, time};
+use std::{
+    io::{Read, Seek, SeekFrom, Write},
+    ops::Deref,
+    os::unix::io::AsRawFd,
+    pin::Pin,
+    sync::atomic::{AtomicBool, Ordering},
+    thread,
+    time,
+};
+
+use libc::c_int;
+use nix::{
+    errno::*,
+    sys::{
+        aio::*,
+        signal::{
+            sigaction,
+            SaFlags,
+            SigAction,
+            SigHandler,
+            SigSet,
+            SigevNotify,
+            Signal,
+        },
+        time::{TimeSpec, TimeValLike},
+    },
+};
 use tempfile::tempfile;
 
-// Helper that polls an AioCb for completion or error
-fn poll_aio(aiocb: &mut Pin<Box<AioCb>>) -> Result<()> {
-    loop {
-        let err = aiocb.error();
-        if err != Err(Errno::EINPROGRESS) { return err; };
-        thread::sleep(time::Duration::from_millis(10));
-    }
+lazy_static! {
+    pub static ref SIGNALED: AtomicBool = AtomicBool::new(false);
 }
 
-// Helper that polls a component of an LioCb for completion or error
-#[cfg(not(any(target_os = "ios", target_os = "macos")))]
-fn poll_lio(liocb: &mut LioCb, i: usize) -> Result<()> {
-    loop {
-        let err = liocb.error(i);
-        if err != Err(Errno::EINPROGRESS) { return err; };
-        thread::sleep(time::Duration::from_millis(10));
-    }
+extern "C" fn sigfunc(_: c_int) {
+    SIGNALED.store(true, Ordering::Relaxed);
 }
 
-#[test]
-fn test_accessors() {
-    let mut rbuf = vec![0; 4];
-    let aiocb = AioCb::from_mut_slice( 1001,
-                           2,   //offset
-                           &mut rbuf,
-                           42,   //priority
-                           SigevNotify::SigevSignal {
-                               signal: Signal::SIGUSR2,
-                               si_value: 99
-                           },
-                           LioOpcode::LIO_NOP);
-    assert_eq!(1001, aiocb.fd());
-    assert_eq!(Some(LioOpcode::LIO_NOP), aiocb.lio_opcode());
-    assert_eq!(4, aiocb.nbytes());
-    assert_eq!(2, aiocb.offset());
-    assert_eq!(42, aiocb.priority());
-    let sev = aiocb.sigevent().sigevent();
-    assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
-    assert_eq!(99, sev.sigev_value.sival_ptr as i64);
+// Helper that polls an AioCb for completion or error
+macro_rules! poll_aio {
+    ($aiocb: expr) => {
+        loop {
+            let err = $aiocb.as_mut().error();
+            if err != Err(Errno::EINPROGRESS) {
+                break err;
+            };
+            thread::sleep(time::Duration::from_millis(10));
+        }
+    };
 }
 
-// Tests AioCb.cancel.  We aren't trying to test the OS's implementation, only
-// our bindings.  So it's sufficient to check that AioCb.cancel returned any
-// AioCancelStat value.
-#[test]
-#[cfg_attr(target_env = "musl", ignore)]
-fn test_cancel() {
-    let wbuf: &[u8] = b"CDEF";
-
-    let f = tempfile().unwrap();
-    let mut aiocb = AioCb::from_slice( f.as_raw_fd(),
-                            0,   //offset
-                            wbuf,
-                            0,   //priority
-                            SigevNotify::SigevNone,
-                            LioOpcode::LIO_NOP);
-    aiocb.write().unwrap();
-    let err = aiocb.error();
-    assert!(err == Ok(()) || err == Err(Errno::EINPROGRESS));
+mod aio_fsync {
+    use super::*;
+
+    #[test]
+    fn test_accessors() {
+        let aiocb = AioFsync::new(
+            1001,
+            AioFsyncMode::O_SYNC,
+            42,
+            SigevNotify::SigevSignal {
+                signal:   Signal::SIGUSR2,
+                si_value: 99,
+            },
+        );
+        assert_eq!(1001, aiocb.fd());
+        assert_eq!(AioFsyncMode::O_SYNC, aiocb.mode());
+        assert_eq!(42, aiocb.priority());
+        let sev = aiocb.sigevent().sigevent();
+        assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
+        assert_eq!(99, sev.sigev_value.sival_ptr as i64);
+    }
 
-    let cancelstat = aiocb.cancel();
-    assert!(cancelstat.is_ok());
+    /// `AioFsync::submit` should not modify the `AioCb` object if
+    /// `libc::aio_fsync` returns an error
+    // Skip on Linux, because Linux's AIO implementation can't detect errors
+    // synchronously
+    #[test]
+    #[cfg(any(target_os = "freebsd", target_os = "macos"))]
+    fn error() {
+        use std::mem;
+
+        const INITIAL: &[u8] = b"abcdef123456";
+        // Create an invalid AioFsyncMode
+        let mode = unsafe { mem::transmute(666) };
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let mut aiof = Box::pin(AioFsync::new(
+            f.as_raw_fd(),
+            mode,
+            0,
+            SigevNotify::SigevNone,
+        ));
+        let err = aiof.as_mut().submit();
+        assert!(err.is_err());
+    }
 
-    // Wait for aiocb to complete, but don't care whether it succeeded
-    let _ = poll_aio(&mut aiocb);
-    let _ = aiocb.aio_return();
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn ok() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let fd = f.as_raw_fd();
+        let mut aiof = Box::pin(AioFsync::new(
+            fd,
+            AioFsyncMode::O_SYNC,
+            0,
+            SigevNotify::SigevNone,
+        ));
+        let err = aiof.as_mut().submit();
+        assert!(err.is_ok());
+        poll_aio!(&mut aiof).unwrap();
+        aiof.as_mut().aio_return().unwrap();
+    }
 }
 
-// Tests using aio_cancel_all for all outstanding IOs.
-#[test]
-#[cfg_attr(target_env = "musl", ignore)]
-fn test_aio_cancel_all() {
-    let wbuf: &[u8] = b"CDEF";
-
-    let f = tempfile().unwrap();
-    let mut aiocb = AioCb::from_slice(f.as_raw_fd(),
-                            0,   //offset
-                            wbuf,
-                            0,   //priority
-                            SigevNotify::SigevNone,
-                            LioOpcode::LIO_NOP);
-    aiocb.write().unwrap();
-    let err = aiocb.error();
-    assert!(err == Ok(()) || err == Err(Errno::EINPROGRESS));
+mod aio_read {
+    use super::*;
+
+    #[test]
+    fn test_accessors() {
+        let mut rbuf = vec![0; 4];
+        let aiocb = AioRead::new(
+            1001,
+            2, //offset
+            &mut rbuf,
+            42, //priority
+            SigevNotify::SigevSignal {
+                signal:   Signal::SIGUSR2,
+                si_value: 99,
+            },
+        );
+        assert_eq!(1001, aiocb.fd());
+        assert_eq!(4, aiocb.nbytes());
+        assert_eq!(2, aiocb.offset());
+        assert_eq!(42, aiocb.priority());
+        let sev = aiocb.sigevent().sigevent();
+        assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
+        assert_eq!(99, sev.sigev_value.sival_ptr as i64);
+    }
 
-    let cancelstat = aio_cancel_all(f.as_raw_fd());
-    assert!(cancelstat.is_ok());
+    // Tests AioWrite.cancel.  We aren't trying to test the OS's implementation,
+    // only our bindings.  So it's sufficient to check that cancel
+    // returned any AioCancelStat value.
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn cancel() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut rbuf = vec![0; 4];
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let fd = f.as_raw_fd();
+        let mut aior =
+            Box::pin(AioRead::new(fd, 2, &mut rbuf, 0, SigevNotify::SigevNone));
+        aior.as_mut().submit().unwrap();
+
+        let cancelstat = aior.as_mut().cancel();
+        assert!(cancelstat.is_ok());
+
+        // Wait for aiow to complete, but don't care whether it succeeded
+        let _ = poll_aio!(&mut aior);
+        let _ = aior.as_mut().aio_return();
+    }
 
-    // Wait for aiocb to complete, but don't care whether it succeeded
-    let _ = poll_aio(&mut aiocb);
-    let _ = aiocb.aio_return();
-}
+    /// `AioRead::submit` should not modify the `AioCb` object if
+    /// `libc::aio_read` returns an error
+    // Skip on Linux, because Linux's AIO implementation can't detect errors
+    // synchronously
+    #[test]
+    #[cfg(any(target_os = "freebsd", target_os = "macos"))]
+    fn error() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut rbuf = vec![0; 4];
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let mut aior = Box::pin(AioRead::new(
+            f.as_raw_fd(),
+            -1, //an invalid offset
+            &mut rbuf,
+            0, //priority
+            SigevNotify::SigevNone,
+        ));
+        assert!(aior.as_mut().submit().is_err());
+    }
 
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_fsync() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    let mut aiocb = AioCb::from_fd( f.as_raw_fd(),
-                            0,   //priority
-                            SigevNotify::SigevNone);
-    let err = aiocb.fsync(AioFsyncMode::O_SYNC);
-    assert!(err.is_ok());
-    poll_aio(&mut aiocb).unwrap();
-    aiocb.aio_return().unwrap();
-}
+    // Test a simple aio operation with no completion notification.  We must
+    // poll for completion
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn ok() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut rbuf = vec![0; 4];
+        const EXPECT: &[u8] = b"cdef";
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        {
+            let fd = f.as_raw_fd();
+            let mut aior = Box::pin(AioRead::new(
+                fd,
+                2,
+                &mut rbuf,
+                0,
+                SigevNotify::SigevNone,
+            ));
+            aior.as_mut().submit().unwrap();
 
-/// `AioCb::fsync` should not modify the `AioCb` object if `libc::aio_fsync` returns
-/// an error
-// Skip on Linux, because Linux's AIO implementation can't detect errors
-// synchronously
-#[test]
-#[cfg(any(target_os = "freebsd", target_os = "macos"))]
-fn test_fsync_error() {
-    use std::mem;
+            let err = poll_aio!(&mut aior);
+            assert_eq!(err, Ok(()));
+            assert_eq!(aior.as_mut().aio_return().unwrap(), EXPECT.len());
+        }
+        assert_eq!(EXPECT, rbuf.deref().deref());
+    }
 
-    const INITIAL: &[u8] = b"abcdef123456";
-    // Create an invalid AioFsyncMode
-    let mode = unsafe { mem::transmute(666) };
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    let mut aiocb = AioCb::from_fd( f.as_raw_fd(),
-                            0,   //priority
-                            SigevNotify::SigevNone);
-    let err = aiocb.fsync(mode);
-    assert!(err.is_err());
+    // Like ok, but allocates the structure on the stack.
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn on_stack() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut rbuf = vec![0; 4];
+        const EXPECT: &[u8] = b"cdef";
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        {
+            let fd = f.as_raw_fd();
+            let mut aior =
+                AioRead::new(fd, 2, &mut rbuf, 0, SigevNotify::SigevNone);
+            let mut aior = unsafe { Pin::new_unchecked(&mut aior) };
+            aior.as_mut().submit().unwrap();
+
+            let err = poll_aio!(&mut aior);
+            assert_eq!(err, Ok(()));
+            assert_eq!(aior.as_mut().aio_return().unwrap(), EXPECT.len());
+        }
+        assert_eq!(EXPECT, rbuf.deref().deref());
+    }
 }
 
-#[test]
-// On Cirrus on Linux, this test fails due to a glibc bug.
-// https://github.com/nix-rust/nix/issues/1099
-#[cfg_attr(target_os = "linux", ignore)]
-// On Cirrus, aio_suspend is failing with EINVAL
-// https://github.com/nix-rust/nix/issues/1361
-#[cfg_attr(target_os = "macos", ignore)]
-fn test_aio_suspend() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    const WBUF: &[u8] = b"CDEFG";
-    let timeout = TimeSpec::seconds(10);
-    let mut rbuf = vec![0; 4];
-    let rlen = rbuf.len();
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
+#[cfg(target_os = "freebsd")]
+#[cfg(fbsd14)]
+mod aio_readv {
+    use std::io::IoSliceMut;
+
+    use super::*;
+
+    #[test]
+    fn test_accessors() {
+        let mut rbuf0 = vec![0; 4];
+        let mut rbuf1 = vec![0; 8];
+        let mut rbufs =
+            [IoSliceMut::new(&mut rbuf0), IoSliceMut::new(&mut rbuf1)];
+        let aiocb = AioReadv::new(
+            1001,
+            2, //offset
+            &mut rbufs,
+            42, //priority
+            SigevNotify::SigevSignal {
+                signal:   Signal::SIGUSR2,
+                si_value: 99,
+            },
+        );
+        assert_eq!(1001, aiocb.fd());
+        assert_eq!(2, aiocb.iovlen());
+        assert_eq!(2, aiocb.offset());
+        assert_eq!(42, aiocb.priority());
+        let sev = aiocb.sigevent().sigevent();
+        assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
+        assert_eq!(99, sev.sigev_value.sival_ptr as i64);
+    }
 
-    let mut wcb = AioCb::from_slice( f.as_raw_fd(),
-                           2,   //offset
-                           WBUF,
-                           0,   //priority
-                           SigevNotify::SigevNone,
-                           LioOpcode::LIO_WRITE);
-
-    let mut rcb = AioCb::from_mut_slice( f.as_raw_fd(),
-                            8,   //offset
-                            &mut rbuf,
-                            0,   //priority
-                            SigevNotify::SigevNone,
-                            LioOpcode::LIO_READ);
-    wcb.write().unwrap();
-    rcb.read().unwrap();
-    loop {
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn ok() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let mut rbuf0 = vec![0; 4];
+        let mut rbuf1 = vec![0; 2];
+        let mut rbufs =
+            [IoSliceMut::new(&mut rbuf0), IoSliceMut::new(&mut rbuf1)];
+        const EXPECT0: &[u8] = b"cdef";
+        const EXPECT1: &[u8] = b"12";
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
         {
-            let cbbuf = [wcb.as_ref(), rcb.as_ref()];
-            let r = aio_suspend(&cbbuf[..], Some(timeout));
-            match r {
-                Err(Errno::EINTR) => continue,
-                Err(e) => panic!("aio_suspend returned {:?}", e),
-                Ok(_) => ()
-            };
-        }
-        if rcb.error() != Err(Errno::EINPROGRESS) &&
-           wcb.error() != Err(Errno::EINPROGRESS) {
-            break
+            let fd = f.as_raw_fd();
+            let mut aior = Box::pin(AioReadv::new(
+                fd,
+                2,
+                &mut rbufs,
+                0,
+                SigevNotify::SigevNone,
+            ));
+            aior.as_mut().submit().unwrap();
+
+            let err = poll_aio!(&mut aior);
+            assert_eq!(err, Ok(()));
+            assert_eq!(
+                aior.as_mut().aio_return().unwrap(),
+                EXPECT0.len() + EXPECT1.len()
+            );
         }
+        assert_eq!(&EXPECT0, &rbuf0);
+        assert_eq!(&EXPECT1, &rbuf1);
     }
-
-    assert_eq!(wcb.aio_return().unwrap() as usize, WBUF.len());
-    assert_eq!(rcb.aio_return().unwrap() as usize, rlen);
 }
 
-// Test a simple aio operation with no completion notification.  We must poll
-// for completion
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_read() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let mut rbuf = vec![0; 4];
-    const EXPECT: &[u8] = b"cdef";
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    {
-        let mut aiocb = AioCb::from_mut_slice( f.as_raw_fd(),
-                               2,   //offset
-                               &mut rbuf,
-                               0,   //priority
-                               SigevNotify::SigevNone,
-                               LioOpcode::LIO_NOP);
-        aiocb.read().unwrap();
-
-        let err = poll_aio(&mut aiocb);
-        assert_eq!(err, Ok(()));
-        assert_eq!(aiocb.aio_return().unwrap() as usize, EXPECT.len());
+mod aio_write {
+    use super::*;
+
+    #[test]
+    fn test_accessors() {
+        let wbuf = vec![0; 4];
+        let aiocb = AioWrite::new(
+            1001,
+            2, //offset
+            &wbuf,
+            42, //priority
+            SigevNotify::SigevSignal {
+                signal:   Signal::SIGUSR2,
+                si_value: 99,
+            },
+        );
+        assert_eq!(1001, aiocb.fd());
+        assert_eq!(4, aiocb.nbytes());
+        assert_eq!(2, aiocb.offset());
+        assert_eq!(42, aiocb.priority());
+        let sev = aiocb.sigevent().sigevent();
+        assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
+        assert_eq!(99, sev.sigev_value.sival_ptr as i64);
     }
 
-    assert_eq!(EXPECT, rbuf.deref().deref());
-}
+    // Tests AioWrite.cancel.  We aren't trying to test the OS's implementation,
+    // only our bindings.  So it's sufficient to check that cancel
+    // returned any AioCancelStat value.
+    #[test]
+    #[cfg_attr(target_env = "musl", ignore)]
+    fn cancel() {
+        let wbuf: &[u8] = b"CDEF";
 
-/// `AioCb::read` should not modify the `AioCb` object if `libc::aio_read`
-/// returns an error
-// Skip on Linux, because Linux's AIO implementation can't detect errors
-// synchronously
-#[test]
-#[cfg(any(target_os = "freebsd", target_os = "macos"))]
-fn test_read_error() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let mut rbuf = vec![0; 4];
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    let mut aiocb = AioCb::from_mut_slice( f.as_raw_fd(),
-                           -1,   //an invalid offset
-                           &mut rbuf,
-                           0,   //priority
-                           SigevNotify::SigevNone,
-                           LioOpcode::LIO_NOP);
-    assert!(aiocb.read().is_err());
-}
+        let f = tempfile().unwrap();
+        let mut aiow = Box::pin(AioWrite::new(
+            f.as_raw_fd(),
+            0,
+            wbuf,
+            0,
+            SigevNotify::SigevNone,
+        ));
+        aiow.as_mut().submit().unwrap();
+        let err = aiow.as_mut().error();
+        assert!(err == Ok(()) || err == Err(Errno::EINPROGRESS));
 
-// Tests from_mut_slice
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_read_into_mut_slice() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let mut rbuf = vec![0; 4];
-    const EXPECT: &[u8] = b"cdef";
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    {
-        let mut aiocb = AioCb::from_mut_slice( f.as_raw_fd(),
-                               2,   //offset
-                               &mut rbuf,
-                               0,   //priority
-                               SigevNotify::SigevNone,
-                               LioOpcode::LIO_NOP);
-        aiocb.read().unwrap();
-
-        let err = poll_aio(&mut aiocb);
-        assert_eq!(err, Ok(()));
-        assert_eq!(aiocb.aio_return().unwrap() as usize, EXPECT.len());
+        let cancelstat = aiow.as_mut().cancel();
+        assert!(cancelstat.is_ok());
+
+        // Wait for aiow to complete, but don't care whether it succeeded
+        let _ = poll_aio!(&mut aiow);
+        let _ = aiow.as_mut().aio_return();
     }
 
-    assert_eq!(rbuf, EXPECT);
-}
+    // Test a simple aio operation with no completion notification.  We must
+    // poll for completion.
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn ok() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let wbuf = "CDEF".to_string().into_bytes();
+        let mut rbuf = Vec::new();
+        const EXPECT: &[u8] = b"abCDEF123456";
+
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let mut aiow = Box::pin(AioWrite::new(
+            f.as_raw_fd(),
+            2,
+            &wbuf,
+            0,
+            SigevNotify::SigevNone,
+        ));
+        aiow.as_mut().submit().unwrap();
 
-// Tests from_ptr
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_read_into_pointer() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let mut rbuf = vec![0; 4];
-    const EXPECT: &[u8] = b"cdef";
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    {
-        // Safety: ok because rbuf lives until after poll_aio
-        let mut aiocb = unsafe {
-            AioCb::from_mut_ptr( f.as_raw_fd(),
-                                 2,   //offset
-                                 rbuf.as_mut_ptr() as *mut c_void,
-                                 rbuf.len(),
-                                 0,   //priority
-                                 SigevNotify::SigevNone,
-                                 LioOpcode::LIO_NOP)
-        };
-        aiocb.read().unwrap();
-
-        let err = poll_aio(&mut aiocb);
+        let err = poll_aio!(&mut aiow);
         assert_eq!(err, Ok(()));
-        assert_eq!(aiocb.aio_return().unwrap() as usize, EXPECT.len());
-    }
+        assert_eq!(aiow.as_mut().aio_return().unwrap(), wbuf.len());
 
-    assert_eq!(rbuf, EXPECT);
-}
-
-// Test reading into an immutable buffer.  It should fail
-// FIXME: This test fails to panic on Linux/musl
-#[test]
-#[should_panic(expected = "Can't read into an immutable buffer")]
-#[cfg_attr(target_env = "musl", ignore)]
-fn test_read_immutable_buffer() {
-    let rbuf: &[u8] = b"CDEF";
-    let f = tempfile().unwrap();
-    let mut aiocb = AioCb::from_slice( f.as_raw_fd(),
-                           2,   //offset
-                           rbuf,
-                           0,   //priority
-                           SigevNotify::SigevNone,
-                           LioOpcode::LIO_NOP);
-    aiocb.read().unwrap();
-}
+        f.seek(SeekFrom::Start(0)).unwrap();
+        let len = f.read_to_end(&mut rbuf).unwrap();
+        assert_eq!(len, EXPECT.len());
+        assert_eq!(rbuf, EXPECT);
+    }
 
+    // Like ok, but allocates the structure on the stack.
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn on_stack() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let wbuf = "CDEF".to_string().into_bytes();
+        let mut rbuf = Vec::new();
+        const EXPECT: &[u8] = b"abCDEF123456";
+
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let mut aiow = AioWrite::new(
+            f.as_raw_fd(),
+            2, //offset
+            &wbuf,
+            0, //priority
+            SigevNotify::SigevNone,
+        );
+        let mut aiow = unsafe { Pin::new_unchecked(&mut aiow) };
+        aiow.as_mut().submit().unwrap();
 
-// Test a simple aio operation with no completion notification.  We must poll
-// for completion.  Unlike test_aio_read, this test uses AioCb::from_slice
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_write() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let wbuf = "CDEF".to_string().into_bytes();
-    let mut rbuf = Vec::new();
-    const EXPECT: &[u8] = b"abCDEF123456";
+        let err = poll_aio!(&mut aiow);
+        assert_eq!(err, Ok(()));
+        assert_eq!(aiow.as_mut().aio_return().unwrap(), wbuf.len());
 
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    let mut aiocb = AioCb::from_slice( f.as_raw_fd(),
-                           2,   //offset
-                           &wbuf,
-                           0,   //priority
-                           SigevNotify::SigevNone,
-                           LioOpcode::LIO_NOP);
-    aiocb.write().unwrap();
-
-    let err = poll_aio(&mut aiocb);
-    assert_eq!(err, Ok(()));
-    assert_eq!(aiocb.aio_return().unwrap() as usize, wbuf.len());
+        f.seek(SeekFrom::Start(0)).unwrap();
+        let len = f.read_to_end(&mut rbuf).unwrap();
+        assert_eq!(len, EXPECT.len());
+        assert_eq!(rbuf, EXPECT);
+    }
 
-    f.seek(SeekFrom::Start(0)).unwrap();
-    let len = f.read_to_end(&mut rbuf).unwrap();
-    assert_eq!(len, EXPECT.len());
-    assert_eq!(rbuf, EXPECT);
+    /// `AioWrite::write` should not modify the `AioCb` object if
+    /// `libc::aio_write` returns an error.
+    // Skip on Linux, because Linux's AIO implementation can't detect errors
+    // synchronously
+    #[test]
+    #[cfg(any(target_os = "freebsd", target_os = "macos"))]
+    fn error() {
+        let wbuf = "CDEF".to_string().into_bytes();
+        let mut aiow = Box::pin(AioWrite::new(
+            666, // An invalid file descriptor
+            0,   //offset
+            &wbuf,
+            0, //priority
+            SigevNotify::SigevNone,
+        ));
+        assert!(aiow.as_mut().submit().is_err());
+        // Dropping the AioWrite at this point should not panic
+    }
 }
 
-// Tests `AioCb::from_ptr`
-#[test]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_write_from_pointer() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    let wbuf = "CDEF".to_string().into_bytes();
-    let mut rbuf = Vec::new();
-    const EXPECT: &[u8] = b"abCDEF123456";
-
-    let mut f = tempfile().unwrap();
-    f.write_all(INITIAL).unwrap();
-    // Safety: ok because aiocb outlives poll_aio
-    let mut aiocb = unsafe {
-        AioCb::from_ptr( f.as_raw_fd(),
-                         2,   //offset
-                         wbuf.as_ptr() as *const c_void,
-                         wbuf.len(),
-                         0,   //priority
-                         SigevNotify::SigevNone,
-                         LioOpcode::LIO_NOP)
-    };
-    aiocb.write().unwrap();
-
-    let err = poll_aio(&mut aiocb);
-    assert_eq!(err, Ok(()));
-    assert_eq!(aiocb.aio_return().unwrap() as usize, wbuf.len());
-
-    f.seek(SeekFrom::Start(0)).unwrap();
-    let len = f.read_to_end(&mut rbuf).unwrap();
-    assert_eq!(len, EXPECT.len());
-    assert_eq!(rbuf, EXPECT);
-}
+#[cfg(target_os = "freebsd")]
+#[cfg(fbsd14)]
+mod aio_writev {
+    use std::io::IoSlice;
+
+    use super::*;
+
+    #[test]
+    fn test_accessors() {
+        let wbuf0 = vec![0; 4];
+        let wbuf1 = vec![0; 8];
+        let wbufs = [IoSlice::new(&wbuf0), IoSlice::new(&wbuf1)];
+        let aiocb = AioWritev::new(
+            1001,
+            2, //offset
+            &wbufs,
+            42, //priority
+            SigevNotify::SigevSignal {
+                signal:   Signal::SIGUSR2,
+                si_value: 99,
+            },
+        );
+        assert_eq!(1001, aiocb.fd());
+        assert_eq!(2, aiocb.iovlen());
+        assert_eq!(2, aiocb.offset());
+        assert_eq!(42, aiocb.priority());
+        let sev = aiocb.sigevent().sigevent();
+        assert_eq!(Signal::SIGUSR2 as i32, sev.sigev_signo);
+        assert_eq!(99, sev.sigev_value.sival_ptr as i64);
+    }
 
-/// `AioCb::write` should not modify the `AioCb` object if `libc::aio_write`
-/// returns an error
-// Skip on Linux, because Linux's AIO implementation can't detect errors
-// synchronously
-#[test]
-#[cfg(any(target_os = "freebsd", target_os = "macos"))]
-fn test_write_error() {
-    let wbuf = "CDEF".to_string().into_bytes();
-    let mut aiocb = AioCb::from_slice( 666, // An invalid file descriptor
-                           0,   //offset
-                           &wbuf,
-                           0,   //priority
-                           SigevNotify::SigevNone,
-                           LioOpcode::LIO_NOP);
-    assert!(aiocb.write().is_err());
-}
+    // Test a simple aio operation with no completion notification.  We must
+    // poll for completion.
+    #[test]
+    #[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
+    fn ok() {
+        const INITIAL: &[u8] = b"abcdef123456";
+        let wbuf0 = b"BC";
+        let wbuf1 = b"DEF";
+        let wbufs = [IoSlice::new(wbuf0), IoSlice::new(wbuf1)];
+        let wlen = wbuf0.len() + wbuf1.len();
+        let mut rbuf = Vec::new();
+        const EXPECT: &[u8] = b"aBCDEF123456";
+
+        let mut f = tempfile().unwrap();
+        f.write_all(INITIAL).unwrap();
+        let mut aiow = Box::pin(AioWritev::new(
+            f.as_raw_fd(),
+            1,
+            &wbufs,
+            0,
+            SigevNotify::SigevNone,
+        ));
+        aiow.as_mut().submit().unwrap();
 
-lazy_static! {
-    pub static ref SIGNALED: AtomicBool = AtomicBool::new(false);
-}
+        let err = poll_aio!(&mut aiow);
+        assert_eq!(err, Ok(()));
+        assert_eq!(aiow.as_mut().aio_return().unwrap(), wlen);
 
-extern fn sigfunc(_: c_int) {
-    SIGNALED.store(true, Ordering::Relaxed);
+        f.seek(SeekFrom::Start(0)).unwrap();
+        let len = f.read_to_end(&mut rbuf).unwrap();
+        assert_eq!(len, EXPECT.len());
+        assert_eq!(rbuf, EXPECT);
+    }
 }
 
 // Test an aio operation with completion delivered by a signal
-// FIXME: This test is ignored on mips because of failures in qemu in CI
 #[test]
-#[cfg_attr(any(all(target_env = "musl", target_arch = "x86_64"), target_arch = "mips", target_arch = "mips64"), ignore)]
-fn test_write_sigev_signal() {
+#[cfg_attr(
+    any(
+        all(target_env = "musl", target_arch = "x86_64"),
+        target_arch = "mips",
+        target_arch = "mips64"
+    ),
+    ignore
+)]
+fn sigev_signal() {
     let _m = crate::SIGNAL_MTX.lock();
-    let sa = SigAction::new(SigHandler::Handler(sigfunc),
-                            SaFlags::SA_RESETHAND,
-                            SigSet::empty());
+    let sa = SigAction::new(
+        SigHandler::Handler(sigfunc),
+        SaFlags::SA_RESETHAND,
+        SigSet::empty(),
+    );
     SIGNALED.store(false, Ordering::Relaxed);
     unsafe { sigaction(Signal::SIGUSR2, &sa) }.unwrap();
 
@@ -420,201 +530,107 @@ fn test_write_sigev_signal() {
 
     let mut f = tempfile().unwrap();
     f.write_all(INITIAL).unwrap();
-    let mut aiocb = AioCb::from_slice( f.as_raw_fd(),
-                           2,   //offset
-                           WBUF,
-                           0,   //priority
-                           SigevNotify::SigevSignal {
-                               signal: Signal::SIGUSR2,
-                               si_value: 0  //TODO: validate in sigfunc
-                           },
-                           LioOpcode::LIO_NOP);
-    aiocb.write().unwrap();
+    let mut aiow = Box::pin(AioWrite::new(
+        f.as_raw_fd(),
+        2, //offset
+        WBUF,
+        0, //priority
+        SigevNotify::SigevSignal {
+            signal:   Signal::SIGUSR2,
+            si_value: 0, //TODO: validate in sigfunc
+        },
+    ));
+    aiow.as_mut().submit().unwrap();
     while !SIGNALED.load(Ordering::Relaxed) {
         thread::sleep(time::Duration::from_millis(10));
     }
 
-    assert_eq!(aiocb.aio_return().unwrap() as usize, WBUF.len());
+    assert_eq!(aiow.as_mut().aio_return().unwrap(), WBUF.len());
     f.seek(SeekFrom::Start(0)).unwrap();
     let len = f.read_to_end(&mut rbuf).unwrap();
     assert_eq!(len, EXPECT.len());
     assert_eq!(rbuf, EXPECT);
 }
 
-// Test LioCb::listio with LIO_WAIT, so all AIO ops should be complete by the
-// time listio returns.
-#[test]
-#[cfg(not(any(target_os = "ios", target_os = "macos")))]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_liocb_listio_wait() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    const WBUF: &[u8] = b"CDEF";
-    let mut rbuf = vec![0; 4];
-    let rlen = rbuf.len();
-    let mut rbuf2 = Vec::new();
-    const EXPECT: &[u8] = b"abCDEF123456";
-    let mut f = tempfile().unwrap();
-
-    f.write_all(INITIAL).unwrap();
-
-    {
-        let mut liocb = LioCbBuilder::with_capacity(2)
-            .emplace_slice(
-                f.as_raw_fd(),
-                2,   //offset
-                WBUF,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_WRITE
-            ).emplace_mut_slice(
-                f.as_raw_fd(),
-                8,   //offset
-                &mut rbuf,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_READ
-            ).finish();
-        let err = liocb.listio(LioMode::LIO_WAIT, SigevNotify::SigevNone);
-        err.expect("lio_listio");
-
-        assert_eq!(liocb.aio_return(0).unwrap() as usize, WBUF.len());
-        assert_eq!(liocb.aio_return(1).unwrap() as usize, rlen);
-    }
-    assert_eq!(rbuf.deref().deref(), b"3456");
-
-    f.seek(SeekFrom::Start(0)).unwrap();
-    let len = f.read_to_end(&mut rbuf2).unwrap();
-    assert_eq!(len, EXPECT.len());
-    assert_eq!(rbuf2, EXPECT);
-}
-
-// Test LioCb::listio with LIO_NOWAIT and no SigEvent, so we must use some other
-// mechanism to check for the individual AioCb's completion.
+// Tests using aio_cancel_all for all outstanding IOs.
 #[test]
-#[cfg(not(any(target_os = "ios", target_os = "macos")))]
-#[cfg_attr(all(target_env = "musl", target_arch = "x86_64"), ignore)]
-fn test_liocb_listio_nowait() {
-    const INITIAL: &[u8] = b"abcdef123456";
-    const WBUF: &[u8] = b"CDEF";
-    let mut rbuf = vec![0; 4];
-    let rlen = rbuf.len();
-    let mut rbuf2 = Vec::new();
-    const EXPECT: &[u8] = b"abCDEF123456";
-    let mut f = tempfile().unwrap();
+#[cfg_attr(target_env = "musl", ignore)]
+fn test_aio_cancel_all() {
+    let wbuf: &[u8] = b"CDEF";
 
-    f.write_all(INITIAL).unwrap();
+    let f = tempfile().unwrap();
+    let mut aiocb = Box::pin(AioWrite::new(
+        f.as_raw_fd(),
+        0, //offset
+        wbuf,
+        0, //priority
+        SigevNotify::SigevNone,
+    ));
+    aiocb.as_mut().submit().unwrap();
+    let err = aiocb.as_mut().error();
+    assert!(err == Ok(()) || err == Err(Errno::EINPROGRESS));
 
-    {
-        let mut liocb = LioCbBuilder::with_capacity(2)
-            .emplace_slice(
-                f.as_raw_fd(),
-                2,   //offset
-                WBUF,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_WRITE
-            ).emplace_mut_slice(
-                f.as_raw_fd(),
-                8,   //offset
-                &mut rbuf,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_READ
-            ).finish();
-        let err = liocb.listio(LioMode::LIO_NOWAIT, SigevNotify::SigevNone);
-        err.expect("lio_listio");
-
-        poll_lio(&mut liocb, 0).unwrap();
-        poll_lio(&mut liocb, 1).unwrap();
-        assert_eq!(liocb.aio_return(0).unwrap() as usize, WBUF.len());
-        assert_eq!(liocb.aio_return(1).unwrap() as usize, rlen);
-    }
-    assert_eq!(rbuf.deref().deref(), b"3456");
+    let cancelstat = aio_cancel_all(f.as_raw_fd());
+    assert!(cancelstat.is_ok());
 
-    f.seek(SeekFrom::Start(0)).unwrap();
-    let len = f.read_to_end(&mut rbuf2).unwrap();
-    assert_eq!(len, EXPECT.len());
-    assert_eq!(rbuf2, EXPECT);
+    // Wait for aiocb to complete, but don't care whether it succeeded
+    let _ = poll_aio!(&mut aiocb);
+    let _ = aiocb.as_mut().aio_return();
 }
 
-// Test LioCb::listio with LIO_NOWAIT and a SigEvent to indicate when all
-// AioCb's are complete.
-// FIXME: This test is ignored on mips/mips64 because of failures in qemu in CI.
 #[test]
-#[cfg(not(any(target_os = "ios", target_os = "macos")))]
-#[cfg_attr(any(target_arch = "mips", target_arch = "mips64", target_env = "musl"), ignore)]
-fn test_liocb_listio_signal() {
-    let _m = crate::SIGNAL_MTX.lock();
+// On Cirrus on Linux, this test fails due to a glibc bug.
+// https://github.com/nix-rust/nix/issues/1099
+#[cfg_attr(target_os = "linux", ignore)]
+// On Cirrus, aio_suspend is failing with EINVAL
+// https://github.com/nix-rust/nix/issues/1361
+#[cfg_attr(target_os = "macos", ignore)]
+fn test_aio_suspend() {
     const INITIAL: &[u8] = b"abcdef123456";
-    const WBUF: &[u8] = b"CDEF";
+    const WBUF: &[u8] = b"CDEFG";
+    let timeout = TimeSpec::seconds(10);
     let mut rbuf = vec![0; 4];
     let rlen = rbuf.len();
-    let mut rbuf2 = Vec::new();
-    const EXPECT: &[u8] = b"abCDEF123456";
     let mut f = tempfile().unwrap();
-    let sa = SigAction::new(SigHandler::Handler(sigfunc),
-                            SaFlags::SA_RESETHAND,
-                            SigSet::empty());
-    let sigev_notify = SigevNotify::SigevSignal { signal: Signal::SIGUSR2,
-                                                  si_value: 0 };
-
     f.write_all(INITIAL).unwrap();
 
-    {
-        let mut liocb = LioCbBuilder::with_capacity(2)
-            .emplace_slice(
-                f.as_raw_fd(),
-                2,   //offset
-                WBUF,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_WRITE
-            ).emplace_mut_slice(
-                f.as_raw_fd(),
-                8,   //offset
-                &mut rbuf,
-                0,   //priority
-                SigevNotify::SigevNone,
-                LioOpcode::LIO_READ
-            ).finish();
-        SIGNALED.store(false, Ordering::Relaxed);
-        unsafe { sigaction(Signal::SIGUSR2, &sa) }.unwrap();
-        let err = liocb.listio(LioMode::LIO_NOWAIT, sigev_notify);
-        err.expect("lio_listio");
-        while !SIGNALED.load(Ordering::Relaxed) {
-            thread::sleep(time::Duration::from_millis(10));
+    let mut wcb = Box::pin(AioWrite::new(
+        f.as_raw_fd(),
+        2, //offset
+        WBUF,
+        0, //priority
+        SigevNotify::SigevNone,
+    ));
+
+    let mut rcb = Box::pin(AioRead::new(
+        f.as_raw_fd(),
+        8, //offset
+        &mut rbuf,
+        0, //priority
+        SigevNotify::SigevNone,
+    ));
+    wcb.as_mut().submit().unwrap();
+    rcb.as_mut().submit().unwrap();
+    loop {
+        {
+            let cbbuf = [
+                &*wcb as &dyn AsRef<libc::aiocb>,
+                &*rcb as &dyn AsRef<libc::aiocb>,
+            ];
+            let r = aio_suspend(&cbbuf[..], Some(timeout));
+            match r {
+                Err(Errno::EINTR) => continue,
+                Err(e) => panic!("aio_suspend returned {:?}", e),
+                Ok(_) => (),
+            };
+        }
+        if rcb.as_mut().error() != Err(Errno::EINPROGRESS) &&
+            wcb.as_mut().error() != Err(Errno::EINPROGRESS)
+        {
+            break;
         }
-
-        assert_eq!(liocb.aio_return(0).unwrap() as usize, WBUF.len());
-        assert_eq!(liocb.aio_return(1).unwrap() as usize, rlen);
     }
-    assert_eq!(rbuf.deref().deref(), b"3456");
-
-    f.seek(SeekFrom::Start(0)).unwrap();
-    let len = f.read_to_end(&mut rbuf2).unwrap();
-    assert_eq!(len, EXPECT.len());
-    assert_eq!(rbuf2, EXPECT);
-}
 
-// Try to use LioCb::listio to read into an immutable buffer.  It should fail
-// FIXME: This test fails to panic on Linux/musl
-#[test]
-#[cfg(not(any(target_os = "ios", target_os = "macos")))]
-#[should_panic(expected = "Can't read into an immutable buffer")]
-#[cfg_attr(target_env = "musl", ignore)]
-fn test_liocb_listio_read_immutable() {
-    let rbuf: &[u8] = b"abcd";
-    let f = tempfile().unwrap();
-
-
-    let mut liocb = LioCbBuilder::with_capacity(1)
-        .emplace_slice(
-            f.as_raw_fd(),
-            2,   //offset
-            rbuf,
-            0,   //priority
-            SigevNotify::SigevNone,
-            LioOpcode::LIO_READ
-        ).finish();
-    let _ = liocb.listio(LioMode::LIO_NOWAIT, SigevNotify::SigevNone);
+    assert_eq!(wcb.as_mut().aio_return().unwrap(), WBUF.len());
+    assert_eq!(rcb.as_mut().aio_return().unwrap(), rlen);
 }