add openpty and forkpty implementation for illumos systems

At time of writing, illumos systems do not provide an implementation of
the openpty() and forkpty() wrappers provided on some other UNIX
systems.  While we expect to grow an implementation, it seems prudent to
provide a compatibility routine here first to unblock illumos support in
the popular nix crate.

libc-test/build.rs

@@ -725,6 +725,7 @@ fn test_solarish(target: &str) {
         "sys/socket.h",
         "sys/stat.h",
         "sys/statvfs.h",
+        "sys/stropts.h",
         "sys/shm.h",
         "sys/time.h",
         "sys/times.h",

src/unix/solarish/compat.rs

@@ -3,6 +3,9 @@
 
 use unix::solarish::*;
 
+const PTEM: &[u8] = b"ptem\0";
+const LDTERM: &[u8] = b"ldterm\0";
+
 pub unsafe fn cfmakeraw(termios: *mut ::termios) {
     (*termios).c_iflag &= !(IMAXBEL
         | IGNBRK
@@ -45,3 +48,136 @@ pub unsafe fn cfsetspeed(
     ::cfsetospeed(termios, speed);
     0
 }
+
+unsafe fn bail(fdm: ::c_int, fds: ::c_int) -> ::c_int {
+    let e = *___errno();
+    if fds >= 0 {
+        ::close(fds);
+    }
+    if fdm >= 0 {
+        ::close(fdm);
+    }
+    *___errno() = e;
+    return -1;
+}
+
+pub unsafe fn openpty(
+    amain: *mut ::c_int,
+    asubord: *mut ::c_int,
+    name: *mut ::c_char,
+    termp: *const termios,
+    winp: *const ::winsize,
+) -> ::c_int {
+    // Open the main pseudo-terminal device, making sure not to set it as the
+    // controlling terminal for this process:
+    let fdm = ::posix_openpt(O_RDWR | O_NOCTTY);
+    if fdm < 0 {
+        return -1;
+    }
+
+    // Set permissions and ownership on the subordinate device and unlock it:
+    if ::grantpt(fdm) < 0 || ::unlockpt(fdm) < 0 {
+        return bail(fdm, -1);
+    }
+
+    // Get the path name of the subordinate device:
+    let subordpath = ::ptsname(fdm);
+    if subordpath.is_null() {
+        return bail(fdm, -1);
+    }
+
+    // Open the subordinate device without setting it as the controlling
+    // terminal for this process:
+    let fds = ::open(subordpath, O_RDWR | O_NOCTTY);
+    if fds < 0 {
+        return bail(fdm, -1);
+    }
+
+    // Check if the STREAMS modules are already pushed:
+    let setup = ::ioctl(fds, I_FIND, LDTERM.as_ptr());
+    if setup < 0 {
+        return bail(fdm, fds);
+    } else if setup == 0 {
+        // The line discipline is not present, so push the appropriate STREAMS
+        // modules for the subordinate device:
+        if ::ioctl(fds, I_PUSH, PTEM.as_ptr()) < 0
+            || ::ioctl(fds, I_PUSH, LDTERM.as_ptr()) < 0
+        {
+            return bail(fdm, fds);
+        }
+    }
+
+    // If provided, set the terminal parameters:
+    if !termp.is_null() && ::tcsetattr(fds, TCSAFLUSH, termp) != 0 {
+        return bail(fdm, fds);
+    }
+
+    // If provided, set the window size:
+    if !winp.is_null() && ::ioctl(fds, TIOCSWINSZ, winp) < 0 {
+        return bail(fdm, fds);
+    }
+
+    // If the caller wants the name of the subordinate device, copy it out.
+    //
+    // Note that this is a terrible interface: there appears to be no standard
+    // upper bound on the copy length for this pointer.  Nobody should pass
+    // anything but NULL here, preferring instead to use ptsname(3C) directly.
+    if !name.is_null() {
+        ::strcpy(name, subordpath);
+    }
+
+    *amain = fdm;
+    *asubord = fds;
+    0
+}
+
+pub unsafe fn forkpty(
+    amain: *mut ::c_int,
+    name: *mut ::c_char,
+    termp: *const termios,
+    winp: *const ::winsize,
+) -> ::pid_t {
+    let mut fds = -1;
+
+    if openpty(amain, &mut fds, name, termp, winp) != 0 {
+        return -1;
+    }
+
+    let pid = ::fork();
+    if pid < 0 {
+        return bail(*amain, fds);
+    } else if pid > 0 {
+        // In the parent process, we close the subordinate device and return the
+        // process ID of the new child:
+        ::close(fds);
+        return pid;
+    }
+
+    // The rest of this function executes in the child process.
+
+    // Close the main side of the pseudo-terminal pair:
+    ::close(*amain);
+
+    // Use TIOCSCTTY to set the subordinate device as our controlling
+    // terminal.  This will fail (with ENOTTY) if we are not the leader in
+    // our own session, so we call setsid() first.  Finally, arrange for
+    // the pseudo-terminal to occupy the standard I/O descriptors.
+    if ::setsid() < 0
+        || ::ioctl(fds, TIOCSCTTY, 0) < 0
+        || ::dup2(fds, 0) < 0
+        || ::dup2(fds, 1) < 0
+        || ::dup2(fds, 2) < 0
+    {
+        // At this stage there are no particularly good ways to handle failure.
+        // Exit as abruptly as possible, using _exit() to avoid messing with any
+        // state still shared with the parent process.
+        ::_exit(EXIT_FAILURE);
+    }
+    // Close the inherited descriptor, taking care to avoid closing the standard
+    // descriptors by mistake:
+    if fds > 2 {
+        ::close(fds);
+    }
+
+    0
+}

src/unix/solarish/mod.rs

@@ -1949,6 +1949,44 @@ pub const VLNEXT: usize = 15;
 pub const VSTATUS: usize = 16;
 pub const VERASE2: usize = 17;
 
+// <sys/stropts.h>
+const STR: ::c_int = (b'S' as ::c_int) << 8;
+pub const I_NREAD: ::c_int = STR | 0o1;
+pub const I_PUSH: ::c_int = STR | 0o2;
+pub const I_POP: ::c_int = STR | 0o3;
+pub const I_LOOK: ::c_int = STR | 0o4;
+pub const I_FLUSH: ::c_int = STR | 0o5;
+pub const I_SRDOPT: ::c_int = STR | 0o6;
+pub const I_GRDOPT: ::c_int = STR | 0o7;
+pub const I_STR: ::c_int = STR | 0o10;
+pub const I_SETSIG: ::c_int = STR | 0o11;
+pub const I_GETSIG: ::c_int = STR | 0o12;
+pub const I_FIND: ::c_int = STR | 0o13;
+pub const I_LINK: ::c_int = STR | 0o14;
+pub const I_UNLINK: ::c_int = STR | 0o15;
+pub const I_PEEK: ::c_int = STR | 0o17;
+pub const I_FDINSERT: ::c_int = STR | 0o20;
+pub const I_SENDFD: ::c_int = STR | 0o21;
+pub const I_RECVFD: ::c_int = STR | 0o16;
+pub const I_SWROPT: ::c_int = STR | 0o23;
+pub const I_GWROPT: ::c_int = STR | 0o24;
+pub const I_LIST: ::c_int = STR | 0o25;
+pub const I_PLINK: ::c_int = STR | 0o26;
+pub const I_PUNLINK: ::c_int = STR | 0o27;
+pub const I_ANCHOR: ::c_int = STR | 0o30;
+pub const I_FLUSHBAND: ::c_int = STR | 0o34;
+pub const I_CKBAND: ::c_int = STR | 0o35;
+pub const I_GETBAND: ::c_int = STR | 0o36;
+pub const I_ATMARK: ::c_int = STR | 0o37;
+pub const I_SETCLTIME: ::c_int = STR | 0o40;
+pub const I_GETCLTIME: ::c_int = STR | 0o41;
+pub const I_CANPUT: ::c_int = STR | 0o42;
+pub const I_SERROPT: ::c_int = STR | 0o43;
+pub const I_GERROPT: ::c_int = STR | 0o44;
+pub const I_ESETSIG: ::c_int = STR | 0o45;
+pub const I_EGETSIG: ::c_int = STR | 0o46;
+pub const __I_PUSH_NOCTTY: ::c_int = STR | 0o47;
+
 // 3SOCKET flags
 pub const SOCK_CLOEXEC: ::c_int = 0x080000;
 pub const SOCK_NONBLOCK: ::c_int = 0x100000;