Fix "not a tty" bug on Solaris 10

* configure.ac (PTY_OPEN): Define to plain 'open'
on SVR4-derived hosts, so that the O_CLOEXEC flag isn't set.
* src/process.c (allocate_pty): Set the O_CLOEXEC flag after
calling PTY_TTY_NAME_SPRINTF, for the benefit of SVR4-derived
hosts that call grantpt which does its work via a setuid subcommand
(Bug#19191, Bug#19927, Bug#20555, Bug#20686).
Also, set O_CLOEXEC even if PTY_OPEN is not defined, since it
seems relevant in that case too.

diff --git a/configure.ac b/configure.ac
index 9c6a74afacef87c7c5fa0fe9c0e9e37937da4a80..070b06124715b0a11e54f627900cf88dced61011 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -4397,14 +4397,17 @@ case $opsys in
     ;;
 
   sol2* )
-    dnl On SysVr4, grantpt(3) forks a subprocess, so keep sigchld_handler()
+    dnl On SysVr4, grantpt(3) forks a subprocess, so do not use
+    dnl O_CLOEXEC when opening the pty, and keep the SIGCHLD handler
     dnl from intercepting that death.  If any child but grantpt's should die
     dnl within, it should be caught after sigrelse(2).
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt (fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1 || unlockpt (fd) == -1 || !(ptyname = ptsname (fd))) { emacs_close (fd); return -1; } snprintf (pty_name, PTY_NAME_SIZE, "%s", ptyname); }])
     ;;
 
   unixware )
     dnl Comments are as per sol2*.
+    AC_DEFINE(PTY_OPEN, [fd = open (pty_name, O_RDWR | O_NONBLOCK)])
     AC_DEFINE(PTY_TTY_NAME_SPRINTF, [{ char *ptsname (int), *ptyname; int grantpt_result; sigset_t blocked; sigemptyset (&blocked); sigaddset (&blocked, SIGCHLD); pthread_sigmask (SIG_BLOCK, &blocked, 0); grantpt_result = grantpt (fd); pthread_sigmask (SIG_UNBLOCK, &blocked, 0); if (grantpt_result == -1) fatal("could not grant slave pty"); if (unlockpt(fd) == -1) fatal("could not unlock slave pty"); if (!(ptyname = ptsname(fd))) fatal ("could not enable slave pty"); snprintf (pty_name, PTY_NAME_SIZE, "%s", ptyname); }])
     ;;
 esac

diff --git a/src/process.c b/src/process.c
index 17fe708a954a995b0f925ec2f7b81e36560bad6c..b4f979fd4840431a61de5dcbe10aa8944552b0d1 100644 (file)
--- a/src/process.c
+++ b/src/process.c
@@ -658,22 +658,24 @@ allocate_pty (char pty_name[PTY_NAME_SIZE])
 
        if (fd >= 0)
          {
-#ifdef PTY_OPEN
+#ifdef PTY_TTY_NAME_SPRINTF
+           PTY_TTY_NAME_SPRINTF
+#else
+           sprintf (pty_name, "/dev/tty%c%x", c, i);
+#endif /* no PTY_TTY_NAME_SPRINTF */
+
            /* Set FD's close-on-exec flag.  This is needed even if
               PT_OPEN calls posix_openpt with O_CLOEXEC, since POSIX
               doesn't require support for that combination.
+              Do this after PTY_TTY_NAME_SPRINTF, which on some platforms
+              doesn't work if the close-on-exec flag is set (Bug#20555).
               Multithreaded platforms where posix_openpt ignores
               O_CLOEXEC (or where PTY_OPEN doesn't call posix_openpt)
               have a race condition between the PTY_OPEN and here.  */
            fcntl (fd, F_SETFD, FD_CLOEXEC);
-#endif
-           /* Check to make certain that both sides are available
-              this avoids a nasty yet stupid bug in rlogins.  */
-#ifdef PTY_TTY_NAME_SPRINTF
-           PTY_TTY_NAME_SPRINTF
-#else
-           sprintf (pty_name, "/dev/tty%c%x", c, i);
-#endif /* no PTY_TTY_NAME_SPRINTF */
+
+           /* Check to make certain that both sides are available.
+              This avoids a nasty yet stupid bug in rlogins.  */
            if (faccessat (AT_FDCWD, pty_name, R_OK | W_OK, AT_EACCESS) != 0)
              {
                emacs_close (fd);