pty: open the slave before the fork, so a pane's size reaches itv0.1.1

On Darwin a pty master has no line discipline until its slave has been
opened: every TIOCSWINSZ on the master until then fails with ENOTTY and
the size is dropped without a word. A pane is born 1x1 and told how big
it really is by the layout an instant after the fork -- and with the
child doing the open(), that message raced the child and usually lost.
Nothing ever sent it again, so the program in the pane spent its whole
life believing it had one row and one column. `C-a e` hit it every
time, because the editor's pane is always brand new: nvim drew its
screen into two rows in the corner of a 94x15 pane.

The old code knew half of this -- the child set the winsize on the
slave it had just opened, which is what kept a fresh pane from being
0x0 -- but that only fixed the *spawn* size. Every resize after it
still went to a master that might not have a slave yet.

So open the slave here, before the fork, and let the child inherit it.
The fork copies the fd table, so the slave is open from the child's
first instruction and every resize the parent sends afterwards lands;
BSD's forkpty() does the same thing for the same reason. The winsize is
set on that slave before the fork, which orders it ahead of anything
the parent sends later, and the child no longer sets it at all: by the
time the child ran, the size it was spawned with could already be
stale, and writing it back would undo the very resize this is about.

test_pty_size.py asks the pane's own program what size it thinks it is
-- for the first pane, for a pane from a split, and after a resize --
and compares it to the pane. It printed "1 1" against a 74x18 pane
before this.

live_m0.py asked `tput cols`, which needs a terminfo entry for
xterm-ghostty and therefore fails on most macs for a reason that has
nothing to do with resizing. `stty size` asks the tty itself, which is
the thing under test.

3 files changed, 138 insertions, 11 deletions

diff --git a/src/pty.c b/src/pty.c
index 7f7ddb5..dbabcef 100644
--- a/src/pty.c
+++ b/src/pty.c
@@ -29,18 +29,49 @@ int pty_spawn(pty_t *p, const char *const argv[], uint16_t cols, uint16_t rows,
     return -1;
   }
 
+  /* The slave is opened *here*, before the fork, and not by the child.
+   *
+   * Darwin attaches no line discipline to a pty master until its slave has
+   * been opened: until then every TIOCSWINSZ on the master fails with ENOTTY
+   * and the size is silently dropped. A pane is born 1x1 and told its real
+   * size by the layout an instant later -- so with the child opening the
+   * slave, that message raced the child's open() and usually lost, and
+   * nothing ever sent it again. The pane's program spent its life believing
+   * it had one row and one column, which is how `C-a e` produced an editor
+   * drawing into a corner of the pane on macOS and nowhere else.
+   *
+   * Opening it in the parent closes the window entirely: the fork copies the
+   * fd table, so the slave is open from the child's first instruction, and
+   * every resize the parent sends afterwards lands. It is also what BSD
+   * forkpty() does, for this reason.
+   *
+   * O_NOCTTY: we are not the process that wants this terminal. The child is,
+   * and it says so with TIOCSCTTY after setsid(). */
+  int slave = open(slave_name, O_RDWR | O_NOCTTY);
+  if (slave < 0) {
+    close(master);
+    return -1;
+  }
+
   /* The pixel fields matter: a program drawing images asks the tty how big a
    * cell is, and a winsize that says zero is a program that cannot size an
    * image. They are the *terminal's* pixels, so they are cols/rows times the
-   * cell size the client reported. */
+   * cell size the client reported.
+   *
+   * Set on the slave, which works on Linux and Darwin alike, and set before
+   * the fork so it is ordered ahead of any resize the parent sends after it.
+   * The child must not set it: by the time it ran, the size it was spawned
+   * with could already be stale, and writing it back would undo the resize we
+   * are here to deliver. */
   struct winsize ws = {.ws_col = cols,
                        .ws_row = rows,
                        .ws_xpixel = (unsigned short)(cols * cell_w),
                        .ws_ypixel = (unsigned short)(rows * cell_h)};
-  ioctl(master, TIOCSWINSZ, &ws);
+  ioctl(slave, TIOCSWINSZ, &ws);
 
   pid_t pid = fork();
   if (pid < 0) {
+    close(slave);
     close(master);
     return -1;
   }
@@ -48,14 +79,7 @@ int pty_spawn(pty_t *p, const char *const argv[], uint16_t cols, uint16_t rows,
   if (pid == 0) {
     /* child: new session, pty becomes the controlling terminal */
     setsid();
-    int slave = open(slave_name, O_RDWR);
-    if (slave < 0) _exit(127);
     ioctl(slave, TIOCSCTTY, 0);
-    /* Darwin attaches no line discipline to the master until the slave is
-     * opened, so the TIOCSWINSZ above returns ENOTTY there and the size is
-     * dropped: the pane starts 0x0 and stays that way until something else
-     * resizes it. Setting it here, on the slave we just opened, works on both. */
-    ioctl(slave, TIOCSWINSZ, &ws);
     dup2(slave, STDIN_FILENO);
     dup2(slave, STDOUT_FILENO);
     dup2(slave, STDERR_FILENO);
@@ -78,7 +102,9 @@ int pty_spawn(pty_t *p, const char *const argv[], uint16_t cols, uint16_t rows,
     _exit(127);
   }
 
-  /* parent */
+  /* parent: the child holds the slave now, and holding a second copy would
+   * keep the master readable long after the program in the pane had gone. */
+  close(slave);
   int flags = fcntl(master, F_GETFL, 0);
   fcntl(master, F_SETFL, flags | O_NONBLOCK);
   p->fd = master;
diff --git a/tests/live_m0.py b/tests/live_m0.py
index ff2e3c2..3bcecb8 100644
--- a/tests/live_m0.py
+++ b/tests/live_m0.py
@@ -78,7 +78,12 @@ def main():
     fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", 20, 100, 0, 0))
     os.kill(pid, signal.SIGWINCH)
     time.sleep(0.2)
-    os.write(fd, b"tput cols\n")
+    # `stty size` and not `tput cols`: it asks the tty itself, which is the
+    # thing under test, and it needs no terminfo entry -- tput could only
+    # report "unknown terminal xterm-ghostty" on a machine without ghostty's
+    # terminfo installed, which is most macs, and failed there for a reason
+    # that had nothing to do with resizing.
+    os.write(fd, b"stty size\n")
     out = drain(fd)
     # the pane is told its *content* width: 100 columns minus the gap (2 each
     # side, aspect-corrected) minus the frame border (1 each side)
diff --git a/tests/test_pty_size.py b/tests/test_pty_size.py
new file mode 100644
index 0000000..77afd6e
--- /dev/null
+++ b/tests/test_pty_size.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""A pane's program is told how big the pane is.
+
+A pane is born 1x1 and given its real size by the layout a moment later, so
+everything a program knows about its own width and height arrives through that
+second message. If it is dropped, the program spends its whole life believing
+it has one row and one column -- which is a shell wrapping every prompt and an
+editor drawing its screen into a corner.
+
+Darwin drops it if you are not careful: a pty master has no line discipline
+until its slave has been opened, so TIOCSWINSZ on the master returns ENOTTY,
+and the fresh pane's resize races the child's open of the slave. Losing that
+race is silent and permanent, so the size is asserted here rather than left to
+be noticed in an editor.
+
+`stty size` prints "rows cols", and the sleep is the point: it reads the size
+*after* the layout has had every chance to deliver it.
+"""
+
+import sys
+
+from harness import Session, check, report
+
+# Report the size, then stay up so the pane is still there to be measured.
+REPORT = ["/bin/sh", "-c", "sleep 0.3; stty size; read x"]
+
+
+def reported(snap, pane):
+    """The "rows cols" the program printed, as (rows, cols)."""
+    for row in range(pane["content_h"]):
+        line = snap.pane_line(pane, row).strip()
+        parts = line.split()
+        if len(parts) == 2 and all(p.isdigit() for p in parts):
+            return int(parts[0]), int(parts[1])
+    return None
+
+
+def test_the_first_pane_is_told_its_size():
+    with Session(REPORT, cols=80, rows=24) as s:
+        snap = s.until(lambda snap: reported(snap, s.pane(0)) is not None)
+        pane = s.pane(0)
+        got = reported(snap, pane)
+        check("the program was told a size at all", got is not None, snap.screen())
+        check(
+            "and it is the pane's size",
+            got == (pane["content_h"], pane["content_w"]),
+            f"program says {got}, pane is "
+            f"{(pane['content_h'], pane['content_w'])}\n{snap.screen()}",
+        )
+
+
+def test_a_pane_from_a_split_is_told_its_size():
+    """The split is where it hurts: the new pane's size exists only after the
+    layout, so it is delivered to a child that is still starting up."""
+    with Session(REPORT, cols=100, rows=30) as s:
+        s.settle()
+        s.key("\\\\")  # C-a \ -> split into columns
+        s.until(lambda snap: len(s.panes()) == 2)
+        snap = s.until(lambda snap: reported(snap, s.focused()) is not None)
+        fresh = s.focused()
+        got = reported(snap, fresh)
+        check(
+            "the fresh pane's program was told a size", got is not None, snap.screen()
+        )
+        check(
+            "and it is the fresh pane's size",
+            got == (fresh["content_h"], fresh["content_w"]),
+            f"program says {got}, pane is "
+            f"{(fresh['content_h'], fresh['content_w'])}\n{snap.screen()}",
+        )
+
+
+def test_a_resize_reaches_the_program():
+    """The same message, after the pane has been running a while: this is what
+    a window resize, a split beside it or a zoom all come down to."""
+    with Session(["/bin/sh", "-c", "read x; stty size; read y"], cols=80, rows=24) as s:
+        s.settle()
+        s.resize(120, 40)
+        s.settle()
+        s.raw("\\n")
+        snap = s.until(lambda snap: reported(snap, s.pane(0)) is not None)
+        pane = s.pane(0)
+        got = reported(snap, pane)
+        check("the program saw the new size", got is not None, snap.screen())
+        check(
+            "and it matches the pane",
+            got == (pane["content_h"], pane["content_w"]),
+            f"program says {got}, pane is "
+            f"{(pane['content_h'], pane['content_w'])}\n{snap.screen()}",
+        )
+
+
+for name, fn in sorted(list(globals().items())):
+    if name.startswith("test_"):
+        fn()
+sys.exit(report())