Keep the NIC loopback test off the NFS root and tolerate logind session races

docs/debian_nfsroot.md

@@ -329,12 +329,21 @@ boot, and the tests. Raise it for a divided-clock bitstream, or for a first
 build that must download the Buildroot toolchain or the kernel.
 
 The stage checks the NFS mount and systemd and runs the stress and counter
-tests. It loads the memory error driver, requires its scrub self-test and an
+tests. Systemd must report `running`, or `degraded` with only failed
+`session-N.scope` units whose `Result` is `resources`. These logind session
+races are logged as tolerated. Every other failed unit or result fails the
+stage. It loads the memory error driver, requires its scrub self-test and an
 accelerated RAM patrol pass with zero hardware errors, and reloads it at the
 daily default. The patrol check allows 150 one-second waits. It then runs the
-NIC loopback test and writes and syncs a file on the root to prove that the
-root came back after the test took its link down. It leaves `frost_stress`,
-`frost_nettest`, and `frost_memerr.ko` in `/usr/local/bin`. The SoC has no
+NIC loopback test from `/dev/shm`, with dentry and inode caches dropped before
+the test. Sysfs counters and carrier are opened before the link goes down.
+The test holds the interface down for 65 seconds before reading counters,
+exceeding NFS's default directory attribute cache lifetime.
+The shell opens MTU, flags and IPv6 restoration files before the test and
+writes through those descriptors, since absolute sysfs or proc lookups can
+revalidate the NFS root. It writes, syncs and removes a file on the root to
+prove that the root came back after the test took its link down. It leaves
+`frost_stress`, `frost_nettest`, and `frost_memerr.ko` in `/usr/local/bin`. The SoC has no
 software reset, so reload the image before the next boot.
 
 ## Operation
@@ -391,6 +400,6 @@ kernel the board runs.
 | Module missing under `/lib/modules/<version>` | Keep the running kernel installed; install its module, or load a kernel that is installed. An autoremove in the chroot sees the server's kernel. |
 | SSH public-key rejection | Check the key, root ownership, mode 700 on `.ssh`, and mode 600 on `authorized_keys`. |
 | Regression waits at login / `Login incorrect` | Add the console autologin drop-in from step 2. |
-| Regression reports degraded systemd | Inspect `systemctl --failed` and `systemctl status <unit>`. |
+| Regression reports degraded systemd | Inspect `systemctl --failed` and `systemctl show -p Result <unit>`. Only `session-N.scope` failures with `Result=resources` are tolerated. |
 | `/dev/shm/frost_nettest: Permission denied` | Remove `noexec` from the `/dev/shm` tmpfs. |
 | Regression `ENV_FAIL` | Follow the reported export, NFSv3, kernel, initramfs, write-permission, or compiler error. |

fpga/README.md

@@ -200,6 +200,16 @@ counters. With `patrol_seconds=10`, it requires patrol progress and a full
 pass within 150 one-second waits, then reloads the module at the daily default.
 `FROST_MEMERR_MODULE` names a prebuilt module instead.
 
+The Linux stage accepts systemd `running`, or `degraded` when every failed
+unit is a `session-N.scope` with `Result=resources`. It logs the tolerated
+scopes. Any other failed unit or result fails the stage.
+Before NIC loopback, the stage drops dentry and inode caches and copies the
+static test to `/dev/shm`. The test opens its sysfs descriptors while the
+link is up, then holds it down for 65 seconds to expire NFS's default
+directory attribute cache. The shell opens its restoration files before the
+test, restores MTU, flags and IPv6 through those descriptors, then writes,
+syncs and removes a root file to check NFS recovery.
+
 Stage selection follows `--board`; `bootrom` is available only on VCU118.
 The `bootrom` stage requires FSBL, OpenSBI, a device tree and U-Boot in flash.
 It requires the ROM's FSBL CRC verification, the FSBL's verified boot chain

fpga/hw_regression.py

@@ -265,20 +265,32 @@ def console_text(serial_buf: str) -> str:
     r"^\r*(nfs\d*)[ \t]+(\S+)[ \t]+(\S+)[ \t\r]*\n", re.MULTILINE
 )
 LINUX_ROOT_MOUNT_VERS = "vers=3"
-# Bound systemd startup, then print failed units and the current state.
+# Bound systemd startup, then print failed units, their results and the state.
 # ``state=`` distinguishes output from the echoed ``state=$(...)`` command,
 # even when the terminal wraps it.
 LINUX_SYSTEMD_WAIT_S = 300
 # ``--no-pager``, because a long enough list of failed units would otherwise
 # open a pager on this console, which reads the input the stage has typed ahead.
+# The sentinel permits a unit count only after the entire listing is consumed.
 LINUX_SYSTEMD_COMMAND = (
     f"timeout {LINUX_SYSTEMD_WAIT_S} systemctl is-system-running --wait "
     ">/dev/null; "
-    "systemctl --failed --no-legend --plain --no-pager; "
+    "echo systemd-check; "
+    "failed=$(systemctl --failed --no-legend --plain --no-pager); listed=$?; "
+    "printf '%s\\n' \"$failed\"; "
+    "printf '%s\\n' \"$failed\" systemd-list-end | { "
+    "count=0; while read -r unit rest; do "
+    '[ "$unit" = systemd-list-end ] && break; '
+    '[ -z "$unit" ] && continue; '
+    "count=$((count+1)); "
+    'result=$(systemctl show -p Result --value "$unit") || result=unknown; '
+    'echo failed_unit="$unit" result="$result"; done; '
+    '[ "$unit" = systemd-list-end ] && echo failed_count=$count; }; '
+    "echo failed_list_status=$listed; "
     "echo state=$(systemctl is-system-running)"
 )
 LINUX_SYSTEMD_RUNNING = "running"
-# Any complete state other than ``running`` ends capture as a failure.
+# Degraded is accepted only for session scopes whose result is resources.
 # Require a newline so a partial ``state=running`` cannot be read as ``state=r``.
 LINUX_SYSTEMD_STATE_RE = re.compile(r"^\r*state=(\w+)[ \t\r]*\n", re.MULTILINE)
 
@@ -1160,11 +1172,18 @@ def nettest_command(interface: str) -> str:
     Copy the static binary to tmpfs so instruction or library faults cannot
     block on the root while loopback takes the link down. Disable IPv6 during
     the test: autoconfiguration traffic returning through loopback fails idle
-    checks. The test leaves the interface down and loopback off.
-
-    Restore the saved MTU and flags using shell built-ins, then enable IPv6.
-    The login shell still maps the NFS root; reclaim of its pages can block
-    recovery. The bounded helper's exec can also block if the link stays down,
+    checks. Require a successful dentry and inode cache drop before the test.
+    The test opens its sysfs descriptors before taking the interface down.
+    Hold it down for 65 seconds to expire NFS's default directory attribute
+    cache: dropping caches alone allows pathname lookups before interface-down
+    to refresh it.
+    The test leaves the interface down with loopback off.
+
+    Open restoration files while the link is up. Restore the saved MTU and
+    flags using shell built-ins and those descriptors, then enable IPv6.
+    Absolute sysfs and proc lookups can revalidate the NFS root. The login
+    shell maps the NFS root; reclaim of its pages can block recovery.
+    The bounded helper's exec can also block if the link stays down,
     causing a stage TIMEOUT.
 
     Create a file on the root, sync, and remove it under a timeout. The printed
@@ -1175,8 +1194,11 @@ def nettest_command(interface: str) -> str:
     return (
         f"m=$(cat {sysfs}/mtu); f=$(cat {sysfs}/flags); echo 1 > {ipv6}; "
         f"cp {LINUX_ROOT_BIN}/{LINUX_NET_COMMAND} {LINUX_TMPFS}/; "
-        f"{LINUX_TMPFS}/{LINUX_NET_COMMAND}; "
-        f"echo $m > {sysfs}/mtu; echo $f > {sysfs}/flags; echo 0 > {ipv6}; "
+        "{ if echo 2 > /proc/sys/vm/drop_caches; then "
+        f"FROST_NETTEST_EXPIRE_NFS_CACHE=1 {LINUX_TMPFS}/{LINUX_NET_COMMAND}; "
+        "else echo FROST_NET_LOOPBACK'_'FAIL drop_caches; fi; "
+        "echo $m >&3; echo $f >&4; echo 0 >&5; "
+        f"}} 3> {sysfs}/mtu 4> {sysfs}/flags 5> {ipv6}; "
         f"timeout -k 5 {LINUX_ROOT_SYNC_S} sh -c "
         f'"echo frost > {LINUX_ROOT_ALIVE_FILE} && sync && '
         f'rm -f {LINUX_ROOT_ALIVE_FILE}"; '
@@ -1197,9 +1219,46 @@ def counter_values(serial_buf: str) -> dict[str, str]:
     return counts
 
 
+def tolerated_systemd_units(serial_buf: str) -> list[str]:
+    """Return session scopes when they alone explain the last degraded state.
+
+    A complete successful listing and Result=resources for every failed unit
+    are required. Missing results, an empty listing and other failures fail
+    the systemd check.
+    """
+    states = list(LINUX_SYSTEMD_STATE_RE.finditer(serial_buf))
+    if not states or states[-1].group(1) != "degraded":
+        return []
+    checks = list(re.finditer(r"^\r*systemd-check[\r]*\n", serial_buf, re.MULTILINE))
+    if not checks or checks[-1].end() > states[-1].start():
+        return []
+    check = serial_buf[checks[-1].end() : states[-1].start()]
+    statuses = re.findall(
+        r"^\r*failed_list_status=([^\r\n]*)[\r]*\n", check, re.MULTILINE
+    )
+    if statuses != ["0"]:
+        return []
+    units = re.findall(
+        r"^\r*failed_unit=(\S+) result=([^\r\n]*)[\r]*\n", check, re.MULTILINE
+    )
+    counts = re.findall(r"^\r*failed_count=([^\r\n]*)[\r]*\n", check, re.MULTILINE)
+    if (
+        counts != [str(len(units))]
+        or not units
+        or any(
+            not re.fullmatch(r"session-[0-9]+\.scope", unit) or result != "resources"
+            for unit, result in units
+        )
+    ):
+        return []
+    return [unit for unit, _ in units]
+
+
 def systemd_state(serial_buf: str) -> str | None:
-    """Return the last ``systemctl is-system-running`` state line, if any."""
+    """Return the last state, accepting only logind session resource failures."""
     states = LINUX_SYSTEMD_STATE_RE.findall(serial_buf)
+    if tolerated_systemd_units(serial_buf):
+        return LINUX_SYSTEMD_RUNNING
     return states[-1] if states else None
 
 
@@ -1239,8 +1298,9 @@ def linux_stage(root: LinuxRoot) -> UartStage:
     """Build UART checks for Debian booting from the NFS root.
 
     Require the pinned kernel and driver before login. After login, check the
-    NFSv3 mount, systemd startup, stress result, nonzero counters, the memory
-    error readback, scrub self-test, RAM patrol, and loopback with root access
+    NFSv3 mount, systemd startup (allowing only session scope resource failures),
+    stress result, nonzero counters, the memory error readback, scrub self-test,
+    RAM patrol, and loopback with root access
     restored. Explicit failures end capture immediately.
     """
     failure_markers = LINUX_FAILURE_MARKERS + (
@@ -1277,7 +1337,7 @@ def lx_reasons(serial_buf: str) -> list[str]:
         return hit
 
     def lx_success(raw_buf: str) -> bool:
-        """Login, a running systemd, both counters, frost_nettest, and the root back."""
+        """Login, an accepted systemd state, counters, loopback, and root access."""
         serial_buf = console_text(raw_buf)
         if not lx_login(serial_buf):
             return False
@@ -1349,9 +1409,13 @@ def lx_judge(raw_buf: str) -> tuple[bool, str]:
         summary = " ".join(f"{event}={counts[event]}" for event in LINUX_COUNTER_EVENTS)
         polls = memerr["polls_after"] - memerr["polls_before"]
         patrol = memerr["patrol_after"] - memerr["patrol_before"]
+        tolerated = tolerated_systemd_units(serial_buf)
+        systemd_note = LINUX_SYSTEMD_RUNNING
+        if tolerated:
+            systemd_note += f" (tolerated {', '.join(tolerated)}: resources)"
         return True, (
             f"{KERNEL_RELEASE}, NIC driver, {mount_note}, systemd "
-            f"{LINUX_SYSTEMD_RUNNING}, login, stress token, counters "
+            f"{systemd_note}, login, stress token, counters "
             f"{summary}, no memory errors ({polls} ECC polls), scrub self-test, "
             f"RAM patrol ({patrol} lines, a full pass), "
             f"{LINUX_NET_COMMAND}, root alive"

linux/README.md

@@ -318,6 +318,15 @@ board, the [hardware regression](../fpga/README.md#hardware-regression) boots
 Debian from its NFS root, and `fpga/linux_boot_soak.py` boots the test image
 repeatedly.
 
+The NFS-root stage accepts systemd `running`, or `degraded` when all failed
+units are `session-N.scope` units with `Result=resources`, and logs the
+tolerated scopes. It runs the static NIC loopback test from `/dev/shm` after
+dropping dentry and inode caches. The test opens counter and carrier files
+while the link is up, then holds the interface down for 65 seconds to expire
+NFS's default directory attribute cache. It reads through those descriptors
+during loopback. The shell restores the link through files opened before
+the test, then checks NFS writes and sync.
+
 CI boots the packaged kernel and initramfs on QEMU's `virt` machine with
 256 MiB using both bundled OpenSBI and FROST's firmware. A third boot uses
 FROST's firmware and 3 GiB of RAM starting at `0x8000_0000`. It requires

linux/buildroot-external/README.md

@@ -139,6 +139,17 @@ OpenSBI. RTL simulation does not boot Linux.
 | `frost_sigprobe` | vDSO signal-return probe |
 | `frost_nettest` | Driver loopback test; an optional interface argument selects the NIC. Prints `FROST_NET_LOOPBACK_PASS` or `FROST_NET_LOOPBACK_FAIL <reason>` and leaves the interface down with loopback disabled |
 
+`frost_nettest` opens the interface's sysfs directory, packet and error
+counters, and carrier before taking the link down. Reads seek those open
+files to offset zero, so they do not traverse an NFS root during loopback.
+When the root uses the tested interface, run a static copy from tmpfs and
+open any files needed to restore the interface before starting the test.
+The NFS-root stage drops dentry and inode caches before the test and sets
+`FROST_NETTEST_EXPIRE_NFS_CACHE=1`. This holds the interface down for 65 seconds
+before the first counter read, exceeding NFS's default directory attribute
+cache lifetime. The stage checks a root write, sync and removal after
+restoring MTU, flags and IPv6.
+
 `fpga/linux_boot_soak.py` fails a boot that passes the stress test but
 reports `counters=unavailable`; see
 [counter access](../README.md#counters-and-mcounteren).

linux/buildroot-external/package/frost-stress/src/frost_nettest.c

@@ -37,7 +37,10 @@
  *
  * Every wait the program makes is bounded (one step's frames get 10 s); a
  * system call that never returns is left to the caller's timeout. The test
- * uses no IP: all control goes through ioctls and sysfs.
+ * uses no IP: all control goes through ioctls and sysfs. Sysfs descriptors
+ * are opened before taking the link down, so reads do not traverse an NFS root.
+ * FROST_NETTEST_EXPIRE_NFS_CACHE=1 holds the interface down for 65 seconds
+ * before reading counters, exceeding NFS's default directory attribute lifetime.
  */
 
 #include <arpa/inet.h>
@@ -97,6 +100,18 @@ static uint32_t g_run;
 static uint32_t g_seq;  /* the next frame's sequence number; never reused */
 static int g_ctl = -1;  /* ioctl socket */
 static int g_sock = -1; /* the bound test socket */
+static int g_sysfs = -1;
+static int g_expire_nfs_cache;
+enum { RX_PACKETS, TX_PACKETS, RX_ERRORS, TX_ERRORS, TX_DROPPED, CARRIER };
+static struct {
+    const char *leaf;
+    int fd;
+} g_attrs[] = {{"statistics/rx_packets", -1},
+               {"statistics/tx_packets", -1},
+               {"statistics/rx_errors", -1},
+               {"statistics/tx_errors", -1},
+               {"statistics/tx_dropped", -1},
+               {"carrier", -1}};
 static uint32_t g_feature_words;
 static int g_loopback_bit = -1;
 static char g_reason[256];
@@ -343,17 +358,32 @@ static int set_loopback(int on)
 
 /* ---- sysfs: carrier and the driver's statistics ---- */
 
-static int read_sysfs_u64(const char *leaf, uint64_t *value)
+/* Keep the directory and attributes open until exit, including on failure. */
+static int open_sysfs(void)
+{
+    char path[96];
+    snprintf(path, sizeof(path), "/sys/class/net/%s", g_ifname);
+    g_sysfs = open(path, O_RDONLY | O_DIRECTORY);
+    if (g_sysfs < 0)
+        return fail("%s: %s", path, strerror(errno));
+    for (unsigned i = 0; i < sizeof(g_attrs) / sizeof(g_attrs[0]); i++) {
+        g_attrs[i].fd = openat(g_sysfs, g_attrs[i].leaf, O_RDONLY);
+        if (g_attrs[i].fd < 0)
+            return fail("%s/%s: %s", path, g_attrs[i].leaf, strerror(errno));
+    }
+    return 0;
+}
+
+static int read_sysfs_u64(unsigned attr, uint64_t *value)
 {
     char path[96];
     char text[32];
-    snprintf(path, sizeof(path), "/sys/class/net/%s/%s", g_ifname, leaf);
-    int fd = open(path, O_RDONLY);
-    if (fd < 0)
+    snprintf(path, sizeof(path), "/sys/class/net/%s/%s", g_ifname, g_attrs[attr].leaf);
+    int fd = g_attrs[attr].fd;
+    if (lseek(fd, 0, SEEK_SET) < 0)
         return fail("%s: %s", path, strerror(errno));
     ssize_t n = read(fd, text, sizeof(text) - 1);
     int err = errno;
-    close(fd);
     if (n <= 0)
         return fail("%s: %s", path, n < 0 ? strerror(err) : "empty");
     text[n] = '\0';
@@ -366,11 +396,11 @@ static int read_sysfs_u64(const char *leaf, uint64_t *value)
 
 static int read_stats(struct net_stats *s)
 {
-    if (read_sysfs_u64("statistics/rx_packets", &s->rx_packets) != 0 ||
-        read_sysfs_u64("statistics/tx_packets", &s->tx_packets) != 0 ||
-        read_sysfs_u64("statistics/rx_errors", &s->rx_errors) != 0 ||
-        read_sysfs_u64("statistics/tx_errors", &s->tx_errors) != 0 ||
-        read_sysfs_u64("statistics/tx_dropped", &s->tx_dropped) != 0)
+    if (read_sysfs_u64(RX_PACKETS, &s->rx_packets) != 0 ||
+        read_sysfs_u64(TX_PACKETS, &s->tx_packets) != 0 ||
+        read_sysfs_u64(RX_ERRORS, &s->rx_errors) != 0 ||
+        read_sysfs_u64(TX_ERRORS, &s->tx_errors) != 0 ||
+        read_sysfs_u64(TX_DROPPED, &s->tx_dropped) != 0)
         return -1;
     return 0;
 }
@@ -385,7 +415,7 @@ static int wait_carrier(uint64_t *elapsed_ms)
     int ones = 0;
     for (;;) {
         uint64_t carrier;
-        if (read_sysfs_u64("carrier", &carrier) != 0)
+        if (read_sysfs_u64(CARRIER, &carrier) != 0)
             return -1;
         *elapsed_ms = now_ms() - start;
         ones = carrier == 1 ? ones + 1 : 0;
@@ -407,7 +437,7 @@ static int loopback_off_interval(uint64_t *carrier_ms)
     *carrier_ms = UINT64_MAX;
     for (;;) {
         uint64_t carrier;
-        if (read_sysfs_u64("carrier", &carrier) != 0)
+        if (read_sysfs_u64(CARRIER, &carrier) != 0)
             return -1;
         uint64_t elapsed = now_ms() - start;
         if (carrier != 0 && *carrier_ms == UINT64_MAX)
@@ -706,8 +736,11 @@ static int run(void)
     g_ctl = socket(AF_PACKET, SOCK_DGRAM, 0);
     if (g_ctl < 0)
         return fail("control socket: %s", strerror(errno));
-    if (find_interface() != 0 || find_loopback_bit() != 0 || set_up(0) != 0 ||
-        read_stats(&start) != 0)
+    if (find_interface() != 0 || find_loopback_bit() != 0 || open_sysfs() != 0 || set_up(0) != 0)
+        return -1;
+    if (g_expire_nfs_cache)
+        sleep_ms(65000);
+    if (read_stats(&start) != 0)
         return -1;
     progress("%s: driver %s, address %02x:%02x:%02x:%02x:%02x:%02x, loopback feature bit %d, "
              "run %08x; down",
@@ -828,6 +861,8 @@ static int restore(void)
 int main(int argc, char **argv)
 {
     struct timespec ts;
+    const char *expire_cache = getenv("FROST_NETTEST_EXPIRE_NFS_CACHE");
+    g_expire_nfs_cache = expire_cache && strcmp(expire_cache, "1") == 0;
     setvbuf(stdout, NULL, _IOLBF, 0);
     if (argc > 1 && argv[1][0] != '\0')
         g_requested_ifname = argv[1];