linux.rs: Don't use /proc/sys/fs/file-max

Don't use /proc/sys/fs/file-max when setting the max open file limit for
the virtio-fs device.  This will fail when the value is larger than the
hard limit set for the crosvm process, unless it also has CAP_SYS_ADMIN
in the initial namespace.

Instead, just use the hard limit as returned by `prlimit64`.  Increasing
the soft limit up to the hard limit is allowed even for completely
unprivileged processes.  It is up to the process that spawned crosvm to
ensure that the hard limit is high enough that the virtio-fs server will
not run out of fds.

BUG=b:142344095
TEST=Start a termina VM with a virtio-fs device after applying
     CL:1939193

Change-Id: I4fb4c33ffe6378ed3109fddcb0fc2bf3da850252
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1957767
Tested-by: Chirantan Ekbote <chirantan@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>
Auto-Submit: Chirantan Ekbote <chirantan@chromium.org>
Commit-Queue: Stephen Barber <smbarber@chromium.org>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Reviewed-by: Stephen Barber <smbarber@chromium.org>

1 files changed, 12 insertions, 6 deletions

diff --git a/src/linux.rs b/src/linux.rs
index 1c606b9..0cc341d 100644
--- a/src/linux.rs
+++ b/src/linux.rs
@@ -18,6 +18,7 @@ use std::num::ParseIntError;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::os::unix::net::UnixStream;
 use std::path::{Path, PathBuf};
+use std::ptr;
 use std::str;
 use std::sync::{Arc, Barrier};
 use std::thread;
@@ -290,12 +291,17 @@ impl AsRawFd for TaggedControlSocket {
 }
 
 fn get_max_open_files() -> Result<libc::rlim64_t> {
-    let mut buf = String::with_capacity(32);
-    File::open("/proc/sys/fs/file-max")
-        .and_then(|mut f| f.read_to_string(&mut buf))
-        .map_err(Error::GetMaxOpenFiles)?;
-
-    Ok(buf.trim().parse().map_err(Error::ParseMaxOpenFiles)?)
+    let mut buf = mem::MaybeUninit::<libc::rlimit64>::zeroed();
+
+    // Safe because this will only modify `buf` and we check the return value.
+    let res = unsafe { libc::prlimit64(0, libc::RLIMIT_NOFILE, ptr::null(), buf.as_mut_ptr()) };
+    if res == 0 {
+        // Safe because the kernel guarantees that the struct is fully initialized.
+        let limit = unsafe { buf.assume_init() };
+        Ok(limit.rlim_max)
+    } else {
+        Err(Error::GetMaxOpenFiles(io::Error::last_os_error()))
+    }
 }
 
 fn create_base_minijail(