[PATCH] ntdll: Use the true maximum open FDs to set the limit on Mac.

[Chip Davis]

The actual upper limit is stored in the sysctl(3)
"kern.maxfilesperproc". This value may be higher than the value of the
OPEN_MAX constant.

Using sysconf(3) with _SC_OPEN_MAX just returns the current value of
RLIMIT_NOFILE, so that won't work here.

Signed-off-by: Chip Davis <cdavis5x(a)gmail.com>
---
Fixing this in the place it *actually* needs to be fixed this time...
---
 dlls/ntdll/unix/loader.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/dlls/ntdll/unix/loader.c b/dlls/ntdll/unix/loader.c
index d5867e4aa754..dc54a54a225f 100644
--- a/dlls/ntdll/unix/loader.c
+++ b/dlls/ntdll/unix/loader.c
@@ -452,13 +452,29 @@ static void set_max_limit( int limit )
         rlimit.rlim_cur = rlimit.rlim_max;
         if (setrlimit( limit, &rlimit ) != 0)
         {
-#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && defined(OPEN_MAX)
-            /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
-             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
-            if (limit == RLIMIT_NOFILE && rlimit.rlim_cur > OPEN_MAX)
+#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && (defined(OPEN_MAX) || defined(KERN_MAXFILESPERPROC))
+            if (limit == RLIMIT_NOFILE)
             {
-                rlimit.rlim_cur = OPEN_MAX;
-                setrlimit( limit, &rlimit );
+                /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
+                 * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
+                int open_max = INT_MAX;
+#ifdef KERN_MAXFILESPERPROC
+                /* This is the true OPEN_MAX; it may be greater than the constant.
+                 * sysconf(_SC_OPEN_MAX) won't work here: it just returns getrlimit(RLIMIT_NOFILE). */
+                static int maxfilesperproc_oid[] = { CTL_KERN, KERN_MAXFILESPERPROC };
+                size_t len = sizeof(open_max);
+                if (sysctl(maxfilesperproc_oid, ARRAY_SIZE(maxfilesperproc_oid), &open_max, &len, NULL, 0) != 0)
+                    open_max = INT_MAX;
+#endif
+#ifdef OPEN_MAX
+                if (open_max == INT_MAX)
+                    open_max = OPEN_MAX;
+#endif
+                if (rlimit.rlim_cur > open_max)
+                {
+                    rlimit.rlim_cur = open_max;
+                    setrlimit( limit, &rlimit );
+                }
             }
 #endif
         }
-- 
2.34.1

[Brendan Shanks]

Hi Chip,
> On Apr 14, 2022, at 11:29 AM, Chip Davis <cdavis5x(a)gmail.com> wrote:
>
> The actual upper limit is stored in the sysctl(3)
> "kern.maxfilesperproc". This value may be higher than the value of the
> OPEN_MAX constant.
>
> Using sysconf(3) with _SC_OPEN_MAX just returns the current value of
> RLIMIT_NOFILE, so that won't work here.
>
> Signed-off-by: Chip Davis <cdavis5x(a)gmail.com>
> ---
> Fixing this in the place it *actually* needs to be fixed this time...
> ---
> dlls/ntdll/unix/loader.c | 28 ++++++++++++++++++++++------
> 1 file changed, 22 insertions(+), 6 deletions(-)
>
> diff --git a/dlls/ntdll/unix/loader.c b/dlls/ntdll/unix/loader.c
> index d5867e4aa754..dc54a54a225f 100644
> --- a/dlls/ntdll/unix/loader.c
> +++ b/dlls/ntdll/unix/loader.c
> @@ -452,13 +452,29 @@ static void set_max_limit( int limit )
>         rlimit.rlim_cur = rlimit.rlim_max;
>         if (setrlimit( limit, &rlimit ) != 0)
>         {
> -#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && defined(OPEN_MAX)
> -            /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
> -             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
> -            if (limit == RLIMIT_NOFILE && rlimit.rlim_cur > OPEN_MAX)
> +#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && (defined(OPEN_MAX) || defined(KERN_MAXFILESPERPROC))

Are the OPEN_MAX/KERN_MAXFILESPERPROC ifdefs (and fallback to INT_MAX) necessary? KERN_MAXFILESPERPROC seems to have been present since 10.0 (https://github.com/apple-oss-distributions/xnu/blob/c1dac77fcd1f9be7193cad54...), and OPEN_MAX at least since 10.5 (https://github.com/apple-oss-distributions/xnu/blame/cc9a63552421e7db025ad77...). 

Also, I tested this on macOS 12.4 on M1, and the initial setrlimit(RLIMIT_NOFILE, …) call actually succeeds, even though rlim_max is LONG_MAX, way larger than kern.maxfilesperproc. setrlimit() does fail on 10.13 though.
> +            if (limit == RLIMIT_NOFILE)
>             {
> -                rlimit.rlim_cur = OPEN_MAX;
> -                setrlimit( limit, &rlimit );
> +                /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
> +                 * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
> +                int open_max = INT_MAX;
> +#ifdef KERN_MAXFILESPERPROC
> +                /* This is the true OPEN_MAX; it may be greater than the constant.
> +                 * sysconf(_SC_OPEN_MAX) won't work here: it just returns getrlimit(RLIMIT_NOFILE). */
> +                static int maxfilesperproc_oid[] = { CTL_KERN, KERN_MAXFILESPERPROC };
> +                size_t len = sizeof(open_max);
> +                if (sysctl(maxfilesperproc_oid, ARRAY_SIZE(maxfilesperproc_oid), &open_max, &len, NULL, 0) != 0)

Style nitpick: spaces around arguments.
> +                    open_max = INT_MAX;
> +#endif
> +#ifdef OPEN_MAX
> +                if (open_max == INT_MAX)
> +                    open_max = OPEN_MAX;
> +#endif
> +                if (rlimit.rlim_cur > open_max)
> +                {
> +                    rlimit.rlim_cur = open_max;
> +                    setrlimit( limit, &rlimit );
> +                }
>             }
> #endif
>         }
> -- 
> 2.34.1

Brendan

[Charles Davis]

On Tue, Apr 19, 2022 at 11:42 AM Brendan Shanks <bshanks(a)codeweavers.com> wrote:
> Hi Chip,
> > On Apr 14, 2022, at 11:29 AM, Chip Davis <cdavis5x(a)gmail.com> wrote:
> >
> > The actual upper limit is stored in the sysctl(3)
> > "kern.maxfilesperproc". This value may be higher than the value of the
> > OPEN_MAX constant.
> >
> > Using sysconf(3) with _SC_OPEN_MAX just returns the current value of
> > RLIMIT_NOFILE, so that won't work here.
> >
> > Signed-off-by: Chip Davis <cdavis5x(a)gmail.com>
> > ---
> > Fixing this in the place it *actually* needs to be fixed this time...
> > ---
> > dlls/ntdll/unix/loader.c | 28 ++++++++++++++++++++++------
> > 1 file changed, 22 insertions(+), 6 deletions(-)
> >
> > diff --git a/dlls/ntdll/unix/loader.c b/dlls/ntdll/unix/loader.c
> > index d5867e4aa754..dc54a54a225f 100644
> > --- a/dlls/ntdll/unix/loader.c
> > +++ b/dlls/ntdll/unix/loader.c
> > @@ -452,13 +452,29 @@ static void set_max_limit( int limit )
> >         rlimit.rlim_cur = rlimit.rlim_max;
> >         if (setrlimit( limit, &rlimit ) != 0)
> >         {
> > -#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && defined(OPEN_MAX)
> > -            /* On Leopard, setrlimit(RLIMIT_NOFILE, ...) fails on attempts to set
> > -             * rlim_cur above OPEN_MAX (even if rlim_max > OPEN_MAX). */
> > -            if (limit == RLIMIT_NOFILE && rlimit.rlim_cur > OPEN_MAX)
> > +#if defined(__APPLE__) && defined(RLIMIT_NOFILE) && (defined(OPEN_MAX) || defined(KERN_MAXFILESPERPROC))
>
> Are the OPEN_MAX/KERN_MAXFILESPERPROC ifdefs (and fallback to INT_MAX) necessary? KERN_MAXFILESPERPROC seems to have been present since 10.0 (https://github.com/apple-oss-distributions/xnu/blob/c1dac77fcd1f9be7193cad54...), and OPEN_MAX at least since 10.5 (https://github.com/apple-oss-distributions/xnu/blame/cc9a63552421e7db025ad77...).

Huh. Then I guess the #ifdefs aren't needed after all.
> Also, I tested this on macOS 12.4 on M1, and the initial setrlimit(RLIMIT_NOFILE, …) call actually succeeds, even though rlim_max is LONG_MAX, way larger than kern.maxfilesperproc. setrlimit() does fail on 10.13 though.

It fails on 10.14, too.