Monitor open FDs (#8217)

Add an FD monitor for servers. This is potentially something we can do
more efficiently in Rust at a later date, but a Python implementation
will allow for an easier backport to 5.6.

We use a new shortcut available in modern Linux kernels to avoid
counting directory entries (os.stat's size == # of open FDs).

Threads are used to simplify the implementation.

edb/server/metrics.py

@@ -16,6 +16,8 @@
 # limitations under the License.
 #
 
+import os
+import sys
 
 from edb.common import prometheus as prom
 
@@ -262,3 +264,77 @@
     'Total number of tenants the server failed to reload.',
     labels=("tenant",),
 )
+
+if os.name == 'posix' and (sys.platform == 'linux' or sys.platform == 'darwin'):
+    open_fds = registry.new_gauge(
+        'open_fds',
+        'Number of open file descriptors.',
+    )
+
+    max_open_fds = registry.new_gauge(
+        'max_open_fds',
+        'Maximum number of open file descriptors.',
+    )
+
+# Implement a function that monitors the number of open file descriptors
+# and updates the metrics accordingly. This will be replaced with a more
+# efficient implementation in Rust at a later date.
+
+
+def monitor_open_fds_linux():
+    import time
+    while True:
+        max_open_fds.set(os.sysconf('SC_OPEN_MAX'))
+        # To get the current number of open files, stat /proc/self/fd/
+        # and get the size. If zero, count the number of entries in the
+        # directory.
+        #
+        # This is supported in modern Linux kernels.
+        # https://github.com/torvalds/linux/commit/f1f1f2569901ec5b9d425f2e91c09a0e320768f3
+        try:
+            st = os.stat('/proc/self/fd/')
+            if st.st_size == 0:
+                open_fds.set(len(os.listdir('/proc/self/fd/')))
+            else:
+                open_fds.set(st.st_size)
+        except Exception:
+            open_fds.set(-1)
+
+        time.sleep(30)
+
+
+def monitor_open_fds_macos():
+    import time
+    while True:
+        max_open_fds.set(os.sysconf('SC_OPEN_MAX'))
+        # Iterate the contents of /dev/fd to list all entries.
+        # We assume that MacOS isn't going to be running a large installation
+        # of EdgeDB on a single machine.
+        try:
+            open_fds.set(len(os.listdir('/dev/fd')))
+        except Exception:
+            open_fds.set(-1)
+
+        time.sleep(30)
+
+
+def start_monitoring_open_fds():
+    import threading
+
+    # Supported only on Linux and macOS.
+    if os.name == 'posix':
+        if sys.platform == 'darwin':
+            threading.Thread(
+                target=monitor_open_fds_macos,
+                name='open_fds_monitor',
+                daemon=True
+            ).start()
+        elif sys.platform == 'linux':
+            threading.Thread(
+                target=monitor_open_fds_linux,
+                name='open_fds_monitor',
+                daemon=True
+            ).start()
+
+
+start_monitoring_open_fds()