Skip to content

Commit

Permalink
Add emerge jobs tmpdir blocks and files threshold options
Browse files Browse the repository at this point in the history
--jobs-tmpdir-blocks-threshold[=RATIO]

  Specifies the maximum ratio of used blocks allowed (a
  floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR
  when starting a new job. With no argument, removes a previous
  blocks ratio threshold. For example, use a ratio of 0.85 to stop
  starting new jobs when the blocks usage in PORTAGE_TMPDIR
  exceeds 85%. This option conflicts with FEATURES="keepwork".

--jobs-tmpdir-files-threshold[=RATIO]

  Specifies the maximum ratio of used files (inodes) allowed (a
  floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR
  when starting a new job. With no argument, removes a previous
  files ratio threshold. For example, use a ratio of 0.85 to stop
  starting new jobs when the files usage in PORTAGE_TMPDIR
  exceeds 85%. This option conflicts with FEATURES="keepwork".

Bug: https://bugs.gentoo.org/934382
Signed-off-by: Zac Medico <[email protected]>
  • Loading branch information
zmedico committed Jun 20, 2024
1 parent 819c863 commit 0f2a0b2
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 4 deletions.
54 changes: 52 additions & 2 deletions lib/_emerge/Scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from portage._sets.base import InternalPackageSet
from portage.util import ensure_dirs, writemsg, writemsg_level
from portage.util.futures import asyncio
from portage.util.path import first_existing
from portage.util.SlotObject import SlotObject
from portage.util._async.SchedulerInterface import SchedulerInterface
from portage.package.ebuild.digestcheck import digestcheck
Expand Down Expand Up @@ -64,7 +65,7 @@


class Scheduler(PollScheduler):
# max time between loadavg checks (seconds)
# max time between loadavg and tmpdir statvfs checks (seconds)
_loadavg_latency = 30

# max time between display status updates (seconds)
Expand Down Expand Up @@ -229,6 +230,10 @@ def __init__(
if max_jobs is None:
max_jobs = 1
self._set_max_jobs(max_jobs)
self._jobs_tmpdir_blocks_threshold = myopts.get(
"--jobs-tmpdir-blocks-threshold"
)
self._jobs_tmpdir_files_threshold = myopts.get("--jobs-tmpdir-files-threshold")
self._running_root = trees[trees._running_eroot]["root_config"]
self.edebug = 0
if settings.get("PORTAGE_DEBUG", "") == "1":
Expand Down Expand Up @@ -1573,7 +1578,11 @@ def _main_loop(self):
self._main_exit = self._event_loop.create_future()

if (
self._max_load is not None
(
self._max_load is not None
or self._jobs_tmpdir_blocks_threshold is not None
or self._jobs_tmpdir_files_threshold is not None
)
and self._loadavg_latency is not None
and (self._max_jobs is True or self._max_jobs > 1)
):
Expand Down Expand Up @@ -1792,6 +1801,47 @@ def _is_work_scheduled(self):
def _running_job_count(self):
return self._jobs

def _can_add_job(self):
if not super()._can_add_job():
return False

if (
self._jobs_tmpdir_blocks_threshold is not None
or self._jobs_tmpdir_files_threshold is not None
) and hasattr(os, "statvfs"):
tmpdirs = set()
for root in self.trees:
settings = self.trees[root]["root_config"].settings
if settings["PORTAGE_TMPDIR"] in tmpdirs:
continue
tmpdirs.add(settings["PORTAGE_TMPDIR"])
tmpdir = first_existing(
os.path.join(settings["PORTAGE_TMPDIR"], "portage")
)
try:
vfs_stat = os.statvfs(tmpdir)
except OSError as e:
writemsg_level(
f"!!! statvfs('{tmpdir}'): {e}\n",
noiselevel=-1,
level=logging.ERROR,
)
else:
if (
self._jobs_tmpdir_blocks_threshold is not None
and (vfs_stat.f_blocks - vfs_stat.f_bavail) / vfs_stat.f_blocks
) >= self._jobs_tmpdir_blocks_threshold:
return False
if (
self._jobs_tmpdir_files_threshold is not None
and vfs_stat.f_files > 0
and ((vfs_stat.f_files - vfs_stat.f_favail) / vfs_stat.f_files)
>= self._jobs_tmpdir_files_threshold
):
return False

return True

def _schedule_tasks(self):
while True:
state_change = 0
Expand Down
69 changes: 68 additions & 1 deletion lib/_emerge/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 1999-2023 Gentoo Authors
# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import argparse
Expand Down Expand Up @@ -165,6 +165,8 @@ def __contains__(self, s):
"--getbinpkgonly": y_or_n,
"--ignore-world": y_or_n,
"--jobs": valid_integers,
"--jobs-tmpdir-blocks-threshold": valid_floats,
"--jobs-tmpdir-files-threshold": valid_floats,
"--keep-going": y_or_n,
"--load-average": valid_floats,
"--onlydeps-with-ideps": y_or_n,
Expand Down Expand Up @@ -523,6 +525,14 @@ def parse_opts(tmpcmdline, silent=False):
"help": "Specifies the number of packages to build " + "simultaneously.",
"action": "store",
},
"--jobs-tmpdir-blocks-threshold": {
"help": "Specifies maximum used blocks ratio when starting a new job.",
"action": "store",
},
"--jobs-tmpdir-files-threshold": {
"help": "Specifies maximum used files ratio when starting a new job.",
"action": "store",
},
"--keep-going": {
"help": "continue as much as possible after an error",
"choices": true_y_or_n,
Expand Down Expand Up @@ -1033,6 +1043,42 @@ def parse_opts(tmpcmdline, silent=False):

myoptions.jobs = jobs

if myoptions.jobs_tmpdir_blocks_threshold == "True":
myoptions.jobs_tmpdir_blocks_threshold = None

if myoptions.jobs_tmpdir_blocks_threshold:
try:
jobs_tmpdir_blocks_threshold = float(myoptions.jobs_tmpdir_blocks_threshold)
except ValueError:
jobs_tmpdir_blocks_threshold = 0.0

if jobs_tmpdir_blocks_threshold <= 0.0 or jobs_tmpdir_blocks_threshold > 1.0:
jobs_tmpdir_blocks_threshold = None
if not silent:
parser.error(
f"Invalid --jobs-tmpdir-blocks-threshold: '{myoptions.jobs_tmpdir_blocks_threshold}'\n"
)

myoptions.jobs_tmpdir_blocks_threshold = jobs_tmpdir_blocks_threshold

if myoptions.jobs_tmpdir_files_threshold == "True":
myoptions.jobs_tmpdir_files_threshold = None

if myoptions.jobs_tmpdir_files_threshold:
try:
jobs_tmpdir_files_threshold = float(myoptions.jobs_tmpdir_files_threshold)
except ValueError:
jobs_tmpdir_files_threshold = 0.0

if jobs_tmpdir_files_threshold <= 0.0 or jobs_tmpdir_files_threshold > 1.0:
jobs_tmpdir_files_threshold = None
if not silent:
parser.error(
f"Invalid --jobs-tmpdir-files-threshold: '{myoptions.jobs_tmpdir_files_threshold}'\n"
)

myoptions.jobs_tmpdir_files_threshold = jobs_tmpdir_files_threshold

if myoptions.load_average == "True":
myoptions.load_average = None

Expand Down Expand Up @@ -1304,6 +1350,27 @@ def emerge_main(args: Optional[list[str]] = None):
emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts(
tmpcmdline
)
if (
"--jobs-tmpdir-blocks-threshold" in emerge_config.opts
and "keepwork" in emerge_config.running_config.settings.features
):
writemsg_level(
"--jobs-tmpdir-blocks-threshold conflicts with FEATURES=keepwork\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1

if (
"--jobs-tmpdir-files-threshold" in emerge_config.opts
and "keepwork" in emerge_config.running_config.settings.features
):
writemsg_level(
"--jobs-tmpdir-files-threshold conflicts with FEATURES=keepwork\n",
level=logging.ERROR,
noiselevel=-1,
)
return 1

try:
return run_action(emerge_config)
Expand Down
18 changes: 17 additions & 1 deletion man/emerge.1
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage"
.TH "EMERGE" "1" "Jun 2024" "Portage @VERSION@" "Portage"
.SH "NAME"
emerge \- Command\-line interface to the Portage system
.SH "SYNOPSIS"
Expand Down Expand Up @@ -693,6 +693,22 @@ Note that interactive packages currently force a setting
of \fI\-\-jobs=1\fR. This issue can be temporarily avoided
by specifying \fI\-\-accept\-properties=\-interactive\fR.
.TP
.BR \-\-jobs\-tmpdir\-blocks\-threshold[=RATIO]
Specifies the maximum ratio of used blocks allowed (a floating\-point
number between \fI0.0\fR and \fI1.0\fR) in \fBPORTAGE_TMPDIR\fR when
starting a new job. With noargument, removes a previous blocks ratio
threshold. For example, use a ratio of \fI0.85\fR to stop starting new
jobs when the blocks usage in \fBPORTAGE_TMPDIR\fR exceeds \fI85%\fR.
This option conflicts with \fBFEATURES="keepwork"\fR.
.TP
.BR \-\-jobs\-tmpdir\-files\-threshold[=RATIO]
Specifies the maximum ratio of used files (inodes) allowed (a
floating\-point number between \fI0.0\fR and \fI1.0\fR) in
\fBPORTAGE_TMPDIR\fR when starting a new job. With no argument, removes
a previous files ratio threshold. For example, use a ratio of \fI0.85\fR
to stop starting new jobs when the files usage in \fBPORTAGE_TMPDIR\fR
exceeds \fI85%\fR. This option conflicts with \fBFEATURES="keepwork"\fR.
.TP
.BR "\-\-keep\-going [ y | n ]"
Continue as much as possible after an error. When an error occurs,
dependencies are recalculated for remaining packages and any with
Expand Down

0 comments on commit 0f2a0b2

Please sign in to comment.