diff --git a/frappe/__init__.py b/frappe/__init__.py index 23c755adbf59..e28b7e31b867 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -12,15 +12,11 @@ """ import copy -import faulthandler import functools -import gc import importlib import inspect import json import os -import re -import signal import sys import traceback import warnings @@ -89,7 +85,6 @@ _one_time_setup: dict[str, bool] = {} _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) -_tune_gc = bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))) if _dev_server: warnings.simplefilter("always", DeprecationWarning) @@ -285,7 +280,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force: bool = if not _one_time_setup.get(local.conf.db_type): patch_query_execute() patch_query_aggregation() - _register_fault_handler() _one_time_setup[local.conf.db_type] = True setup_module_map(include_all_apps=not (frappe.request or frappe.job or frappe.flags.in_migrate)) @@ -2436,14 +2430,14 @@ def get_website_settings(key): def get_system_settings(key: str): """Return the value associated with the given `key` from System Settings DocType.""" - if not hasattr(local, "system_settings"): + if not (system_settings := getattr(local, "system_settings", None)): try: - local.system_settings = get_cached_doc("System Settings") + local.system_settings = system_settings = get_cached_doc("System Settings") except DoesNotExistError: # possible during new install clear_last_message() return - return local.system_settings.get(key) + return system_settings.get(key) def get_active_domains(): @@ -2577,24 +2571,7 @@ def wrapper(*args, **kwargs): return wrapper -def _register_fault_handler(): - import io +import frappe.optimizations +from frappe.utils.error import log_error # Backward compatibility - # Some libraries monkey patch stderr, we need actual fd - if isinstance(sys.__stderr__, io.TextIOWrapper): - faulthandler.register(signal.SIGUSR1, file=sys.__stderr__) - - -from frappe.utils.error import log_error - -if _tune_gc: - # generational GC gets triggered after certain allocs (g0) which is 700 by default. - # This number is quite small for frappe where a single query can potentially create 700+ - # objects easily. - # Bump this number higher, this will make GC less aggressive but that improves performance of - # everything else. - g0, g1, g2 = gc.get_threshold() # defaults are 700, 10, 10. - gc.set_threshold(g0 * 10, g1 * 2, g2 * 2) - -# Remove references to pattern that are pre-compiled and loaded to global scopes. -re.purge() +frappe.optimizations.optimize_all() diff --git a/frappe/app.py b/frappe/app.py index 0519dc92e4a7..894745895dbc 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -35,35 +35,34 @@ # If gc.freeze is done then importing modules before forking allows us to share the memory -if frappe._tune_gc: - import gettext - - import babel - import babel.messages - import bleach - import num2words - import pydantic - - import frappe.boot - import frappe.client - import frappe.core.doctype.file.file - import frappe.core.doctype.user.user - import frappe.database.mariadb.database # Load database related utils - import frappe.database.query - import frappe.desk.desktop # workspace - import frappe.desk.form.save - import frappe.model.db_query - import frappe.query_builder - import frappe.utils.background_jobs # Enqueue is very common - import frappe.utils.data # common utils - import frappe.utils.jinja # web page rendering - import frappe.utils.jinja_globals - import frappe.utils.redis_wrapper # Exact redis_wrapper - import frappe.utils.safe_exec - import frappe.utils.typing_validations # any whitelisted method uses this - import frappe.website.path_resolver # all the page types and resolver - import frappe.website.router # Website router - import frappe.website.website_generator # web page doctypes +import gettext + +import babel +import babel.messages +import bleach +import num2words +import pydantic + +import frappe.boot +import frappe.client +import frappe.core.doctype.file.file +import frappe.core.doctype.user.user +import frappe.database.mariadb.database # Load database related utils +import frappe.database.query +import frappe.desk.desktop # workspace +import frappe.desk.form.save +import frappe.model.db_query +import frappe.query_builder +import frappe.utils.background_jobs # Enqueue is very common +import frappe.utils.data # common utils +import frappe.utils.jinja # web page rendering +import frappe.utils.jinja_globals +import frappe.utils.redis_wrapper # Exact redis_wrapper +import frappe.utils.safe_exec +import frappe.utils.typing_validations # any whitelisted method uses this +import frappe.website.path_resolver # all the page types and resolver +import frappe.website.router # Website router +import frappe.website.website_generator # web page doctypes # end: module pre-loading @@ -519,20 +518,3 @@ def application_with_statics(): application = StaticDataMiddleware(application, {"/files": str(os.path.abspath(_sites_path))}) return application - - -# Remove references to pattern that are pre-compiled and loaded to global scopes. -re.purge() - -# Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing -# most of the memory if there are no writes made to data because of Copy on Write, however, -# python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the -# generational GC which stores and mutates every python object: `PyGC_Head` -# -# Calling gc.freeze() moves all the objects imported so far into permanant generation and hence -# doesn't mutate `PyGC_Head` -# -# Refer to issue for more info: https://github.com/frappe/frappe/issues/18927 -if frappe._tune_gc: - gc.collect() # clean up any garbage created so far before freeze - gc.freeze() diff --git a/frappe/build.py b/frappe/build.py index 750ff05ab6bc..d4fe3189c14b 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -196,7 +196,7 @@ def symlink(target, link_name, overwrite=False): if os.path.isdir(link_name): raise IsADirectoryError(f"Cannot symlink over existing directory: '{link_name}'") try: - os.replace(temp_link_name, link_name) + shutil.move(temp_link_name, link_name) except AttributeError: os.renames(temp_link_name, link_name) except Exception: diff --git a/frappe/model/document.py b/frappe/model/document.py index 66072cebbdec..0b153d0b07a2 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1541,8 +1541,17 @@ def round_floats_in(self, doc, fieldnames=None): for df in doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}) ) + # PERF: flt internally has to resolve this if we don't specify it. + rounding_method = frappe.get_system_settings("rounding_method") for fieldname in fieldnames: - doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) + doc.set( + fieldname, + flt( + doc.get(fieldname), + self.precision(fieldname, doc.get("parentfield")), + rounding_method=rounding_method, + ), + ) def get_url(self): """Return Desk URL for this document.""" diff --git a/frappe/optimizations.py b/frappe/optimizations.py new file mode 100644 index 000000000000..85f4381ceae4 --- /dev/null +++ b/frappe/optimizations.py @@ -0,0 +1,81 @@ +import faulthandler +import gc +import io +import os +import re +import signal +import sys + + +def optimize_all(): + """Single entry point to enable all optimizations at right time automatically.""" + + # Note: + # - This function is ALWAYS executed as soon as `import frappe` ends. + # - Any deferred work should be deferred using os module's fork hooks. + # - Respect configurations using environement variables. + # - fork hooks can not be unregistered, so care should be taken to execute them only when they + # make sense. + _optimize_regex_cache() + _optimize_gc_parameters() + _optimize_gc_for_copy_on_write() + _register_fault_handler() + os.register_at_fork(after_in_child=_register_fault_handler) + + +def _optimize_gc_parameters(): + from frappe.utils import sbool + + if not bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))): + return + + # generational GC gets triggered after certain allocs (g0) which is 700 by default. + # This number is quite small for frappe where a single query can potentially create 700+ + # objects easily. + # Bump this number higher, this will make GC less aggressive but that improves performance of + # everything else. + g0, g1, g2 = gc.get_threshold() # defaults are 700, 10, 10. + gc.set_threshold(g0 * 10, g1 * 2, g2 * 2) + + +def _optimize_regex_cache(): + # Remove references to pattern that are pre-compiled and loaded to global scopes. + # Leave that cache for dynamically generated regex. + os.register_at_fork(before=re.purge) + + +def _register_fault_handler(): + # Some libraries monkey patch stderr, we need actual fd + if isinstance(sys.__stderr__, io.TextIOWrapper): + faulthandler.register(signal.SIGUSR1, file=sys.__stderr__) + + +def _optimize_gc_for_copy_on_write(): + from frappe.utils import sbool + + if not bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))): + return + + os.register_at_fork(before=_freeze_gc) + + +_gc_frozen = False + + +def _freeze_gc(): + global _gc_frozen + if _gc_frozen: + return + # Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing + # most of the memory if there are no writes made to data because of Copy on Write, however, + # python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the + # generational GC which stores and mutates every python object: `PyGC_Head` + # + # Calling gc.freeze() moves all the objects imported so far into permanant generation and hence + # doesn't mutate `PyGC_Head` + # + # Refer to issue for more info: https://github.com/frappe/frappe/issues/18927 + gc.collect() + gc.freeze() + # RQ workers constantly fork, there' no benefit in doing this in that case. + _gc_frozen = True diff --git a/frappe/public/js/workflow_builder/store.js b/frappe/public/js/workflow_builder/store.js index 179150fc3258..a5cf1abffe4a 100644 --- a/frappe/public/js/workflow_builder/store.js +++ b/frappe/public/js/workflow_builder/store.js @@ -173,8 +173,18 @@ export const useStore = defineStore("workflow-builder-store", () => { actions.forEach((action) => { let states = workflow.value.elements.filter((e) => e.type == "state"); - let state = states.find((state) => state.data.state == action.data.from); - let next_state = states.find((state) => state.data.state == action.data.to); + + let state = states.find( + (state) => state.data.workflow_builder_id == action.data.from_id + ); + let next_state = states.find( + (state) => state.data.workflow_builder_id == action.data.to_id + ); + + if (action.data.to.length === 0 && next_state != undefined) { + action.data.to = next_state.data.state; + } + let error = validate_transitions(state.data, next_state.data); if (error) { frappe.throw({ diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 0575ddab55e0..6b5534e90366 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -667,6 +667,17 @@ def test_timesmap_utils(self): self.assertEqual(get_year_ending(date(2021, 1, 1)), date(2021, 12, 31)) self.assertEqual(get_year_ending(date(2021, 1, 31)), date(2021, 12, 31)) + @given(st.datetimes()) + def test_get_datetime(self, original): + parsed = get_datetime(str(original)) + self.assertEqual(parsed, original) + + @given(st.datetimes(timezones=st.timezones())) + def test_get_datetime_tz_aware(self, original): + parsed = get_datetime(str(original)) + original = original.replace(tzinfo=None) + self.assertEqual(parsed, original) + def test_pretty_date(self): from frappe import _ diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 8c7627f43d2f..d84bc468eae8 100644 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -298,7 +298,6 @@ def start_worker( strategy = DequeueStrategy.DEFAULT _start_sentry() - _freeze_gc() with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json @@ -365,11 +364,8 @@ def start_worker_pool( import frappe.utils.scheduler import frappe.utils.typing_validations # any whitelisted method uses this import frappe.website.path_resolver # all the page types and resolver - # end: module pre-loading - _freeze_gc() - with frappe.init_site(): redis_connection = get_redis_conn() @@ -394,12 +390,6 @@ def start_worker_pool( pool.start(logging_level=logging_level, burst=burst) -def _freeze_gc(): - if frappe._tune_gc: - gc.collect() - gc.freeze() - - def get_worker_name(queue): """When limiting worker to a specific queue, also append queue name to default worker name""" name = None diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index 51e6d97ee7ff..4381d3b0cd53 100755 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -2,7 +2,6 @@ import importlib import json -import linecache import os import sys import traceback @@ -67,37 +66,9 @@ class CliCtxObj: def handle_exception(cmd, info_name, exc): - tb = sys.exc_info()[2] - while tb.tb_next: - tb = tb.tb_next - frame = tb.tb_frame - filename = frame.f_code.co_filename - lineno = frame.f_lineno - - click.secho("\n:: ", nl=False) - click.secho(f"{exc}", fg="red", bold=True, nl=False) - click.secho(" ::") - click.secho("\nContext:", fg="yellow", bold=True) - click.secho(f" File '{filename}', line {lineno}\n") - context_lines = 5 - start = max(1, lineno - context_lines) - end = lineno + context_lines + 1 - - for i in range(start, end): - line = linecache.getline(filename, i).rstrip() - if i == lineno: - click.secho(f"{i:4d}> {line}", fg="red") - else: - click.echo(f"{i:4d}: {line}") - - show_exception = (not sys.stdout.isatty()) or click.confirm( - "\nDo you want to see the full traceback?", default=False - ) - if show_exception: - click.secho("\nFull traceback:", fg="red") - click.echo(traceback.format_exc()) + click.echo(traceback.format_exc()) - click.echo(exc) + click.echo(exc) def main(): diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 440d8b821156..d7259027641f 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -146,7 +146,7 @@ def get_datetime( if datetime_str is None: return now_datetime() - if isinstance(datetime_str, datetime.datetime | datetime.timedelta): + elif isinstance(datetime_str, datetime.datetime | datetime.timedelta): return datetime_str elif isinstance(datetime_str, list | tuple): @@ -155,11 +155,19 @@ def get_datetime( elif isinstance(datetime_str, datetime.date): return datetime.datetime.combine(datetime_str, datetime.time()) - if is_invalid_date_string(datetime_str): + elif is_invalid_date_string(datetime_str): return None try: - return datetime.datetime.strptime(datetime_str, DATETIME_FORMAT) + # PERF: Our DATETIME_FORMAT is same as ISO format. + # fromisoformat is written in C so it's better than using strptime parser + dt_object = datetime.datetime.fromisoformat(datetime_str) + + # fromisoformat also adds tzinfo if present in src string, + # so we strip it before returning + if dt_object.tzinfo: + return dt_object.replace(tzinfo=None) + return dt_object except ValueError: return parser.parse(datetime_str) @@ -1197,10 +1205,10 @@ def rounded(num, precision=0, rounding_method=None): rounding_method or frappe.get_system_settings("rounding_method") or "Banker's Rounding (legacy)" ) - if rounding_method == "Banker's Rounding (legacy)": - return _bankers_rounding_legacy(num, precision) - elif rounding_method == "Banker's Rounding": + if rounding_method == "Banker's Rounding": return _bankers_rounding(num, precision) + elif rounding_method == "Banker's Rounding (legacy)": + return _bankers_rounding_legacy(num, precision) elif rounding_method == "Commercial Rounding": return _round_away_from_zero(num, precision) else: @@ -1671,8 +1679,8 @@ def pretty_date(iso_datetime: datetime.datetime | str) -> str: from babel.dates import format_timedelta if isinstance(iso_datetime, str): - iso_datetime = datetime.datetime.strptime(iso_datetime, DATETIME_FORMAT) - now_dt = datetime.datetime.strptime(now(), DATETIME_FORMAT) + iso_datetime = get_datetime(iso_datetime) + now_dt = now_datetime() locale = frappe.local.lang.replace("-", "_") if frappe.local.lang else None return format_timedelta(iso_datetime - now_dt, add_direction=True, locale=locale)