diff --git a/server/analytics/base_report/__init__.py b/server/analytics/base_report/__init__.py index 884b667d3..a1c82c4cc 100644 --- a/server/analytics/base_report/__init__.py +++ b/server/analytics/base_report/__init__.py @@ -8,9 +8,10 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license -from flask import json, current_app as app from eve_elastic.elastic import set_filters, ElasticCursor +from superdesk.core import json, get_app_config, get_current_app +from superdesk.resource_fields import ITEMS from superdesk import get_resource_service, es_utils from superdesk.resource import Resource from superdesk.utils import ListCursor @@ -272,8 +273,9 @@ def run_query(self, params, args): filters = self._get_filters(types, excluded_stages) # if the system has a setting value for the maximum search depth then apply the filter - if not app.settings["MAX_SEARCH_DEPTH"] == -1: - query["terminate_after"] = app.settings["MAX_SEARCH_DEPTH"] + max_search_depth = get_app_config("MAX_SEARCH_DEPTH") + if max_search_depth != -1: + query["terminate_after"] = max_search_depth if filters: set_filters(query, filters) @@ -284,8 +286,9 @@ def run_query(self, params, args): docs = self.elastic.search(query, types, params={}) + app = get_current_app().as_any() for resource in types: - response = {app.config["ITEMS"]: [doc for doc in docs if doc["_type"] == resource]} + response = {ITEMS: [doc for doc in docs if doc["_type"] == resource]} getattr(app, "on_fetched_resource")(resource, response) getattr(app, "on_fetched_resource_%s" % resource)(response) @@ -330,7 +333,7 @@ def get(self, req, **lookup): return ListCursor([report]) def get_utc_offset(self): - return get_timezone_offset(app.config["DEFAULT_TIMEZONE"], utcnow()) + return get_timezone_offset(get_app_config("DEFAULT_TIMEZONE"), utcnow()) def format_date(self, date, end_of_day=False): time_suffix = "T23:59:59" if end_of_day else "T00:00:00" diff --git a/server/analytics/commands/send_schedule_reports_test.py b/server/analytics/commands/send_schedule_reports_test.py index 26afc4859..b939f5932 100644 --- a/server/analytics/commands/send_schedule_reports_test.py +++ b/server/analytics/commands/send_schedule_reports_test.py @@ -8,6 +8,14 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from datetime import datetime +from dateutil.rrule import rrule, HOURLY +import pytz +from unittest import mock +from os import urandom +from base64 import b64encode, b64decode + +from superdesk.core import get_app_config from superdesk import get_resource_service from superdesk.utc import local_to_utc @@ -16,13 +24,7 @@ from analytics.common import MIME_TYPES from analytics.email_report.email_report import EmailReportService -from datetime import datetime -from flask import current_app as app -from dateutil.rrule import rrule, HOURLY -import pytz -from unittest import mock -from os import urandom -from base64 import b64encode, b64decode + def to_naive(date_str): @@ -30,11 +32,11 @@ def to_naive(date_str): def to_utc(date_str): - return local_to_utc(app.config["DEFAULT_TIMEZONE"], datetime.strptime(date_str, "%Y-%m-%dT%H")) + return local_to_utc(get_app_config("DEFAULT_TIMEZONE"), datetime.strptime(date_str, "%Y-%m-%dT%H")) def to_local(date_str): - local_tz = pytz.timezone(app.config["DEFAULT_TIMEZONE"]) + local_tz = pytz.timezone(get_app_config("DEFAULT_TIMEZONE")) local_datetime = datetime.strptime(date_str, "%Y-%m-%dT%H") return local_tz.localize(local_datetime) @@ -94,8 +96,9 @@ def setUp(self): def _test(self, report, start, end, expected_hits): count = 0 + default_timezone = get_app_config("DEFAULT_TIMEZONE") for now in rrule(HOURLY, dtstart=to_naive(start), until=to_naive(end)): - local_tz = pytz.timezone(app.config["DEFAULT_TIMEZONE"]) + local_tz = pytz.timezone(default_timezone) now_local = local_tz.localize(now) response = self.should_send(report, now_local) @@ -108,7 +111,7 @@ def _test(self, report, start, end, expected_hits): if response: # Update the last sent time to now - report["_last_sent"] = local_to_utc(app.config["DEFAULT_TIMEZONE"], now_local) + report["_last_sent"] = local_to_utc(default_timezone, now_local) count += 1 self.assertEqual(len(expected_hits), count) @@ -145,11 +148,12 @@ def test_run_hourly_png(self, mocked): # Simulate running every hour for a few hours start_date = to_naive("2018-06-30T00") end_date = to_naive("2018-06-30T03") - local_tz = pytz.timezone(app.config["DEFAULT_TIMEZONE"]) + default_timezone = get_app_config("DEFAULT_TIMEZONE") + local_tz = default_timezone with self.app.mail.record_messages() as outbox: for now in rrule(HOURLY, dtstart=start_date, until=end_date): now_local = local_tz.localize(now) - now_utc = local_to_utc(app.config["DEFAULT_TIMEZONE"], now_local) + now_utc = local_to_utc(default_timezone, now_local) SendScheduledReports().run(now_utc) @@ -214,11 +218,12 @@ def test_run_daily_jpeg(self, mocked): start_date = to_naive("2018-06-30T00") end_date = to_naive("2018-06-30T03") should_have_updated = False + default_timezone = get_app_config("DEFAULT_TIMEZONE") with self.app.mail.record_messages() as outbox: for now in rrule(HOURLY, dtstart=start_date, until=end_date): - local_tz = pytz.timezone(app.config["DEFAULT_TIMEZONE"]) + local_tz = pytz.timezone(default_timezone) now_local = local_tz.localize(now) - now_utc = local_to_utc(app.config["DEFAULT_TIMEZONE"], now_local) + now_utc = local_to_utc(default_timezone, now_local) SendScheduledReports().run(now_utc) diff --git a/server/analytics/commands/send_scheduled_reports.py b/server/analytics/commands/send_scheduled_reports.py index c313281ac..b28037dac 100644 --- a/server/analytics/commands/send_scheduled_reports.py +++ b/server/analytics/commands/send_scheduled_reports.py @@ -8,14 +8,14 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from datetime import datetime + +from superdesk.core import get_app_config from superdesk import Command, command, Option, get_resource_service from superdesk.logging import logger from superdesk.errors import SuperdeskApiError from superdesk.utc import utc_to_local, utcnow, local_to_utc -from flask import current_app as app -from datetime import datetime - class SendScheduledReports(Command): """ @@ -40,19 +40,17 @@ class SendScheduledReports(Command): ] def run(self, now=None): + default_timezone = get_app_config("DEFAULT_TIMEZONE") if now: now_utc = ( now if isinstance(now, datetime) - else local_to_utc( - app.config["DEFAULT_TIMEZONE"], - datetime.strptime(now, "%Y-%m-%dT%H"), - ) + else local_to_utc(default_timezone, datetime.strptime(now, "%Y-%m-%dT%H")) ) else: now_utc = utcnow() - now_local = utc_to_local(app.config["DEFAULT_TIMEZONE"], now_utc) + now_local = utc_to_local(default_timezone, now_utc) logger.info("Starting to send scheduled reports: {}".format(now_utc)) @@ -99,7 +97,7 @@ def should_send_report(scheduled_report, now_local): last_sent = None if scheduled_report.get("_last_sent"): - last_sent = utc_to_local(app.config["DEFAULT_TIMEZONE"], scheduled_report.get("_last_sent")).replace( + last_sent = utc_to_local(get_app_config("DEFAULT_TIMEZONE"), scheduled_report.get("_last_sent")).replace( minute=0, second=0, microsecond=0 ) diff --git a/server/analytics/common.py b/server/analytics/common.py index 08ade4809..76de0117e 100644 --- a/server/analytics/common.py +++ b/server/analytics/common.py @@ -10,11 +10,6 @@ from typing import NamedTuple from os import path - -from superdesk import get_resource_service -from superdesk.utc import utcnow, utc_to_local -from subprocess import check_call, PIPE -from flask import current_app as app import pytz from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta @@ -22,6 +17,11 @@ import re import logging +from superdesk.core import get_app_config, get_current_app +from superdesk import get_resource_service +from superdesk.utc import utcnow, utc_to_local + + ANALYTICS_PATH = path.abspath(path.dirname(path.realpath(__file__))) logger = logging.getLogger(__name__) @@ -168,6 +168,7 @@ def get_cv_by_qcode(name, field=None): def get_elastic_version(): + app = get_current_app() return app.data.elastic.es.info()["version"]["number"] @@ -181,7 +182,7 @@ def get_weekstart_offset_hr(): """ offset = 0 - start_of_week = app.config.get("START_OF_WEEK") or 0 + start_of_week = get_app_config("START_OF_WEEK") or 0 if start_of_week == 0: offset -= 24 elif start_of_week > 1: @@ -196,7 +197,7 @@ def get_utc_offset_in_minutes(utc_datetime): :param datetime utc_datetime: The date/time instance used to calculate utc offset :return: UTC Offset in minutes """ - timezone = pytz.timezone(app.config["DEFAULT_TIMEZONE"]) + timezone = pytz.timezone(get_app_config("DEFAULT_TIMEZONE")) return timezone.utcoffset(utc_datetime).total_seconds() / 60 @@ -267,7 +268,7 @@ def relative_to_absolute_datetime(value, format, now=None, offset=None): raise if now is None: - now = utc_to_local(app.config.get("DEFAULT_TIMEZONE"), utcnow()) + now = utc_to_local(get_app_config("DEFAULT_TIMEZONE"), utcnow()) if values.get("offset"): # Retrieve the offset value and granularity, then shift the datetime @@ -310,7 +311,7 @@ def relative_to_absolute_datetime(value, format, now=None, offset=None): if isoweekday == 7: isoweekday = 0 - start_of_week = app.config.get("START_OF_WEEK") or 0 + start_of_week = get_app_config("START_OF_WEEK") or 0 offset = 7 - start_of_week + isoweekday if offset < 7: diff --git a/server/analytics/desk_activity_report/desk_activity_report.py b/server/analytics/desk_activity_report/desk_activity_report.py index aa565203b..3d92ad226 100644 --- a/server/analytics/desk_activity_report/desk_activity_report.py +++ b/server/analytics/desk_activity_report/desk_activity_report.py @@ -8,6 +8,9 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from datetime import datetime + +from superdesk.core import get_app_config from superdesk.resource import Resource from superdesk.errors import SuperdeskApiError @@ -21,9 +24,6 @@ MAX_TERMS_SIZE, ) -from flask import current_app as app -from datetime import datetime - class DeskActivityReportResource(Resource): """Desk Activity Report schema""" @@ -202,7 +202,7 @@ def gen_chart_config(): title=title, subtitle=subtitle, chart_type="highcharts", - start_of_week=app.config.get("START_OF_WEEK") or 0, + start_of_week=get_app_config("START_OF_WEEK") or 0, timezone_offset=timezone_offset, use_utc=False, legend_title="Desk Transitions", diff --git a/server/analytics/email_report/analytics_message.py b/server/analytics/email_report/analytics_message.py index 9d91a8383..b4316f456 100644 --- a/server/analytics/email_report/analytics_message.py +++ b/server/analytics/email_report/analytics_message.py @@ -8,8 +8,6 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license -from superdesk.emails import SuperdeskMessage - import re import unicodedata @@ -18,7 +16,6 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart -from flask import current_app from flask_mail import ( sanitize_subject, sanitize_address, @@ -27,6 +24,9 @@ formatdate, ) +from superdesk.core import get_current_app +from superdesk.emails import SuperdeskMessage + string_types = (str,) text_type = str message_policy = policy.SMTP @@ -35,7 +35,7 @@ class AnalyticsMessage(SuperdeskMessage): def _message(self): """Creates email as 'multipart/related' instead of 'multipart/mixed'""" - ascii_attachments = current_app.extensions["mail"].ascii_attachments + ascii_attachments = get_current_app().extensions["mail"].ascii_attachments encoding = self.charset or "utf-8" attachments = self.attachments or [] diff --git a/server/analytics/email_report/email_report.py b/server/analytics/email_report/email_report.py index 6e4877013..aac9b5530 100644 --- a/server/analytics/email_report/email_report.py +++ b/server/analytics/email_report/email_report.py @@ -8,6 +8,13 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from email.charset import Charset, QP +from base64 import b64decode +from uuid import uuid4 +from bson import ObjectId + +from superdesk.core import get_current_app, get_app_config +from superdesk.flask import render_template from superdesk.services import BaseService from superdesk.resource import Resource from superdesk.errors import SuperdeskApiError @@ -23,12 +30,6 @@ from analytics.reports import generate_report from .analytics_message import AnalyticsMessage -from flask import current_app as app, render_template -from email.charset import Charset, QP -from base64 import b64decode -from uuid import uuid4 -from bson import ObjectId - class EmailReportResource(Resource): """Resource to email report charts""" @@ -175,7 +176,7 @@ def _email_report(email, attachments): kwargs={ "_id": str(ObjectId()), "subject": email.get("subject"), - "sender": email.get("sender") or app.config["ADMINS"][0], + "sender": email.get("sender") or get_app_config("ADMINS")[0], "recipients": email.get("recipients"), "text_body": txt.get("body") or "", "html_body": html.get("body") or "", @@ -261,6 +262,7 @@ def send_email_report( html_body=html_body.replace("\r", "").replace("\n", "
"), reports=reports, ) + app = get_current_app() return app.mail.send(msg) except Exception as e: diff --git a/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py b/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py index e55e72cb2..503bd56f1 100644 --- a/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py +++ b/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py @@ -8,6 +8,9 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from eve_elastic.elastic import parse_date + +from superdesk.core import get_app_config from superdesk.resource import Resource from superdesk.utc import utc_to_local @@ -15,9 +18,6 @@ from analytics.chart_config import ChartConfig from analytics.common import REPORT_CONFIG, CHART_TYPES -from flask import current_app as app -from eve_elastic.elastic import parse_date - class FeaturemdiaUpdatesReportResource(Resource): """Featuremedia Updates Report schema""" @@ -175,7 +175,7 @@ def generate_highcharts_config(self, docs, args): rows = [] def gen_date_str(date): - return utc_to_local(app.config["DEFAULT_TIMEZONE"], date).strftime("%d/%m/%Y %H:%M") + return utc_to_local(get_app_config("DEFAULT_TIMEZONE"), date).strftime("%d/%m/%Y %H:%M") for item in items: original_image = item.get("original_image") or {} diff --git a/server/analytics/reports/__init__.py b/server/analytics/reports/__init__.py index a76ac9ea5..b3e33d248 100644 --- a/server/analytics/reports/__init__.py +++ b/server/analytics/reports/__init__.py @@ -17,7 +17,7 @@ from os import path from base64 import b64encode -from flask import json +from superdesk.core import json from superdesk.errors import SuperdeskApiError from superdesk.timer import timer from analytics.common import MIME_TYPES, get_highcharts_cli_path diff --git a/server/analytics/stats/archive_statistics.py b/server/analytics/stats/archive_statistics.py index 6986bbb44..6a702764b 100644 --- a/server/analytics/stats/archive_statistics.py +++ b/server/analytics/stats/archive_statistics.py @@ -8,7 +8,9 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from eve.utils import ParsedRequest, date_to_str +from superdesk.resource_fields import ID_FIELD from superdesk import get_resource_service, json from superdesk.services import BaseService from superdesk.resource import Resource, not_indexed, not_analyzed, not_enabled @@ -25,8 +27,6 @@ from analytics.stats.common import STAT_TYPE -from eve.utils import config, ParsedRequest, date_to_str - class ArchiveStatisticsResource(Resource): endpoint_name = resource_title = url = "archive_statistics" @@ -44,7 +44,7 @@ class ArchiveStatisticsResource(Resource): query_objectid_as_string = True schema = { - config.ID_FIELD: metadata_schema[config.ID_FIELD], + ID_FIELD: metadata_schema[ID_FIELD], "guid": metadata_schema["guid"], "stats_type": {"type": "string"}, "stats": { @@ -257,8 +257,8 @@ def set_last_run_id(self, entry_id, last_run=None): if last_run is None: last_run = self.get_last_run() - if last_run and last_run.get(config.ID_FIELD): - self.patch(last_run[config.ID_FIELD], {"guid": entry_id}) + if last_run and last_run.get(ID_FIELD): + self.patch(last_run[ID_FIELD], {"guid": entry_id}) else: self.post([{"guid": entry_id, "stats_type": "last_run"}]) @@ -292,5 +292,5 @@ def get_history_items(self, last_id, gte, item_id, chunk_size=0): if len(items) < 1: break - last_processed_id = items[-1][config.ID_FIELD] + last_processed_id = items[-1][ID_FIELD] yield items diff --git a/server/analytics/stats/featuremedia_updates.py b/server/analytics/stats/featuremedia_updates.py index 78a4ba1a2..20d5ea01d 100644 --- a/server/analytics/stats/featuremedia_updates.py +++ b/server/analytics/stats/featuremedia_updates.py @@ -8,6 +8,10 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from copy import deepcopy + +from superdesk.core import get_app_config +from superdesk.resource_fields import ID_FIELD from superdesk import get_resource_service from superdesk.metadata.item import ASSOCIATIONS, CONTENT_TYPE from superdesk.logging import logger @@ -15,10 +19,6 @@ from analytics.stats.common import STAT_TYPE, OPERATION, FEATUREMEDIA_OPERATIONS from analytics.stats.gen_archive_statistics import connect_stats_signals -from copy import deepcopy -from flask import current_app as app -from eve.utils import config - class FeaturemediaUpdates: def __init__(self, sender=None): @@ -40,7 +40,7 @@ def init(self, sender, stats): def process(self, sender, entry, new_timeline, updates, update, stats): # Generating stats with PUBLISH_ASSOCIATED_ITEMS=True is currently not supported - if app.config.get("PUBLISH_ASSOCIATED_ITEMS", False): + if get_app_config("PUBLISH_ASSOCIATED_ITEMS", False): return operation = entry.get("operation") @@ -226,7 +226,7 @@ def finish(self, sender): def get_parent_id(doc): if not doc.get("rewrite_of"): - return doc[config.ID_FIELD] + return doc[ID_FIELD] elif doc["rewrite_of"] not in docs: # If the parent item was not part of this stats iteration # then load it now diff --git a/server/analytics/stats/gen_archive_statistics.py b/server/analytics/stats/gen_archive_statistics.py index 4635884ae..d372a5022 100644 --- a/server/analytics/stats/gen_archive_statistics.py +++ b/server/analytics/stats/gen_archive_statistics.py @@ -8,6 +8,10 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from copy import deepcopy +from datetime import timedelta + +from superdesk.resource_fields import ID_FIELD from superdesk import Command, command, get_resource_service, Option from superdesk.logging import logger from superdesk.utc import utcnow @@ -29,10 +33,6 @@ from analytics.stats.common import STAT_TYPE, OPERATION from analytics.stats import desk_transitions -from eve.utils import config -from copy import deepcopy -from datetime import timedelta - gen_stats_signals = { "start": signals.signal("gen_archive_statistics:start"), "generate": signals.signal("gen_archive_statistics:generate"), @@ -207,7 +207,7 @@ def generate_stats(self, item_id, gte, chunk_size): gen_stats_signals["start"].send(self) num_history_items += len(history_items) - last_entry_id = history_items[-1].get(config.ID_FIELD) + last_entry_id = history_items[-1].get(ID_FIELD) items = self.gen_history_timelines(history_items) items_processed += len(items) @@ -277,7 +277,7 @@ def _set_timeline_processed(entry): except Exception: logger.exception( "Failed to process archive_history for item:{} history:{}".format( - item_id, history_item.get(config.ID_FIELD) + item_id, history_item.get(ID_FIELD) ) ) @@ -288,7 +288,7 @@ def gen_archive_stats_from_history(self, item, history): task = history["update"].get("task") or {} entry = { - "history_id": history.get(config.ID_FIELD), + "history_id": history.get(ID_FIELD), "operation": history.get("operation"), "operation_created": history.get("_created"), "task": { @@ -420,8 +420,8 @@ def process_timelines(self, items, failed_ids): if item["updates"].get("rewrite_of") and (item["updates"].get("time_to_first_publish") or 0) > 0: rewrites.append(item_id) - if not item["item"].get(config.ID_FIELD): - item["updates"][config.ID_FIELD] = item_id + if not item["item"].get(ID_FIELD): + item["updates"][ID_FIELD] = item_id item["updates"]["stats_type"] = "archive" items_to_create.append(item["updates"]) else: @@ -437,7 +437,7 @@ def process_timelines(self, items, failed_ids): try: statistics_service.post(items_to_create) except Exception: - item_ids = [item.get(config.ID_FIELD) for item in items_to_create] + item_ids = [item.get(ID_FIELD) for item in items_to_create] logger.exception("Failed to create stat entries for items {}".format(", ".join(item_ids))) failed_ids.extend(failed_ids) diff --git a/server/analytics/stats/stats_report_service.py b/server/analytics/stats/stats_report_service.py index 8df598f05..e5c878404 100644 --- a/server/analytics/stats/stats_report_service.py +++ b/server/analytics/stats/stats_report_service.py @@ -8,7 +8,7 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license -from flask import current_app as app +from superdesk.core import get_app_config from analytics.base_report import BaseReportService @@ -16,7 +16,7 @@ class StatsReportService(BaseReportService): repos = ["archive_statistics"] def get_elastic_index(self, types): - return app.config.get("STATISTICS_ELASTIC_INDEX") or app.config.get("STATISTICS_MONGO_DBNAME") or "statistics" + return get_app_config("STATISTICS_ELASTIC_INDEX") or get_app_config("STATISTICS_MONGO_DBNAME") or "statistics" def get_es_stats_type(self, query, params): query["must"].append({"term": {"stats_type": "archive"}}) diff --git a/server/analytics/update_time_report/update_time_report.py b/server/analytics/update_time_report/update_time_report.py index 29c8a3fc5..c0bb889dd 100644 --- a/server/analytics/update_time_report/update_time_report.py +++ b/server/analytics/update_time_report/update_time_report.py @@ -8,6 +8,10 @@ # AUTHORS and LICENSE files distributed with this source code, or # at https://www.sourcefabric.org/superdesk/license +from datetime import timedelta +from copy import deepcopy + +from superdesk.core import get_app_config from superdesk.resource import Resource from superdesk.utc import utc_to_local, utcnow @@ -16,10 +20,6 @@ from analytics.chart_config import ChartConfig from analytics.common import REPORT_CONFIG, CHART_TYPES -from flask import current_app as app -from datetime import timedelta -from copy import deepcopy - class UpdateTimeReportResource(Resource): """Update Time Report schema""" @@ -89,7 +89,7 @@ def generate_highcharts_config(self, docs, args): rows = [] def gen_date_str(date): - return utc_to_local(app.config["DEFAULT_TIMEZONE"], date).strftime("%d/%m/%Y %H:%M") + return utc_to_local(get_app_config("DEFAULT_TIMEZONE"), date).strftime("%d/%m/%Y %H:%M") def gen_update_string(seconds): times = (utcnow().replace(minute=0, hour=0, second=0)) + timedelta(seconds=seconds)