diff --git a/.gitignore b/.gitignore index 73edd7ff..62673f27 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,6 @@ project_export* tmp/* !tmp/.gitkeep -logs/* \ No newline at end of file +logs/* + +current_config.json \ No newline at end of file diff --git a/api/misc.py b/api/misc.py index 476d26f0..6c9602d5 100644 --- a/api/misc.py +++ b/api/misc.py @@ -1,15 +1,12 @@ -from controller.misc import config_service from starlette.endpoints import HTTPEndpoint from starlette.responses import JSONResponse -from starlette import status +from fastapi import Request +from config_handler import ( + full_config_json, +) -class IsManagedRest(HTTPEndpoint): - def get(self, request) -> JSONResponse: - is_managed = config_service.get_config_value("is_managed") - return JSONResponse(is_managed, status_code=status.HTTP_200_OK) -class IsDemoRest(HTTPEndpoint): - def get(self, request) -> JSONResponse: - is_managed = config_service.get_config_value("is_demo") - return JSONResponse(is_managed, status_code=status.HTTP_200_OK) +class FullConfigRest(HTTPEndpoint): + def get(self, request: Request) -> JSONResponse: + return full_config_json() diff --git a/api/transfer.py b/api/transfer.py index 72eae0db..15fd5f55 100644 --- a/api/transfer.py +++ b/api/transfer.py @@ -28,7 +28,6 @@ from controller.upload_task import manager as upload_task_manager from controller.auth import manager as auth_manager from controller.transfer import association_transfer_manager -from controller.auth import manager as auth from controller.project import manager as project_manager from controller.attribute import manager as attribute_manager @@ -135,7 +134,6 @@ def get(self, request) -> JSONResponse: class PrepareFileImport(HTTPEndpoint): async def post(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] request_body = await request.json() @@ -168,7 +166,6 @@ async def post(self, request) -> JSONResponse: class JSONImport(HTTPEndpoint): async def post(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] request_body = await request.json() user_id = request_body["user_id"] @@ -272,7 +269,6 @@ async def post(self, request) -> JSONResponse: class UploadTaskInfo(HTTPEndpoint): def get(self, request) -> JSONResponse: - auth.check_is_demo_without_info() project_id = request.path_params["project_id"] task_id = request.path_params["task_id"] user_id = request.query_params["user_id"] diff --git a/app.py b/app.py index 32a69f2f..57b3fb04 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,10 @@ import logging - from fastapi import FastAPI from api.healthcheck import Healthcheck from starlette.middleware import Middleware -from api.misc import IsDemoRest, IsManagedRest +from api.misc import ( + FullConfigRest, +) from api.project import ProjectDetails from api.transfer import ( AssociationsImport, @@ -16,6 +17,9 @@ CognitionImport, CognitionPrepareProject, ) +from config_handler import ( + init_config, +) from fast_api.routes.organization import router as org_router from fast_api.routes.project import router as project_router from fast_api.routes.project_setting import router as project_setting_router @@ -65,8 +69,10 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) +init_config() fastapi_app = FastAPI() + fastapi_app.include_router( org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"] ) @@ -110,7 +116,9 @@ fastapi_app_internal.include_router( task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"] ) + routes = [ + Route("/full_config", FullConfigRest), Route("/notify/{path:path}", Notify), Route("/healthcheck", Healthcheck), Route("/project/{project_id:str}", ProjectDetails), @@ -130,8 +138,6 @@ CognitionPrepareProject, ), Route("/project/{project_id:str}/import/task/{task_id:str}", UploadTaskInfo), - Route("/is_managed", IsManagedRest), - Route("/is_demo", IsDemoRest), Mount("/api", app=fastapi_app, name="REST API"), Mount( "/internal/api", app=fastapi_app_internal, name="INTERNAL REST API" diff --git a/base_config.json b/base_config.json new file mode 100644 index 00000000..8df4b168 --- /dev/null +++ b/base_config.json @@ -0,0 +1,8 @@ +{ + "s3_region": null, + "KERN_S3_ENDPOINT": null, + "spacy_downloads": [ + "en_core_web_sm", + "de_core_news_sm" + ] +} \ No newline at end of file diff --git a/check_config_service b/check_config_service deleted file mode 100644 index 00bd9d4c..00000000 --- a/check_config_service +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -CONFIG_EXISTS=0 -if [ -n "$(docker ps -q -f name=dev-setup_refinery-config_1)" ] -then - if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup_refinery-config_1 )" == "running" ]; - then - CONFIG_EXISTS=1 - fi -elif [ -n "$(docker ps -q -f name=dev-setup-refinery-config-1)" ] -then - if [ "$( docker container inspect -f '{{.State.Status}}' dev-setup-refinery-config-1 )" == "running" ]; - then - CONFIG_EXISTS=1 - fi -else - if [ -n "$(docker ps -q -f name=refinery-config)" ]; - then - if [ "$( docker container inspect -f '{{.State.Status}}' refinery-config )" == "running" ]; - then - CONFIG_EXISTS=1 - fi - fi -fi -if [ $CONFIG_EXISTS -eq 0 ] -then - echo "refinery-config couldn't be found - exit" - exit 1 -else - echo "refinery-config found -> proceeding" -fi - diff --git a/config/.gitkeep b/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/config_handler.py b/config_handler.py new file mode 100644 index 00000000..f81ca1c4 --- /dev/null +++ b/config_handler.py @@ -0,0 +1,123 @@ +from typing import Dict, Any, Optional, Union +import os +import json +from notify_handler import notify_others_about_change_thread +from fastapi import responses, status + +__config = None + +BASE_CONFIG_PATH = "base_config.json" +CURRENT_CONFIG_PATH = "/config/current_config.json" + +SERVICES_TO_NOTIFY = { + "TOKENIZER": "http://refinery-tokenizer:80", +} + + +def get_config_value( + key: str, subkey: Optional[str] = None +) -> Union[str, Dict[str, str]]: + if key not in __config: + raise ValueError(f"Key {key} coudn't be found in config") + value = __config[key] + + if not subkey: + return value + + if isinstance(value, dict) and subkey in value: + return value[subkey] + else: + raise ValueError(f"Subkey {subkey} coudn't be found in config[{key}]") + + +def __read_and_change_base_config(): + print("reading base config file", flush=True) + global __config + f = open(BASE_CONFIG_PATH) + __config = json.load(f) + __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1") + __save_current_config() + + +def change_config(changes: Dict[str, Any]) -> bool: + global __config + something_changed = False + for key in changes: + if key == "KERN_S3_ENDPOINT": + continue + if key in __config: + if isinstance(changes[key], dict): + for subkey in changes[key]: + if subkey in __config[key]: + __config[key][subkey] = changes[key][subkey] + something_changed = True + else: + __config[key] = changes[key] + something_changed = True + if something_changed: + __save_current_config() + else: + print("nothing was changed with input", changes, flush=True) + return something_changed + + +def __save_current_config() -> None: + print("saving config file", flush=True) + with open(CURRENT_CONFIG_PATH, "w") as f: + json.dump(__config, f, indent=4) + + +def init_config() -> None: + if not os.path.exists(CURRENT_CONFIG_PATH): + __read_and_change_base_config() + else: + __load_and_remove_outdated_config_keys() + # this one is to be set on every start to ensure its up to date + print("setting s3 endpoint", flush=True) + __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT") + + +def __load_and_remove_outdated_config_keys(): + if not os.path.exists(CURRENT_CONFIG_PATH): + return + + global __config + with open(CURRENT_CONFIG_PATH) as f: + __config = json.load(f) + + with open(BASE_CONFIG_PATH) as f: + base_config = json.load(f) + + to_remove = [key for key in __config if key not in base_config] + + if len(to_remove) > 0: + print("removing outdated config keys", to_remove, flush=True) + for key in to_remove: + del __config[key] + __save_current_config() + + +def get_config() -> Dict[str, Any]: + global __config + return __config + + +def change_json(config_data) -> responses.PlainTextResponse: + try: + has_changed = change_config(config_data) + + if has_changed: + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + + return responses.PlainTextResponse( + f"Did update: {has_changed}", status_code=status.HTTP_200_OK + ) + + except Exception as e: + return responses.PlainTextResponse( + f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + +def full_config_json() -> responses.JSONResponse: + return responses.JSONResponse(status_code=status.HTTP_200_OK, content=get_config()) diff --git a/controller/auth/manager.py b/controller/auth/manager.py index b200c75d..ed285527 100644 --- a/controller/auth/manager.py +++ b/controller/auth/manager.py @@ -1,10 +1,8 @@ from typing import Any, Dict from fastapi import Request -from controller.misc import config_service from exceptions.exceptions import ( AuthManagerError, - NotAllowedInDemoError, ProjectAccessError, ) import jwt @@ -14,7 +12,6 @@ from submodules.model import enums, exceptions from submodules.model.business_objects import organization from submodules.model.models import Organization, Project, User -from controller.misc import manager as misc_manager import sqlalchemy DEV_USER_ID = "741df1c2-a531-43b6-b259-df23bc78e9a2" @@ -127,28 +124,6 @@ def check_is_admin(request: Any) -> bool: return False -def check_demo_access(info: Any) -> None: - if not check_is_admin(info.context["request"]) and config_service.get_config_value( - "is_demo" - ): - check_black_white(info) - - -def check_black_white(info: Any): - black_white = misc_manager.get_black_white_demo() - if str(info.parent_type) == "Mutation": - if info.field_name not in black_white["mutations"]: - raise NotAllowedInDemoError - elif str(info.parent_type) == "Query": - if info.field_name in black_white["queries"]: - raise NotAllowedInDemoError - - -def check_is_demo_without_info() -> None: - if config_service.get_config_value("is_demo"): - raise NotAllowedInDemoError - - def check_is_single_organization() -> bool: return len(organization_manager.get_all_organizations()) == 1 diff --git a/controller/embedding/manager.py b/controller/embedding/manager.py index 8183459a..38bb6a2a 100644 --- a/controller/embedding/manager.py +++ b/controller/embedding/manager.py @@ -42,14 +42,11 @@ def get_current_terms_text( return term_text -def get_recommended_encoders(is_managed: bool) -> List[Any]: - # only use is_managed if it is really managed +def get_recommended_encoders() -> List[Any]: # can run into circular import problems if directly resolved here by helper method recommendations = connector.request_listing_recommended_encoders() - if is_managed: - existing_models = model_manager.get_model_provider_info() - else: - existing_models = [] + existing_models = model_manager.get_model_provider_info() + for model in existing_models: not_yet_known = ( len( diff --git a/controller/information_source/manager.py b/controller/information_source/manager.py index 5ff81be6..79e05986 100644 --- a/controller/information_source/manager.py +++ b/controller/information_source/manager.py @@ -7,7 +7,6 @@ information_source, payload, ) -from controller.misc import config_service from controller.labeling_access_link import manager as link_manager from submodules.model import daemon @@ -86,7 +85,6 @@ def delete_information_source(project_id: str, source_id: str) -> None: if ( information_source_item.type == enums.InformationSourceType.ACTIVE_LEARNING.value - and config_service.get_config_value("is_managed") ): daemon.run_without_db_token( __delete_active_learner_from_inference_dir, project_id, source_id diff --git a/controller/misc/black_white_demo.py b/controller/misc/black_white_demo.py deleted file mode 100644 index b57e5296..00000000 --- a/controller/misc/black_white_demo.py +++ /dev/null @@ -1,89 +0,0 @@ -from typing import Dict, List - - -def get_black_white_demo() -> Dict[str, List[str]]: - global __whitelist_mutation_demo_parsed - global __blacklist_query_demo_parsed - - if not __whitelist_mutation_demo_parsed: - __parse_black_white_list() - toReturn = {"mutations": [], "queries": []} - for v in __whitelist_mutation_demo_parsed: - toReturn["mutations"].append(v) - for v in __blacklist_query_demo_parsed: - toReturn["queries"].append(v) - - return toReturn - - -def __snake_case_to_camel_case(str: str): - return "".join( - [ - word.title() if idx > 0 else word.lower() - for idx, word in enumerate(str.split("_")) - ] - ) - - -def __parse_black_white_list(): - global __whitelist_mutation_demo - global __blacklist_query_demo - global __whitelist_mutation_demo_parsed - global __blacklist_query_demo_parsed - - tmp = {} - for v in __whitelist_mutation_demo: - tmp[v[0].lower() + v[1:]] = v - __whitelist_mutation_demo_parsed = tmp - tmp = {} - for v in __blacklist_query_demo: - tmp[__snake_case_to_camel_case(v)] = v - __blacklist_query_demo_parsed = tmp - - -# CAUTION: this is not necessarily the class name but often times similar or equal. This is the schema name we would use in gql -__whitelist_mutation_demo = { - "CreateDataSlice", - "UpdateDataSlice", - "DeleteDataSlice", - "CreateOutlierSlice", - "UpdateSliceTypeManual", - "CreateInformationSource", - # "DeleteInformationSource", - "ToggleInformationSource", - "SetAllInformationSourceSelected", - # "UpdateInformationSource", - "CreateKnowledgeBase", - # "DeleteKnowledgeBase", - # "UpdateKnowledgeBase", - "AddTermToKnowledgeBase", - "PasteKnowledgeTerms", - "UpdateKnowledgeTerm", - "DeleteKnowledgeTerm", - "BlacklistTerm", - # "CreateLabelingTaskLabel", - "UpdateLabelColor", - "UpdateLabelHotkey", - # "DeleteLabelingTaskLabel", - "PostEvent", - "CreateNotification", - "AddClassificationLabelsToRecord", - "AddExtractionLabelToRecord", - "SetGoldStarAnnotationForTask", - "DeleteRecordLabelAssociationByIds", - "DeleteGoldStarAssociationForTask", - "UpdateRlaIsValidManual", -} - - -# Function names without resolve_ -__blacklist_query_demo = { - "upload_credentials_and_id", - "upload_task_by_id", - "prepare_project_export", -} - - -# Parsed to a dict with gql query names -__whitelist_mutation_demo_parsed = None -__blacklist_query_demo_parsed = None diff --git a/controller/misc/config_service.py b/controller/misc/config_service.py deleted file mode 100644 index 19be9be9..00000000 --- a/controller/misc/config_service.py +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Dict, Any, Optional, Union -import requests -import time -from submodules.model import daemon -from util import service_requests - -__config = None - -# these are ment to be constant values since os variables will sooner or later be removed for addresses (and used with values from config-service) -REQUEST_URL = "http://refinery-config:80/full_config" -CHANGE_URL = "http://refinery-config:80/change_config" - - -def __get_config() -> Dict[str, Any]: - global __config - if __config: - return __config - refresh_config() - return __config - - -def refresh_config(): - response = requests.get(REQUEST_URL) - if response.status_code != 200: - raise ValueError( - f"Config service cant be reached -- response.code{response.status_code}" - ) - global __config - __config = response.json() - daemon.run_without_db_token(invalidate_after, 3600) # one hour as failsave - - -def get_config_value( - key: str, subkey: Optional[str] = None -) -> Union[str, Dict[str, str]]: - config = __get_config() - if key not in config: - raise ValueError(f"Key {key} coudn't be found in config") - value = config[key] - - if not subkey: - return value - - if isinstance(value, dict) and subkey in value: - return value[subkey] - else: - raise ValueError(f"Subkey {subkey} coudn't be found in config[{key}]") - - -def invalidate_after(sec: int) -> None: - time.sleep(sec) - global __config - __config = None - - -def change_config(dict_str: str) -> None: - data = {"dict_string": dict_str} - service_requests.post_call_or_raise(CHANGE_URL, data) diff --git a/controller/misc/manager.py b/controller/misc/manager.py index a2c7b99b..c8db7771 100644 --- a/controller/misc/manager.py +++ b/controller/misc/manager.py @@ -1,6 +1,4 @@ from typing import Any, Dict, List -from controller.misc import config_service -from controller.misc import black_white_demo from fast_api.types import ServiceVersionResult from submodules.model.global_objects import customer_button from datetime import datetime @@ -19,26 +17,6 @@ BASE_URI_UPDATER = os.getenv("UPDATER") -def check_is_managed() -> bool: - return config_service.get_config_value("is_managed") - - -def check_is_demo() -> bool: - return config_service.get_config_value("is_demo") - - -def update_config(dict_str: str) -> None: - return config_service.change_config(dict_str) - - -def refresh_config() -> None: - config_service.refresh_config() - - -def get_black_white_demo() -> Dict[str, List[str]]: - return black_white_demo.get_black_white_demo() - - def get_version_overview() -> List[ServiceVersionResult]: updater_version_overview = __updater_version_overview() date_format = "%Y-%m-%dT%H:%M:%S.%f" # 2022-09-06T12:10:39.167397 diff --git a/controller/organization/manager.py b/controller/organization/manager.py index 99b0451c..5bb805e0 100644 --- a/controller/organization/manager.py +++ b/controller/organization/manager.py @@ -1,5 +1,4 @@ from typing import Any, List, Dict, Optional, Union -from controller.misc import config_service from submodules.model import enums from submodules.model.business_objects import organization, general, user @@ -101,18 +100,6 @@ def get_overview_stats(org_id: str) -> List[Dict[str, Union[str, int]]]: return organization.get_organization_overview_stats(org_id) -def can_create_local(org: bool = True) -> bool: - if config_service.get_config_value("is_managed"): - return False - existing_orgs = organization.get_all() - checkvalue = 0 if org else 1 - if len(existing_orgs) != checkvalue: - return False - if user.get_count_assigned() != 0: - return False - return True - - def __check_notification(org_id: str, key: str, value: Any): if key in ["gdpr_compliant"]: notification.send_organization_update( diff --git a/controller/payload/payload_scheduler.py b/controller/payload/payload_scheduler.py index 5d1fc3ec..420b8f2c 100644 --- a/controller/payload/payload_scheduler.py +++ b/controller/payload/payload_scheduler.py @@ -49,7 +49,6 @@ from util import notification from submodules.s3 import controller as s3 from controller.knowledge_base import util as knowledge_base -from controller.misc import config_service from util.notification import create_notification from util.miscellaneous_functions import chunk_dict from controller.weak_supervision import weak_supervision_service as weak_supervision @@ -509,9 +508,7 @@ def read_container_logs_thread( def get_inference_dir() -> str: - if config_service.get_config_value("is_managed"): - return os.getenv("INFERENCE_DIR") - return None + return os.getenv("INFERENCE_DIR") def update_records( diff --git a/controller/transfer/checks.py b/controller/transfer/checks.py index 05b7fcfe..bff43101 100644 --- a/controller/transfer/checks.py +++ b/controller/transfer/checks.py @@ -1,6 +1,5 @@ from typing import Tuple, List, Union, Dict from controller.auth import manager as auth_manager -from controller.misc.config_service import get_config_value from controller.transfer import util as transfer_util from controller.transfer.valid_arguments import valid_arguments @@ -239,7 +238,7 @@ def get_update_amount(df: pd.DataFrame, project_id: str) -> int: if sql: sql_df = pd.read_sql(sql, con=general.get_bind()) for column in keys: - if not column in df.columns: + if column not in df.columns: return 0 type_name = df[column].dtype.name if type_name in ["int64", "float64", "bool"]: diff --git a/exceptions/exceptions.py b/exceptions/exceptions.py index ce743be7..fa7a4102 100644 --- a/exceptions/exceptions.py +++ b/exceptions/exceptions.py @@ -18,14 +18,6 @@ class MissingArgumentsException(Exception): pass -class NotAllowedInDemoError(Exception): - pass - - -class NotAllowedInOpenSourceError(Exception): - pass - - class BadPasswordError(Exception): pass diff --git a/fast_api/models.py b/fast_api/models.py index 5d1f5387..6385027e 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -210,10 +210,6 @@ class ArchiveAdminMessageBody(BaseModel): archived_reason: StrictStr -class UpdateConfigBody(BaseModel): - dict_str: StrictStr - - class ChangeUserRoleBody(BaseModel): user_id: StrictStr role: StrictStr diff --git a/fast_api/routes/embedding.py b/fast_api/routes/embedding.py index 9f722f86..5c7d36d0 100644 --- a/fast_api/routes/embedding.py +++ b/fast_api/routes/embedding.py @@ -2,7 +2,6 @@ from fast_api.models import CreateEmbeddingBody, UpdateEmbeddingBody from fast_api.routes.client_response import pack_json_result -from controller.misc import manager as misc from fastapi import APIRouter, Body, Depends, Request from controller.embedding import manager from controller.task_master import manager as task_master_manager @@ -30,8 +29,7 @@ def get_embedding_platforms(): @router.get("/recommended-encoders") def data_slices(request: Request, project_id: Optional[str] = None) -> List: - is_managed = misc.check_is_managed() - data = manager.get_recommended_encoders(is_managed) + data = manager.get_recommended_encoders() for v in data: v["applicability"] = json.dumps(v["applicability"]) return pack_json_result({"data": {"recommendedEncoders": data}}) diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index d503c25f..54adf984 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,7 +1,6 @@ import json from fastapi import APIRouter, Body, Request, status from fastapi.responses import PlainTextResponse -from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, ModelProviderDeleteModelBody, @@ -13,7 +12,6 @@ from typing import Dict, Optional from controller.auth import manager as auth from controller.misc import manager -from controller.misc import manager as misc from controller.monitor import manager as controller_manager from controller.model_provider import manager as model_provider_manager from controller.task_master import manager as task_master_manager @@ -37,16 +35,6 @@ def get_is_admin(request: Request) -> Dict: return pack_json_result({"data": {"isAdmin": data}}) -@router.get("/is-demo") -def get_is_demo(request: Request) -> Dict: - is_demo = False - try: - auth.check_demo_access(request.state.info) - except Exception: - is_demo = True - return pack_json_result({"data": {"isDemo": is_demo}}) - - @router.get("/version-overview") def get_version_overview(request: Request) -> Dict: data = manager.get_version_overview() @@ -63,11 +51,8 @@ def has_updates(request: Request) -> Dict: def model_provider_delete_model( request: Request, body: ModelProviderDeleteModelBody = Body(...) ): - if misc.check_is_managed(): - if not auth.check_is_single_organization(): - auth.check_admin_access(request.state.info) - else: - raise ProjectAccessError("Not allowed in open source version.") + if not auth.check_is_single_organization(): + auth.check_admin_access(request.state.info) model_provider_manager.model_provider_delete_model(body.model_name) return pack_json_result({"data": {"modelProviderDeleteModel": {"ok": True}}}) @@ -77,11 +62,8 @@ def model_provider_delete_model( def model_provider_download_model( request: Request, body: ModelProviderDownloadModelBody = Body(...) ): - if misc.check_is_managed(): - if not auth.check_is_single_organization(): - auth.check_admin_access(request.state.info) - else: - raise ProjectAccessError("Not allowed in open source version.") + if not auth.check_is_single_organization(): + auth.check_admin_access(request.state.info) model_provider_manager.model_provider_download_model(body.model_name) return pack_json_result({"data": {"modelProviderDownloadModel": {"ok": True}}}) diff --git a/fast_api/routes/organization.py b/fast_api/routes/organization.py index 277429b0..8314d8e2 100644 --- a/fast_api/routes/organization.py +++ b/fast_api/routes/organization.py @@ -1,6 +1,5 @@ import json -from fastapi import APIRouter, Depends, Request, Body -from controller.misc import config_service +from fastapi import APIRouter, Request, Body from fast_api.models import ( AddUserToOrganizationBody, ArchiveAdminMessageBody, @@ -12,7 +11,6 @@ DeleteUserBody, MappedSortedPaginatedUsers, RemoveUserToOrganizationBody, - UpdateConfigBody, UserLanguageDisplay, ) from controller.auth import manager as auth_manager @@ -24,10 +22,8 @@ from controller.admin_message import manager as admin_message_manager from controller.organization import manager as organization_manager from controller.user import manager as user_manager -from controller.misc import manager as misc from fast_api.routes.client_response import get_silent_success, pack_json_result -from submodules.model import events from submodules.model.business_objects import organization from submodules.model.util import sql_alchemy_to_dict from util import notification @@ -138,19 +134,9 @@ def all_admin_messages(request: Request, limit: int = 100) -> str: return pack_json_result({"data": {"allAdminMessages": data_dict}}) -@router.get("/can-create-local-org") -def can_create_local_org(request: Request): - data = manager.can_create_local() - return pack_json_result({"data": {"canCreateLocalOrg": data}}) - - @router.post("/create-organization") def create_organization(request: Request, body: CreateOrganizationBody = Body(...)): - if config_service.get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) - else: - if not organization_manager.can_create_local(): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) organization = organization_manager.create_organization(body.name) return {"data": {"createOrganization": {"organization": organization}}} @@ -159,11 +145,7 @@ def create_organization(request: Request, body: CreateOrganizationBody = Body(.. def add_user_to_organization( request: Request, body: AddUserToOrganizationBody = Body(...) ): - if config_service.get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) - else: - if not organization_manager.can_create_local(False): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) user_manager.update_organization_of_user(body.organization_name, body.user_mail) return pack_json_result({"data": {"addUserToOrganization": {"ok": True}}}) @@ -179,32 +161,11 @@ def remove_user_from_organization( @router.post("/change-organization") def change_organization(request: Request, body: ChangeOrganizationBody = Body(...)): - if config_service.get_config_value("is_managed"): - auth_manager.check_admin_access(request.state.info) + auth_manager.check_admin_access(request.state.info) organization_manager.change_organization(body.org_id, json.loads(body.changes)) return pack_json_result({"data": {"changeOrganization": {"ok": True}}}) -@router.post("/update-config") -def update_config(request: Request, body: UpdateConfigBody = Body(...)): - if misc.check_is_managed(): - print( - "config should only be changed for open source/local version to prevent limit issues" - ) - return - misc.update_config(body.dict_str) - misc.refresh_config() - orgs = organization.get_all() - if not orgs or len(orgs) != 1: - print("local version should only have one organization") - return - - for org in orgs: - # send to all so all are notified about the change - notification.send_organization_update(None, "config_updated", True, str(org.id)) - return pack_json_result({"data": {"updateConfig": {"ok": True}}}) - - @router.get("/user-roles") def get_user_roles(request: Request): auth_manager.check_admin_access(request.state.info) diff --git a/fast_api/routes/project.py b/fast_api/routes/project.py index ad1a5ccd..3b0425ce 100644 --- a/fast_api/routes/project.py +++ b/fast_api/routes/project.py @@ -31,8 +31,6 @@ to_frontend_obj_raw, ) from util import notification -from controller.misc import manager as misc -from exceptions.exceptions import NotAllowedInOpenSourceError from submodules.model.business_objects import notification as notification_model from submodules.model.business_objects import tokenization, task_queue @@ -292,9 +290,6 @@ def record_export_by_project_id(project_id: str) -> str: @router.get("/model-provider-info") def get_model_provider_info(request: Request) -> Dict: - if not misc.check_is_managed(): - raise NotAllowedInOpenSourceError - data = model_manager.get_model_provider_info() return pack_json_result({"data": {"modelProviderInfo": data}}) diff --git a/middleware/database_session.py b/middleware/database_session.py index 8b44548a..cb3f52dc 100644 --- a/middleware/database_session.py +++ b/middleware/database_session.py @@ -1,10 +1,5 @@ import logging from fastapi import Request -from fastapi.responses import JSONResponse -from exceptions.exceptions import ( - DatabaseSessionError, - NotAllowedInDemoError, -) from fast_api.routes.fastapi_resolve_info import FastAPIResolveInfo from middleware.query_mapping import path_query_map from submodules.model.business_objects import general @@ -26,11 +21,6 @@ async def handle_db_session(request: Request, call_next): request.state.info = info request.state.parsed = {} - if request.url.hostname != "localhost" or request.url.port != 7051: - access_response = _check_access(request, info) - if access_response is not None: - return access_response - log_request = auth_manager.extract_state_info(request, "log_request") length = request.headers.get("content-length") @@ -50,33 +40,6 @@ async def handle_db_session(request: Request, call_next): general.remove_and_refresh_session(session_token) -def _check_access(request, info): - try: - auth_manager.check_demo_access(info) - except NotAllowedInDemoError: - return JSONResponse( - status_code=401, - content={"message": "Unauthorized access"}, - ) - except DatabaseSessionError as e: - return JSONResponse( - status_code=400, - content={"message": e.message}, - ) - except ValueError as e: - return JSONResponse( - status_code=400, - content={"message": str(e)}, - ) - except Exception: - return JSONResponse( - status_code=500, - content={"message": "Internal server error"}, - ) - - return None - - def _prepare_info(request): field_name = None parent_type = None diff --git a/notify_handler.py b/notify_handler.py new file mode 100644 index 00000000..64064f2e --- /dev/null +++ b/notify_handler.py @@ -0,0 +1,25 @@ +from typing import Dict + +import requests +import traceback + +from submodules.model import daemon + + +def notify_others_about_change(notify: Dict[str, str]) -> None: + + for key in notify: + url = f"{notify[key]}/config_changed" + try: + response = requests.put(url) + if response.status_code != 200: + print(f"couldn't notify - code:{response.status_code}", key, flush=True) + except requests.exceptions.ConnectionError: + print("couldn't notify", key, flush=True) + except Exception: + print(traceback.format_exc(), flush=True) + + +def notify_others_about_change_thread(notify: Dict[str, str]) -> None: + # needs to be threaded to work with circular requests (this notifies about changes -> service requests full config from this) + daemon.run_without_db_token(notify_others_about_change, notify) diff --git a/start b/start index 6bda6bf9..c1d531d6 100755 --- a/start +++ b/start @@ -1,8 +1,6 @@ #!/bin/bash trap "echo -ne '\nstopping container...' && docker stop refinery-gateway > /dev/null 2>&1 && echo -ne '\t\t [done]\n'" EXIT -source check_config_service - HOST_IP=$(docker network inspect bridge --format='{{json .IPAM.Config}}' | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}' | tail -1) echo -ne 'stopping old container...' @@ -30,9 +28,25 @@ then fi fi +MINIO_ENDPOINT="http://$HOST_IP:7053" + INFERENCE_DIR=${DEV_SETUP_DIR}inference/ LOG_DIR=${DEV_SETUP_DIR}logs/ +CONFIG_DIR=${PWD%/*}/dev-setup/config/ +if [ ! -d "$CONFIG_DIR" ] +then + CONFIG_DIR=${PWD%/*/*}/dev-setup/config/ + if [ ! -d "$CONFIG_DIR" ] + then + # to include volume for local development, use the dev-setup inference folder: + # alternative use manual logic with + # -v /path/to/dev-setup/config:/config \ + echo "Can't find config directory: $CONFIG_DIR -> stopping" + exit 1 + fi +fi + echo -ne 'building container...' docker build -t graphql-dev -f dev.Dockerfile . > /dev/null 2>&1 echo -ne '\t\t [done]\n' @@ -73,8 +87,10 @@ docker run -d --rm \ -e INFERENCE_DIR=$INFERENCE_DIR \ -e SECRET_KEY=default \ -e POSTGRES_POOL_USE_LIFO=x \ +-e KERN_S3_ENDPOINT=${MINIO_ENDPOINT} \ -v "$INFERENCE_DIR":/inference \ -v "$LOG_DIR":/logs \ +-v "$CONFIG_DIR":/config \ --mount type=bind,source="$(pwd)"/,target=/app \ -v /var/run/docker.sock:/var/run/docker.sock \ --network dev-setup_default \