From 221b6d1743c89ca086ae7d6989de9b69a9f3b923 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Tue, 28 Nov 2023 12:06:12 +0200 Subject: [PATCH 01/10] Adding utility functions, some just skeletons --- app/utils/key_handler.py | 88 ++++++++++++++++++++++++++++++++++++ app/utils/logging_handler.py | 84 ++++++++++++++++++++++++++++++++++ app/utils/redis_updater.py | 65 ++++++++++++++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 app/utils/key_handler.py create mode 100644 app/utils/logging_handler.py create mode 100644 app/utils/redis_updater.py diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py new file mode 100644 index 0000000..48b7a2e --- /dev/null +++ b/app/utils/key_handler.py @@ -0,0 +1,88 @@ +import redis +import pymongo +import secrets +import string +import os +import urllib + +# Needs to be escaped if necessary +mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) +mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) + +# Set up required endpoints. The +redis_client = redis.StrictRedis(host="redis", port=6379, db=0) +mongo_client = pymongo.MongoClient( + "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) +) +db = mongo_client["gateway"] +keyCollection = db["apikeys"] +# Make sure, that key is an index (avoids duplicates); +indices = keyCollection.create_index("key") +userCollection = db["users"] + + +def generate_api_key(length=64): + """ + Function to generate an API key. + + Parameters: + - length (int, optional): Length of the generated API key. Defaults to 64. + + Returns: + - str: The generated API key. + """ + alphabet = string.ascii_letters + string.digits + api_key = "".join(secrets.choice(alphabet) for _ in range(length)) + return api_key + + +def build_new_key_object(key, name): + """ + Function to create a new key object. + + Parameters: + - key (str): The key value. + - name (str): The name associated with the key. + + Returns: + - dict: A dictionary representing the key object with "active" status, key, and name. + """ + return {"active": True, "key": key, "name": name} + + +def check_key(key): + """ + Function to check if a key currently exists + + Parameters: + - key (str): The key to check. + + Returns: + - bool: True if the key exists + """ + return redis_client.sismember("keys", key) + + +def create_key(user, name): + """ + Function to create a new key associated with a user. + + Parameters: + - user (str): The username for which the key is created. + - name (str): The name associated with the key. + + Returns: + - str: The generated API key. + """ + keyCreated = False + api_key = "" + while not keyCreated: + api_key = generate_api_key() + found = keyCollection.find_one({"key": api_key}) + if found == None: + keyCollection.insert_one(build_new_key_object(api_key, name)) + userCollection.find_one_and_update( + {"username": user}, {"$push", {"keys": api_key}} + ) + keyCreated = True + return api_key diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py new file mode 100644 index 0000000..b39ea7f --- /dev/null +++ b/app/utils/logging_handler.py @@ -0,0 +1,84 @@ +import pymongo +from datetime import datetime +import os +import urllib + +# Needs to be escaped if necessary +mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) +mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) + +# Set up required endpoints. +mongo_client = pymongo.MongoClient( + "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) +) +db = mongo_client["gateway"] +logCollection = db["logs"] +userCollection = db["users"] + + +def create_log_entry(tokencount, model, source, sourcetype="apikey"): + """ + Function to create a log entry. + + Parameters: + - tokencount (int): The count of tokens. + - model (str): The model related to the log entry. + - source (str): The source of the log entry. + - sourcetype (str): The type of source.. + + Returns: + - dict: A dictionary representing the log entry with timestamp. + """ + return { + "tokencount": tokencount, + "model": model, + "source": source, + "sourcetype": sourcetype, + "timestamp": datetime.utcnow(), # Current timestamp in UTC + } + + +def log_usage_for_key(tokencount, model, key): + """ + Function to log usage for a specific key. + + Parameters: + - tokencount (int): The count of tokens used. + - model (str): The model associated with the usage. + - key (str): The key for which the usage is logged. + """ + logEntry = create_log_entry(tokencount, model, key) + logCollection.insert_one(logEntry) + + +def log_usage_for_user(tokencount, model, user): + """ + Function to log usage for a specific user. + + Parameters: + - tokencount (int): The count of tokens used. + - model (str): The model associated with the usage. + - user (str): The user for which the usage is logged. + """ + logEntry = create_log_entry(tokencount, model, user, "user") + logCollection.insert_one(logEntry) + + +def get_usage_for_user(username): + # Needs to be implemented + pass + + +def get_usage_for_key(key): + # TODO: needs to be implemented + pass + + +def get_usage_for_model(model): + # TODO: needs to be implemented + pass + + +def get_usage_for_timerange(start, end): + # TODO: needs to be implemented + pass diff --git a/app/utils/redis_updater.py b/app/utils/redis_updater.py new file mode 100644 index 0000000..1fbec4a --- /dev/null +++ b/app/utils/redis_updater.py @@ -0,0 +1,65 @@ +import redis +import pymongo +import schedule +import time +import urllib +import os + +# Needs to be escaped if necessary +mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) +mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) + + +class RedisUpdater: + def __init__(self): + """ + Initializes Redis and MongoDB connections. + """ + # Redis connection + self.redis_client = redis.StrictRedis(host="redis", port=6379, db=0) + self.entries = [] # Placeholder for entries fetched from MongoDB + + # MongoDB connection + self.mongo_client = mongo_client = pymongo.MongoClient( + "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) + ) + self.db = mongo_client["gateway"] + self.keyCollection = db["apikeys"] + + def update_redis(self): + """ + Updates Redis with API keys fetched from MongoDB. + """ + # Fetch entries from MongoDB + self.fetch_entries_from_mongodb() + + # Update Redis with fetched entries + self.redis_client.rpush("keys", *self.entries) + + def fetch_entries_from_mongodb(self): + """ + Retrieves active API keys from MongoDB and stores them in self.entries. + """ + # Retrieve entries from MongoDB + currentKeys = self.collection.find({"active": True}) + self.entries = [ + entry["APIKEY"] for entry in self.collection.find({}, {"key": 1}) + ] + + def start_scheduler(self): + """ + Initiates a scheduler to run update_redis every 15 minutes continuously. + """ + # Schedule the update every 15 minutes + schedule.every(15).minutes.do(self.update_redis) + + # Run the scheduler continuously + while True: + schedule.run_pending() + # Only do this every 60 seconds - thats often enough. + time.sleep(60) + + +# Create an instance of the RedisUpdater and start the scheduler +updater = RedisUpdater() +updater.start_scheduler() From 3618d6bd8cc218507bb76b65202ba080da1bff07 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Tue, 28 Nov 2023 14:34:01 +0200 Subject: [PATCH 02/10] Adding first test --- app/utils/key_handler.py | 167 ++++++++++++++++++---------------- app/utils/redis_updater.py | 9 +- app/utils/test_key_handler.py | 16 ++++ environment_dev.yml | 13 +++ 4 files changed, 122 insertions(+), 83 deletions(-) create mode 100644 app/utils/test_key_handler.py create mode 100644 environment_dev.yml diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index 48b7a2e..8d41cd8 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -5,84 +5,91 @@ import os import urllib -# Needs to be escaped if necessary -mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) -mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) - -# Set up required endpoints. The -redis_client = redis.StrictRedis(host="redis", port=6379, db=0) -mongo_client = pymongo.MongoClient( - "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) -) -db = mongo_client["gateway"] -keyCollection = db["apikeys"] -# Make sure, that key is an index (avoids duplicates); -indices = keyCollection.create_index("key") -userCollection = db["users"] - - -def generate_api_key(length=64): - """ - Function to generate an API key. - - Parameters: - - length (int, optional): Length of the generated API key. Defaults to 64. - - Returns: - - str: The generated API key. - """ - alphabet = string.ascii_letters + string.digits - api_key = "".join(secrets.choice(alphabet) for _ in range(length)) - return api_key - - -def build_new_key_object(key, name): - """ - Function to create a new key object. - - Parameters: - - key (str): The key value. - - name (str): The name associated with the key. - - Returns: - - dict: A dictionary representing the key object with "active" status, key, and name. - """ - return {"active": True, "key": key, "name": name} - - -def check_key(key): - """ - Function to check if a key currently exists - - Parameters: - - key (str): The key to check. - - Returns: - - bool: True if the key exists - """ - return redis_client.sismember("keys", key) - - -def create_key(user, name): - """ - Function to create a new key associated with a user. - - Parameters: - - user (str): The username for which the key is created. - - name (str): The name associated with the key. - - Returns: - - str: The generated API key. - """ - keyCreated = False - api_key = "" - while not keyCreated: - api_key = generate_api_key() - found = keyCollection.find_one({"key": api_key}) - if found == None: - keyCollection.insert_one(build_new_key_object(api_key, name)) - userCollection.find_one_and_update( - {"username": user}, {"$push", {"keys": api_key}} + +class key_handler: + def __init__(self, testing=False): + if not testing: + # Needs to be escaped if necessary + mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) + mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) + + # Set up required endpoints. The + mongo_client = pymongo.MongoClient( + "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) ) - keyCreated = True - return api_key + redis_client = redis.StrictRedis(host="redis", port=6379, db=0) + self.setup(mongo_client, redis_client) + self.keyCollection.create_index("key") + + def setup(self, mongo_client, redis_client): + self.redis_client = redis_client + self.mongo_client = mongo_client + self.db = mongo_client["gateway"] + self.keyCollection = self.db["apikeys"] + # Make sure, that key is an index (avoids duplicates); + self.userCollection = self.db["users"] + + def generate_api_key(self, length=64): + """ + Function to generate an API key. + + Parameters: + - length (int, optional): Length of the generated API key. Defaults to 64. + + Returns: + - str: The generated API key. + """ + alphabet = string.ascii_letters + string.digits + api_key = "".join(secrets.choice(alphabet) for _ in range(length)) + return api_key + + def build_new_key_object(self, key, name): + """ + Function to create a new key object. + + Parameters: + - key (str): The key value. + - name (str): The name associated with the key. + + Returns: + - dict: A dictionary representing the key object with "active" status, key, and name. + """ + return {"active": True, "key": key, "name": name} + + def check_key(self, key): + """ + Function to check if a key currently exists + + Parameters: + - key (str): The key to check. + + Returns: + - bool: True if the key exists + """ + return self.redis_client.sismember("keys", key) + + def create_key(self, user, name): + """ + Generates a unique API key and associates it with a specified user. + + Args: + - user: Username of the user to whom the API key will be associated. + - name: Name or label for the API key. + - userCollection: Collection object representing the 'users' collection in MongoDB (default: global variable). + - keyCollection: Collection object representing the 'apikeys' collection in MongoDB (default: global variable). + + Returns: + - api_key: The generated unique API key associated with the user. + """ + keyCreated = False + api_key = "" + while not keyCreated: + api_key = self.generate_api_key() + found = self.keyCollection.find_one({"key": api_key}) + if found == None: + self.keyCollection.insert_one(self.build_new_key_object(api_key, name)) + self.userCollection.find_one_and_update( + {"username": user}, {"$push", {"keys": api_key}} + ) + keyCreated = True + return api_key diff --git a/app/utils/redis_updater.py b/app/utils/redis_updater.py index 1fbec4a..989713e 100644 --- a/app/utils/redis_updater.py +++ b/app/utils/redis_updater.py @@ -24,7 +24,7 @@ def __init__(self): "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) ) self.db = mongo_client["gateway"] - self.keyCollection = db["apikeys"] + self.keyCollection = self.db["apikeys"] def update_redis(self): """ @@ -32,9 +32,12 @@ def update_redis(self): """ # Fetch entries from MongoDB self.fetch_entries_from_mongodb() - + # Clean up the current keys, and then add the new ones. + # optimally, this would be done in an atomic call, but we will have to + # see how often someone + self.redis_client.delete("keys") # Update Redis with fetched entries - self.redis_client.rpush("keys", *self.entries) + self.redis_client.sadd("keys", *self.entries) def fetch_entries_from_mongodb(self): """ diff --git a/app/utils/test_key_handler.py b/app/utils/test_key_handler.py new file mode 100644 index 0000000..8b455ce --- /dev/null +++ b/app/utils/test_key_handler.py @@ -0,0 +1,16 @@ +from pytest_mock_resources import create_redis_fixture +from pytest_mock_resources import create_mongo_fixture +from key_handler import key_handler + +redis = create_redis_fixture() +mongo = create_mongo_fixture() + + +def test_check_key(redis, mongo): + handler = key_handler(True) + handler.setup(mongo, redis) + keys = ["ABC", "DEF"] + redis.sadd("keys", *keys) + assert handler.check_key("ABC") == True + assert handler.check_key("DEF") == True + assert handler.check_key("BCE") == False diff --git a/environment_dev.yml b/environment_dev.yml new file mode 100644 index 0000000..35cd3ba --- /dev/null +++ b/environment_dev.yml @@ -0,0 +1,13 @@ +name: LLMGateWay-Dev +channels: + - conda-forge +dependencies: + - fastapi + - redis-py + - pymongo + - schedule + - pytest + - pip + - pip: + - pytest-mock-resources[mongo] + - pytest-mock-resources[redis] From b872e18dc6645018ff4583159f16af0f814f6628 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Tue, 28 Nov 2023 14:44:16 +0200 Subject: [PATCH 03/10] Making a class out of logging handler and adding vscode stuff to gitignore --- .gitignore | 3 + app/utils/logging_handler.py | 154 +++++++++++++++++------------------ 2 files changed, 79 insertions(+), 78 deletions(-) diff --git a/.gitignore b/.gitignore index 9da8332..1822f67 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,6 @@ bundles/ vendor/pkg/ pyenv Vagrantfile + +# vscode stuff +.vscode/**/* \ No newline at end of file diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index b39ea7f..db703d5 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -3,82 +3,80 @@ import os import urllib -# Needs to be escaped if necessary -mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) -mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) - -# Set up required endpoints. -mongo_client = pymongo.MongoClient( - "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) -) -db = mongo_client["gateway"] -logCollection = db["logs"] -userCollection = db["users"] - - -def create_log_entry(tokencount, model, source, sourcetype="apikey"): - """ - Function to create a log entry. - - Parameters: - - tokencount (int): The count of tokens. - - model (str): The model related to the log entry. - - source (str): The source of the log entry. - - sourcetype (str): The type of source.. - - Returns: - - dict: A dictionary representing the log entry with timestamp. - """ - return { - "tokencount": tokencount, - "model": model, - "source": source, - "sourcetype": sourcetype, - "timestamp": datetime.utcnow(), # Current timestamp in UTC - } - - -def log_usage_for_key(tokencount, model, key): - """ - Function to log usage for a specific key. - - Parameters: - - tokencount (int): The count of tokens used. - - model (str): The model associated with the usage. - - key (str): The key for which the usage is logged. - """ - logEntry = create_log_entry(tokencount, model, key) - logCollection.insert_one(logEntry) - -def log_usage_for_user(tokencount, model, user): - """ - Function to log usage for a specific user. - - Parameters: - - tokencount (int): The count of tokens used. - - model (str): The model associated with the usage. - - user (str): The user for which the usage is logged. - """ - logEntry = create_log_entry(tokencount, model, user, "user") - logCollection.insert_one(logEntry) - - -def get_usage_for_user(username): - # Needs to be implemented - pass - - -def get_usage_for_key(key): - # TODO: needs to be implemented - pass - - -def get_usage_for_model(model): - # TODO: needs to be implemented - pass - - -def get_usage_for_timerange(start, end): - # TODO: needs to be implemented - pass +# Needs to be escaped if necessary +class logging_handler: + def __init__(self, testing=False): + if not testing: + mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) + mongo_password = urllib.parse.quote_plus(os.environ.get("MONGOPASSWORD")) + # Set up required endpoints. + mongo_client = pymongo.MongoClient( + "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) + ) + + def setup(self, mongo_client): + self.db = mongo_client["gateway"] + self.logCollection = self.db["logs"] + self.userCollection = self.db["users"] + + def create_log_entry(self, tokencount, model, source, sourcetype="apikey"): + """ + Function to create a log entry. + + Parameters: + - tokencount (int): The count of tokens. + - model (str): The model related to the log entry. + - source (str): The source of the log entry. + - sourcetype (str): The type of source.. + + Returns: + - dict: A dictionary representing the log entry with timestamp. + """ + return { + "tokencount": tokencount, + "model": model, + "source": source, + "sourcetype": sourcetype, + "timestamp": datetime.utcnow(), # Current timestamp in UTC + } + + def log_usage_for_key(self, tokencount, model, key): + """ + Function to log usage for a specific key. + + Parameters: + - tokencount (int): The count of tokens used. + - model (str): The model associated with the usage. + - key (str): The key for which the usage is logged. + """ + logEntry = self.create_log_entry(tokencount, model, key) + self.logCollection.insert_one(logEntry) + + def log_usage_for_user(self, tokencount, model, user): + """ + Function to log usage for a specific user. + + Parameters: + - tokencount (int): The count of tokens used. + - model (str): The model associated with the usage. + - user (str): The user for which the usage is logged. + """ + logEntry = self.create_log_entry(tokencount, model, user, "user") + self.logCollection.insert_one(logEntry) + + def get_usage_for_user(self, username): + # Needs to be implemented + pass + + def get_usage_for_key(self, key): + # TODO: needs to be implemented + pass + + def get_usage_for_model(self, model): + # TODO: needs to be implemented + pass + + def get_usage_for_timerange(self, start, end): + # TODO: needs to be implemented + pass From d67289923cc3221a5fc73eb92e87c88f83bd8a1c Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 09:19:01 +0200 Subject: [PATCH 04/10] Adding tests and making update instantaneous --- app/utils/key_handler.py | 65 ++++++++++++++++++++++++++----- app/utils/logging_handler.py | 2 +- app/utils/test_key_handler.py | 61 +++++++++++++++++++++++++++++ app/utils/test_logging_handler.py | 42 ++++++++++++++++++++ 4 files changed, 160 insertions(+), 10 deletions(-) create mode 100644 app/utils/test_logging_handler.py diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index 8d41cd8..e2bbcc4 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -7,7 +7,7 @@ class key_handler: - def __init__(self, testing=False): + def __init__(self, testing: bool = False): if not testing: # Needs to be escaped if necessary mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) @@ -19,17 +19,24 @@ def __init__(self, testing=False): ) redis_client = redis.StrictRedis(host="redis", port=6379, db=0) self.setup(mongo_client, redis_client) - self.keyCollection.create_index("key") - def setup(self, mongo_client, redis_client): + def setup(self, mongo_client: pymongo.MongoClient, redis_client: redis.Redis): self.redis_client = redis_client self.mongo_client = mongo_client self.db = mongo_client["gateway"] self.keyCollection = self.db["apikeys"] + keyindices = self.keyCollection.index_information() # Make sure, that key is an index (avoids duplicates); + if not "key" in keyindices: + self.keyCollection.create_index("key") self.userCollection = self.db["users"] + # Make sure, that username is an index (avoids duplicates when creating keys, which automatically adds a user if necessary); + userindices = self.userCollection.index_information() + if not "username" in userindices: + self.userCollection.create_index("username") + self.userCollection.find - def generate_api_key(self, length=64): + def generate_api_key(self, length: int = 64): """ Function to generate an API key. @@ -43,7 +50,7 @@ def generate_api_key(self, length=64): api_key = "".join(secrets.choice(alphabet) for _ in range(length)) return api_key - def build_new_key_object(self, key, name): + def build_new_key_object(self, key: string, name: string): """ Function to create a new key object. @@ -56,7 +63,7 @@ def build_new_key_object(self, key, name): """ return {"active": True, "key": key, "name": name} - def check_key(self, key): + def check_key(self, key: string): """ Function to check if a key currently exists @@ -68,7 +75,46 @@ def check_key(self, key): """ return self.redis_client.sismember("keys", key) - def create_key(self, user, name): + def delete_key(self, key: string, user: string): + """ + Function to delete an existing key + + Parameters: + - key (str): The key to check. + - user (str): The user that requests this deletion + + """ + updated_user = self.userCollection.find_one_and_update( + {"username": user, "keys": {"$elemMatch": {"$eq": key}}}, + {"$pull": {"keys": key}}, + ) + if not updated_user == None: + # We found, and updated the user, so we can remove the key + # removal should be instantaneous + self.keyCollection.delete_one({"key": key}) + self.redis_client.srem("keys", key) + + def set_key_activity(self, key: string, user: string, active: bool): + """ + Function to set the activity of a key + + Parameters: + - key (str): The key to check. + - user (str): The user that requests this deletion + - active (bool): whether to activate or deactivate the key + """ + user_has_key = self.userCollection.find_one( + {"username": user, "keys": {"$elemMatch": {"$eq": key}}} + ) + if not user_has_key == None: + # the requesting user has access to this key + self.keyCollection.update_one({"key": key}, {"$set": {"active": active}}) + if active: + self.redis_client.sadd("keys", key) + else: + self.redis_client.srem("keys", key) + + def create_key(self, user: string, name: string): """ Generates a unique API key and associates it with a specified user. @@ -88,8 +134,9 @@ def create_key(self, user, name): found = self.keyCollection.find_one({"key": api_key}) if found == None: self.keyCollection.insert_one(self.build_new_key_object(api_key, name)) - self.userCollection.find_one_and_update( - {"username": user}, {"$push", {"keys": api_key}} + self.userCollection.update_one( + {"username": user}, {"$addToSet": {"keys": api_key}}, upsert=True ) + self.redis_client.sadd("keys", api_key) keyCreated = True return api_key diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index db703d5..1ed996e 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -28,7 +28,7 @@ def create_log_entry(self, tokencount, model, source, sourcetype="apikey"): - tokencount (int): The count of tokens. - model (str): The model related to the log entry. - source (str): The source of the log entry. - - sourcetype (str): The type of source.. + - sourcetype (str): The type of source either apikey or user.. Returns: - dict: A dictionary representing the log entry with timestamp. diff --git a/app/utils/test_key_handler.py b/app/utils/test_key_handler.py index 8b455ce..6252844 100644 --- a/app/utils/test_key_handler.py +++ b/app/utils/test_key_handler.py @@ -1,6 +1,8 @@ from pytest_mock_resources import create_redis_fixture from pytest_mock_resources import create_mongo_fixture from key_handler import key_handler +import redis +import pymongo redis = create_redis_fixture() mongo = create_mongo_fixture() @@ -14,3 +16,62 @@ def test_check_key(redis, mongo): assert handler.check_key("ABC") == True assert handler.check_key("DEF") == True assert handler.check_key("BCE") == False + + +def test_create_key(redis, mongo): + handler = key_handler(True) + handler.setup(mongo, redis) + newKey = handler.create_key("NewUser", "NewKey") + db = mongo["gateway"] + userCollection = db["users"] + keyCollection = db["apikeys"] + assert userCollection.count_documents({}) == 1 + user = userCollection.find_one({}) + assert user["username"] == "NewUser" + assert len(user["keys"]) == 1 + assert keyCollection.count_documents({}) == 1 + assert handler.check_key(newKey) == True + + +def test_delete_key(redis, mongo): + handler = key_handler(True) + handler.setup(mongo, redis) + newKey = handler.create_key("NewUser", "NewKey") + db = mongo["gateway"] + userCollection = db["users"] + keyCollection = db["apikeys"] + assert userCollection.count_documents({}) == 1 + assert keyCollection.count_documents({}) == 1 + assert handler.check_key(newKey) == True + handler.delete_key(newKey, "NewUser") + user = userCollection.find_one({}) + assert user["username"] == "NewUser" + assert len(user["keys"]) == 0 + assert keyCollection.count_documents({}) == 0 + assert handler.check_key(newKey) == False + + +def test_set_key_activity(redis, mongo): + handler = key_handler(True) + handler.setup(mongo, redis) + # Create key + newKey = handler.create_key("NewUser", "NewKey") + db = mongo["gateway"] + userCollection = db["users"] + keyCollection = db["apikeys"] + # Make sure key is valid at start + assert handler.check_key(newKey) == True + # deactivate key + handler.set_key_activity(newKey, "NewUser", False) + key_data = keyCollection.find_one({"key": newKey}) + assert key_data["active"] == False + # Ensure key still exists + user = userCollection.find_one({"username": "NewUser"}) + assert len(user["keys"]) == 1 + # and key is inactive + assert handler.check_key(newKey) == False + # reactivate and test that the key is now active again + handler.set_key_activity(newKey, "NewUser", True) + key_data = keyCollection.find_one({"key": newKey}) + assert key_data["active"] == True + assert handler.check_key(newKey) == True diff --git a/app/utils/test_logging_handler.py b/app/utils/test_logging_handler.py new file mode 100644 index 0000000..fd570e7 --- /dev/null +++ b/app/utils/test_logging_handler.py @@ -0,0 +1,42 @@ +from pytest_mock_resources import create_mongo_fixture +from logging_handler import logging_handler + +mongo = create_mongo_fixture() + + +def test_log_usage_for_key(mongo): + handler = logging_handler(True) + handler.setup(mongo) + db = mongo["gateway"] + logCollection = db["logs"] + handler.log_usage_for_key(50, "newModel", "123") + assert logCollection.count_documents({}) == 1 + handler.log_usage_for_key(70, "model2", "123") + assert logCollection.count_documents({}) == 2 + handler.log_usage_for_key(50, "model2", "321") + assert logCollection.count_documents({"model": "model2"}) == 2 + assert logCollection.count_documents({"source": "123"}) == 2 + assert logCollection.count_documents({"source": "321"}) == 1 + assert logCollection.count_documents({"sourcetype": "apikey"}) == 3 + firstLog = logCollection.find_one({"model": "newModel"}) + secondLog = logCollection.find_one({"tokencount": 70}) + assert firstLog["timestamp"] <= secondLog["timestamp"] + + +def test_log_usage_for_user(mongo): + handler = logging_handler(True) + handler.setup(mongo) + db = mongo["gateway"] + logCollection = db["logs"] + handler.log_usage_for_user(50, "newModel", "123") + assert logCollection.count_documents({}) == 1 + handler.log_usage_for_user(70, "model2", "123") + assert logCollection.count_documents({}) == 2 + handler.log_usage_for_user(50, "model2", "321") + assert logCollection.count_documents({"model": "model2"}) == 2 + assert logCollection.count_documents({"source": "123"}) == 2 + assert logCollection.count_documents({"source": "321"}) == 1 + assert logCollection.count_documents({"sourcetype": "user"}) == 3 + firstLog = logCollection.find_one({"model": "newModel"}) + secondLog = logCollection.find_one({"tokencount": 70}) + assert firstLog["timestamp"] <= secondLog["timestamp"] From ed80119a4c313e36c80f7a0f38cd7d5a68658009 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 09:23:51 +0200 Subject: [PATCH 05/10] Mistake in non testing setup... --- app/utils/logging_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index 1ed996e..b55a21e 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -14,6 +14,7 @@ def __init__(self, testing=False): mongo_client = pymongo.MongoClient( "mongodb://%s:%s@mongo:27017/" % (mongo_user, mongo_password) ) + self.setup(mongo_client) def setup(self, mongo_client): self.db = mongo_client["gateway"] From a7fcf31ba1dea10d900b7c9ff82f6915f1221c2a Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 10:01:31 +0200 Subject: [PATCH 06/10] Improving comments --- app/utils/key_handler.py | 7 ++++--- app/utils/logging_handler.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index e2bbcc4..cee0f4b 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -28,12 +28,12 @@ def setup(self, mongo_client: pymongo.MongoClient, redis_client: redis.Redis): keyindices = self.keyCollection.index_information() # Make sure, that key is an index (avoids duplicates); if not "key" in keyindices: - self.keyCollection.create_index("key") + self.keyCollection.create_index("key", "unique") self.userCollection = self.db["users"] # Make sure, that username is an index (avoids duplicates when creating keys, which automatically adds a user if necessary); userindices = self.userCollection.index_information() if not "username" in userindices: - self.userCollection.create_index("username") + self.userCollection.create_index("username", "unique") self.userCollection.find def generate_api_key(self, length: int = 64): @@ -96,7 +96,8 @@ def delete_key(self, key: string, user: string): def set_key_activity(self, key: string, user: string, active: bool): """ - Function to set the activity of a key + Function to set whether a key is active or not. + The key has to be owned by the user indicated. Parameters: - key (str): The key to check. diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index b55a21e..7a9a19e 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -28,8 +28,8 @@ def create_log_entry(self, tokencount, model, source, sourcetype="apikey"): Parameters: - tokencount (int): The count of tokens. - model (str): The model related to the log entry. - - source (str): The source of the log entry. - - sourcetype (str): The type of source either apikey or user.. + - source (str): The source that authorized the request that is being logged. This could be a user name or an apikey. + - sourcetype (str): Specification of what kind of source authorized the request that is being logged (either 'apikey' or 'user'). Returns: - dict: A dictionary representing the log entry with timestamp. From 5d94404e2f5c60cb6ef6ff3fb43fb4ae0334f540 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 10:03:25 +0200 Subject: [PATCH 07/10] :bug: Removing nonsensical code --- app/utils/key_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index cee0f4b..6cfe02f 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -34,7 +34,6 @@ def setup(self, mongo_client: pymongo.MongoClient, redis_client: redis.Redis): userindices = self.userCollection.index_information() if not "username" in userindices: self.userCollection.create_index("username", "unique") - self.userCollection.find def generate_api_key(self, length: int = 64): """ From f06d8eea198fc387ceedca8df20a44f12a9cc40d Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 10:04:30 +0200 Subject: [PATCH 08/10] Correcting no implemented methods --- app/utils/logging_handler.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index 7a9a19e..1a902e6 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -67,17 +67,13 @@ def log_usage_for_user(self, tokencount, model, user): self.logCollection.insert_one(logEntry) def get_usage_for_user(self, username): - # Needs to be implemented - pass + raise NotImplementedError def get_usage_for_key(self, key): - # TODO: needs to be implemented - pass + raise NotImplementedError def get_usage_for_model(self, model): - # TODO: needs to be implemented - pass + raise NotImplementedError def get_usage_for_timerange(self, start, end): - # TODO: needs to be implemented - pass + raise NotImplementedError From 6fdb9152711dd029155fc5aec4ef0fdfe792d629 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 10:48:51 +0200 Subject: [PATCH 09/10] Updating casing --- app/utils/key_handler.py | 36 ++++++++++++------------ app/utils/logging_handler.py | 14 +++++----- app/utils/test_key_handler.py | 42 ++++++++++++++-------------- app/utils/test_logging_handler.py | 46 +++++++++++++++---------------- 4 files changed, 68 insertions(+), 70 deletions(-) diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index 6cfe02f..13441af 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -6,7 +6,7 @@ import urllib -class key_handler: +class KeyHandler: def __init__(self, testing: bool = False): if not testing: # Needs to be escaped if necessary @@ -24,16 +24,16 @@ def setup(self, mongo_client: pymongo.MongoClient, redis_client: redis.Redis): self.redis_client = redis_client self.mongo_client = mongo_client self.db = mongo_client["gateway"] - self.keyCollection = self.db["apikeys"] - keyindices = self.keyCollection.index_information() + self.key_collection = self.db["apikeys"] + keyindices = self.key_collection.index_information() # Make sure, that key is an index (avoids duplicates); if not "key" in keyindices: - self.keyCollection.create_index("key", "unique") - self.userCollection = self.db["users"] + self.key_collection.create_index("key", "unique") + self.user_collection = self.db["users"] # Make sure, that username is an index (avoids duplicates when creating keys, which automatically adds a user if necessary); - userindices = self.userCollection.index_information() + userindices = self.user_collection.index_information() if not "username" in userindices: - self.userCollection.create_index("username", "unique") + self.user_collection.create_index("username", "unique") def generate_api_key(self, length: int = 64): """ @@ -83,14 +83,14 @@ def delete_key(self, key: string, user: string): - user (str): The user that requests this deletion """ - updated_user = self.userCollection.find_one_and_update( + updated_user = self.user_collection.find_one_and_update( {"username": user, "keys": {"$elemMatch": {"$eq": key}}}, {"$pull": {"keys": key}}, ) if not updated_user == None: # We found, and updated the user, so we can remove the key # removal should be instantaneous - self.keyCollection.delete_one({"key": key}) + self.key_collection.delete_one({"key": key}) self.redis_client.srem("keys", key) def set_key_activity(self, key: string, user: string, active: bool): @@ -103,12 +103,12 @@ def set_key_activity(self, key: string, user: string, active: bool): - user (str): The user that requests this deletion - active (bool): whether to activate or deactivate the key """ - user_has_key = self.userCollection.find_one( + user_has_key = self.user_collection.find_one( {"username": user, "keys": {"$elemMatch": {"$eq": key}}} ) if not user_has_key == None: # the requesting user has access to this key - self.keyCollection.update_one({"key": key}, {"$set": {"active": active}}) + self.key_collection.update_one({"key": key}, {"$set": {"active": active}}) if active: self.redis_client.sadd("keys", key) else: @@ -121,22 +121,20 @@ def create_key(self, user: string, name: string): Args: - user: Username of the user to whom the API key will be associated. - name: Name or label for the API key. - - userCollection: Collection object representing the 'users' collection in MongoDB (default: global variable). - - keyCollection: Collection object representing the 'apikeys' collection in MongoDB (default: global variable). Returns: - api_key: The generated unique API key associated with the user. """ - keyCreated = False + key_created = False api_key = "" - while not keyCreated: + while not key_created: api_key = self.generate_api_key() - found = self.keyCollection.find_one({"key": api_key}) + found = self.key_collection.find_one({"key": api_key}) if found == None: - self.keyCollection.insert_one(self.build_new_key_object(api_key, name)) - self.userCollection.update_one( + self.key_collection.insert_one(self.build_new_key_object(api_key, name)) + self.user_collection.update_one( {"username": user}, {"$addToSet": {"keys": api_key}}, upsert=True ) self.redis_client.sadd("keys", api_key) - keyCreated = True + key_created = True return api_key diff --git a/app/utils/logging_handler.py b/app/utils/logging_handler.py index 1a902e6..3e49977 100644 --- a/app/utils/logging_handler.py +++ b/app/utils/logging_handler.py @@ -5,7 +5,7 @@ # Needs to be escaped if necessary -class logging_handler: +class LoggingHandler: def __init__(self, testing=False): if not testing: mongo_user = urllib.parse.quote_plus(os.environ.get("MONGOUSER")) @@ -18,8 +18,8 @@ def __init__(self, testing=False): def setup(self, mongo_client): self.db = mongo_client["gateway"] - self.logCollection = self.db["logs"] - self.userCollection = self.db["users"] + self.log_collection = self.db["logs"] + self.user_collection = self.db["users"] def create_log_entry(self, tokencount, model, source, sourcetype="apikey"): """ @@ -51,8 +51,8 @@ def log_usage_for_key(self, tokencount, model, key): - model (str): The model associated with the usage. - key (str): The key for which the usage is logged. """ - logEntry = self.create_log_entry(tokencount, model, key) - self.logCollection.insert_one(logEntry) + log_entry = self.create_log_entry(tokencount, model, key) + self.log_collection.insert_one(log_entry) def log_usage_for_user(self, tokencount, model, user): """ @@ -63,8 +63,8 @@ def log_usage_for_user(self, tokencount, model, user): - model (str): The model associated with the usage. - user (str): The user for which the usage is logged. """ - logEntry = self.create_log_entry(tokencount, model, user, "user") - self.logCollection.insert_one(logEntry) + log_entry = self.create_log_entry(tokencount, model, user, "user") + self.log_collection.insert_one(log_entry) def get_usage_for_user(self, username): raise NotImplementedError diff --git a/app/utils/test_key_handler.py b/app/utils/test_key_handler.py index 6252844..9b248f4 100644 --- a/app/utils/test_key_handler.py +++ b/app/utils/test_key_handler.py @@ -1,6 +1,6 @@ from pytest_mock_resources import create_redis_fixture from pytest_mock_resources import create_mongo_fixture -from key_handler import key_handler +from key_handler import KeyHandler import redis import pymongo @@ -9,7 +9,7 @@ def test_check_key(redis, mongo): - handler = key_handler(True) + handler = KeyHandler(True) handler.setup(mongo, redis) keys = ["ABC", "DEF"] redis.sadd("keys", *keys) @@ -19,59 +19,59 @@ def test_check_key(redis, mongo): def test_create_key(redis, mongo): - handler = key_handler(True) + handler = KeyHandler(True) handler.setup(mongo, redis) newKey = handler.create_key("NewUser", "NewKey") db = mongo["gateway"] - userCollection = db["users"] - keyCollection = db["apikeys"] - assert userCollection.count_documents({}) == 1 - user = userCollection.find_one({}) + user_collection = db["users"] + key_collection = db["apikeys"] + assert user_collection.count_documents({}) == 1 + user = user_collection.find_one({}) assert user["username"] == "NewUser" assert len(user["keys"]) == 1 - assert keyCollection.count_documents({}) == 1 + assert key_collection.count_documents({}) == 1 assert handler.check_key(newKey) == True def test_delete_key(redis, mongo): - handler = key_handler(True) + handler = KeyHandler(True) handler.setup(mongo, redis) newKey = handler.create_key("NewUser", "NewKey") db = mongo["gateway"] - userCollection = db["users"] - keyCollection = db["apikeys"] - assert userCollection.count_documents({}) == 1 - assert keyCollection.count_documents({}) == 1 + user_collection = db["users"] + key_collection = db["apikeys"] + assert user_collection.count_documents({}) == 1 + assert key_collection.count_documents({}) == 1 assert handler.check_key(newKey) == True handler.delete_key(newKey, "NewUser") - user = userCollection.find_one({}) + user = user_collection.find_one({}) assert user["username"] == "NewUser" assert len(user["keys"]) == 0 - assert keyCollection.count_documents({}) == 0 + assert key_collection.count_documents({}) == 0 assert handler.check_key(newKey) == False def test_set_key_activity(redis, mongo): - handler = key_handler(True) + handler = KeyHandler(True) handler.setup(mongo, redis) # Create key newKey = handler.create_key("NewUser", "NewKey") db = mongo["gateway"] - userCollection = db["users"] - keyCollection = db["apikeys"] + user_collection = db["users"] + key_collection = db["apikeys"] # Make sure key is valid at start assert handler.check_key(newKey) == True # deactivate key handler.set_key_activity(newKey, "NewUser", False) - key_data = keyCollection.find_one({"key": newKey}) + key_data = key_collection.find_one({"key": newKey}) assert key_data["active"] == False # Ensure key still exists - user = userCollection.find_one({"username": "NewUser"}) + user = user_collection.find_one({"username": "NewUser"}) assert len(user["keys"]) == 1 # and key is inactive assert handler.check_key(newKey) == False # reactivate and test that the key is now active again handler.set_key_activity(newKey, "NewUser", True) - key_data = keyCollection.find_one({"key": newKey}) + key_data = key_collection.find_one({"key": newKey}) assert key_data["active"] == True assert handler.check_key(newKey) == True diff --git a/app/utils/test_logging_handler.py b/app/utils/test_logging_handler.py index fd570e7..d2fef30 100644 --- a/app/utils/test_logging_handler.py +++ b/app/utils/test_logging_handler.py @@ -1,42 +1,42 @@ from pytest_mock_resources import create_mongo_fixture -from logging_handler import logging_handler +from logging_handler import LoggingHandler mongo = create_mongo_fixture() def test_log_usage_for_key(mongo): - handler = logging_handler(True) + handler = LoggingHandler(True) handler.setup(mongo) db = mongo["gateway"] - logCollection = db["logs"] + log_collection = db["logs"] handler.log_usage_for_key(50, "newModel", "123") - assert logCollection.count_documents({}) == 1 + assert log_collection.count_documents({}) == 1 handler.log_usage_for_key(70, "model2", "123") - assert logCollection.count_documents({}) == 2 + assert log_collection.count_documents({}) == 2 handler.log_usage_for_key(50, "model2", "321") - assert logCollection.count_documents({"model": "model2"}) == 2 - assert logCollection.count_documents({"source": "123"}) == 2 - assert logCollection.count_documents({"source": "321"}) == 1 - assert logCollection.count_documents({"sourcetype": "apikey"}) == 3 - firstLog = logCollection.find_one({"model": "newModel"}) - secondLog = logCollection.find_one({"tokencount": 70}) - assert firstLog["timestamp"] <= secondLog["timestamp"] + assert log_collection.count_documents({"model": "model2"}) == 2 + assert log_collection.count_documents({"source": "123"}) == 2 + assert log_collection.count_documents({"source": "321"}) == 1 + assert log_collection.count_documents({"sourcetype": "apikey"}) == 3 + first_log = log_collection.find_one({"model": "newModel"}) + second_log = log_collection.find_one({"tokencount": 70}) + assert first_log["timestamp"] <= second_log["timestamp"] def test_log_usage_for_user(mongo): - handler = logging_handler(True) + handler = LoggingHandler(True) handler.setup(mongo) db = mongo["gateway"] - logCollection = db["logs"] + log_collection = db["logs"] handler.log_usage_for_user(50, "newModel", "123") - assert logCollection.count_documents({}) == 1 + assert log_collection.count_documents({}) == 1 handler.log_usage_for_user(70, "model2", "123") - assert logCollection.count_documents({}) == 2 + assert log_collection.count_documents({}) == 2 handler.log_usage_for_user(50, "model2", "321") - assert logCollection.count_documents({"model": "model2"}) == 2 - assert logCollection.count_documents({"source": "123"}) == 2 - assert logCollection.count_documents({"source": "321"}) == 1 - assert logCollection.count_documents({"sourcetype": "user"}) == 3 - firstLog = logCollection.find_one({"model": "newModel"}) - secondLog = logCollection.find_one({"tokencount": 70}) - assert firstLog["timestamp"] <= secondLog["timestamp"] + assert log_collection.count_documents({"model": "model2"}) == 2 + assert log_collection.count_documents({"source": "123"}) == 2 + assert log_collection.count_documents({"source": "321"}) == 1 + assert log_collection.count_documents({"sourcetype": "user"}) == 3 + first_log = log_collection.find_one({"model": "newModel"}) + second_log = log_collection.find_one({"tokencount": 70}) + assert first_log["timestamp"] <= second_log["timestamp"] From d3a2f3cc785d837d517bcbf9b44a3279f37316a4 Mon Sep 17 00:00:00 2001 From: Thomas Pfau Date: Wed, 29 Nov 2023 10:53:01 +0200 Subject: [PATCH 10/10] :bug: wrong argument use for 'unique' index --- app/utils/key_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/utils/key_handler.py b/app/utils/key_handler.py index 13441af..cd06d0e 100644 --- a/app/utils/key_handler.py +++ b/app/utils/key_handler.py @@ -28,12 +28,12 @@ def setup(self, mongo_client: pymongo.MongoClient, redis_client: redis.Redis): keyindices = self.key_collection.index_information() # Make sure, that key is an index (avoids duplicates); if not "key" in keyindices: - self.key_collection.create_index("key", "unique") + self.key_collection.create_index("key", unique=True) self.user_collection = self.db["users"] # Make sure, that username is an index (avoids duplicates when creating keys, which automatically adds a user if necessary); userindices = self.user_collection.index_information() if not "username" in userindices: - self.user_collection.create_index("username", "unique") + self.user_collection.create_index("username", unique=True) def generate_api_key(self, length: int = 64): """