diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 1f5d76aff..d47696d51 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -8,7 +8,11 @@ RUN date RUN pip install --upgrade pip RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends default-mysql-client ack bash-completion unixodbc unixodbc-dev + && apt-get -y install --no-install-recommends default-mysql-client ack bash-completion unixodbc unixodbc-dev \ + build-essential libssl-dev libffi-dev freetds-dev freetds-bin unixodbc-dev tdsodbc + +RUN sudo bash -c "echo -e '[FreeTDS]\nDescription=FreeTDS Driver\nDriver=/usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so\ +\nSetup=/usr/lib/x86_64-linux-gnu/odbc/libtdsS.so' >> /etc/odbcinst.ini" COPY requirements.txt /tmp/pip-tmp/ RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt && rm -rf /tmp/pip-tmp diff --git a/.gitignore b/.gitignore index dc6a53c59..9b6e06caa 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ app/static/files/ local-override.yml geckodriver.log .coverage +cron.log diff --git a/app/logic/emailHandler.py b/app/logic/emailHandler.py index 093685b82..2ec261280 100644 --- a/app/logic/emailHandler.py +++ b/app/logic/emailHandler.py @@ -278,4 +278,7 @@ def replaceStaticPlaceholders(eventId, email_body): event_link="{event_link}", recipient_name="{recipient_name}", relative_time=event.relativeTime) - return new_body \ No newline at end of file + return new_body + + + diff --git a/app/scripts/import_users.py b/app/scripts/import_users.py index 25e0d7358..478aa4b7b 100644 --- a/app/scripts/import_users.py +++ b/app/scripts/import_users.py @@ -1,4 +1,7 @@ +import logging import pyodbc +import sys +import argparse from ldap3 import Server, Connection, ALL import peewee @@ -6,23 +9,59 @@ from app.models.user import User from app.logic.utils import getUsernameFromEmail -def main(): - """ - This function runs the updateRecords function once the script is run. - """ - print("Don't forget to put the correct Tracy and LDAP passwords in app/config/local-override.yml") - print("\nGetting Updated Names, Majors, and Class Levels\n--------------------------------------------------\n") +# Argument parser for log levels +def parseArgs(): + parser = argparse.ArgumentParser(description="Import users script with logging.") + parser.add_argument( + '--log-level', + default='INFO', + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], + help='Set the logging level' + ) + return parser.parse_args() + +arguments = parseArgs() +logLevels = getattr(logging, arguments.log_level.upper(), logging.INFO) + +# Configure logging +logger = logging.getLogger(__name__) +logger.setLevel(logLevels) - addToDb(getStudentData()) - print("done.") - addToDb(getFacultyStaffData()) - print("done.") +consoleHandler = logging.StreamHandler(sys.stdout) +consoleHandler.setLevel(logLevels) +formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +consoleHandler.setFormatter(formatter) - print("\n\nGetting Preferred Names\n--------------------------\n") +logger.addHandler(consoleHandler) + +def main(): + """ + This function runs the updateRecords function once the script is run. + """ + logger.info("Script started.") + logger.info("Don't forget to put the correct Tracy and LDAP passwords in app/config/local-override.yml") + + logger.info("Getting Updated Names, Majors, and Class Levels") + + studentData = addToDb(getStudentData()) + studentAdded = studentData[0] + studentUpdated = studentData[1] + logger.info(f"{studentAdded} students were added.") + logger.info(f"{studentUpdated} students were updated.") + logger.info("Finished updating student data.") + + facultyStaffData = addToDb(getFacultyStaffData()) + facultyStaffAdded = facultyStaffData[0] + facultyStaffUpdated = facultyStaffData[1] + logger.info(f"{facultyStaffAdded} faculties/staffs were added.") + logger.info(f"{facultyStaffUpdated} faculties/staffs were updated.") + logger.info("Finished updating faculty and staff data.") + + logger.info("Getting Preferred Names from LDAP") ldap = getLdapConn() - print("LDAP Connected.") + logger.info("LDAP Connected.") people = fetchLdapList(ldap, alphaRange('a','d')) people += fetchLdapList(ldap, alphaRange('e','j')) @@ -30,28 +69,35 @@ def main(): people += fetchLdapList(ldap, alphaRange('q','z')) updateFromLdap(people) - print("Update Complete.") + logger.info("Update from LDAP Complete.") -def alphaRange(start,end): +def alphaRange(start, end): return [chr(i) for i in range(ord(start), ord(end)+1)] def getLdapConn(): - server = Server ('berea.edu', port=389, use_ssl=False, get_info='ALL') - conn = Connection (server, user=app.config['ldap']['user'], password=app.config['ldap']['password']) - if not conn.bind(): - print(conn.result) - raise Exception("BindError") - - return conn + try: + server = Server('berea.edu', port=389, use_ssl=False, get_info=ALL) + conn = Connection(server, user=app.config['ldap']['user'], password=app.config['ldap']['password']) + if not conn.bind(): + logger.error(f"LDAP bind failed: {conn.result}") + raise Exception("BindError") + return conn + except Exception as e: + logger.error(f"Failed to connect to LDAP: {e}") + raise def fetchLdapList(conn, startletters): - # Get the givennames from LDAP - we have to segment them to make sure each request is under 1500 - conn.search('dc=berea,dc=edu', - f"(|" + "".join(map(lambda s: f"(givenname={s}*)", startletters)) + ")", - attributes = ['samaccountname', 'givenname', 'sn', 'employeeid'] - ) - print(f"Found {len(conn.entries)} {startletters[0]}-{startletters[-1]} in AD"); - return conn.entries + try: + conn.search( + 'dc=berea,dc=edu', + f"(|" + "".join(map(lambda s: f"(givenname={s}*)", startletters)) + ")", + attributes=['samaccountname', 'givenname', 'sn', 'employeeid'] + ) + logger.info(f"Found {len(conn.entries)} entries for {startletters[0]}-{startletters[-1]} in AD") + return conn.entries + except Exception as e: + logger.error(f"Failed to fetch LDAP list for {startletters[0]}-{startletters[-1]}: {e}") + raise def updateFromLdap(people): for person in people: @@ -59,12 +105,13 @@ def updateFromLdap(people): preferred = str(get_key(person, 'givenname')).strip() if preferred: - count = User.update(firstName=preferred).where(User.bnumber == bnumber).execute() - if count: - print(f"Updating {bnumber} name to {preferred}") + try: + count = User.update(firstName=preferred).where(User.bnumber == bnumber).execute() + if count: + logger.debug(f"Updated {bnumber} name to {preferred}") + except Exception as e: + logger.error(f"Failed to update user {bnumber} with preferred name {preferred}: {e}") -# Return the value for a key or None -# Can't use .get() because it's a ldap3.abstract.entry.Entry instead of a Dict def get_key(entry, key): if key in entry: return entry[key] @@ -78,81 +125,94 @@ def getMssqlCursor(): "host": app.config["tracy"]["host"], "db": app.config["tracy"]["name"] } - pyodbc_uri = 'DRIVER=FreeTDS;SERVER={};PORT=1433;DATABASE={};UID={};PWD={};TDS_Version=8.0;'.format(details['host'],details['db'],details['user'],details['password']) - - pyconn = pyodbc.connect(pyodbc_uri) # connects a tcp based client socket to a tcp based server socket - return pyconn.cursor() # allows python to execute sql database commands + pyodbc_uri = 'DRIVER=FreeTDS;SERVER={};PORT=1433;DATABASE={};UID={};PWD={};TDS_Version=8.0;'.format( + details['host'], details['db'], details['user'], details['password'] + ) + try: + pyconn = pyodbc.connect(pyodbc_uri) + logger.info("Connected to Tracy database.") + return pyconn.cursor() + except Exception as e: + logger.error(f"Failed to connect to Tracy database: {e}") + raise def addToDb(userList): + usersAdded = 0 + usersUpdated = 0 for user in userList: try: User.insert(user).execute() - + logger.debug(f"Inserted user {user['bnumber']}") + usersAdded += 1 except peewee.IntegrityError as e: - if user['username']: - (User.update(firstName = user['firstName'], lastName = user['lastName'], email = user['email'], major = user['major'], classLevel = user['classLevel'], cpoNumber = user['cpoNumber']) - .where(user['bnumber'] == User.bnumber)).execute() - else: - print(f"No username for {user['bnumber']}!", user) - + try: + if user['username']: + (User.update( + firstName=user['firstName'], + lastName=user['lastName'], + email=user['email'], + major=user['major'], + classLevel=user['classLevel'], + cpoNumber=user['cpoNumber'] + ).where(User.bnumber == user['bnumber'])).execute() + logger.debug(f"Updated user {user['bnumber']}") + usersUpdated += 1 + else: + logger.warning(f"No username for {user['bnumber']}!", user) + except Exception as e: + logger.error(f"Failed to update user {user['bnumber']}: {e}") except Exception as e: - print(e) + logger.error(f"Failed to insert or update user {user['bnumber']}: {e}") + + return [usersAdded, usersUpdated] def getFacultyStaffData(): - """ - This function pulls all the faculty and staff data from Tracy and formats for our table - - Tracy's STUSTAFF table has the following columns: - 1. PIDM - 2. ID - 3. FIRST_NAME - 4. LAST_NAME - 5. EMAIL - 6. CPO - 7. ORG - 8. DEPT_NAME - """ - print("Retrieving Faculty data from Tracy...",end="") - c = getMssqlCursor() - return [ - { "username": getUsernameFromEmail(row[4].strip()), - "bnumber": row[1].strip(), - "email": row[4].strip(), - "phoneNumber": None, - "firstName": row[2].strip(), - "lastName": row[3].strip(), - "isStudent": False, - "isFaculty": True, - "isStaff": False, - "major": None, - "classLevel": None, - "cpoNumber": row[5].strip(), - } - for row in c.execute('select * from STUSTAFF') - ] + logger.info("Retrieving Faculty and Staff data from Tracy...") + try: + c = getMssqlCursor() + return [ + { + "username": getUsernameFromEmail(row[4].strip()), + "bnumber": row[1].strip(), + "email": row[4].strip(), + "phoneNumber": None, + "firstName": row[2].strip(), + "lastName": row[3].strip(), + "isStudent": False, + "isFaculty": True, + "isStaff": False, + "major": None, + "classLevel": None, + "cpoNumber": row[5].strip(), + } + for row in c.execute('select * from STUSTAFF') + ] + except Exception as e: + logger.error(f"Failed to retrieve Faculty and Staff data: {e}") + raise def getStudentData(): - """ - This function pulls all the student data from Tracy and formats for our table - """ - print("Retrieving Student data from Tracy...",end="") - c = getMssqlCursor() - return [ - { "username": getUsernameFromEmail(row[9].strip()), - "bnumber": row[1].strip(), - "email": row[9].strip(), - "phoneNumber": None, - "firstName": row[2].strip(), - "lastName": row[3].strip(), - "isStudent": True, - "major": row[6].strip(), - "classLevel": row[4].strip(), - "cpoNumber": row[10].strip(), - } - for row in c.execute('select * from STUDATA') - ] + logger.info("Retrieving Student data from Tracy...") + try: + c = getMssqlCursor() + return [ + { + "username": getUsernameFromEmail(row[9].strip()), + "bnumber": row[1].strip(), + "email": row[9].strip(), + "phoneNumber": None, + "firstName": row[2].strip(), + "lastName": row[3].strip(), + "isStudent": True, + "major": row[6].strip(), + "classLevel": row[4].strip(), + "cpoNumber": row[10].strip(), + } + for row in c.execute('select * from STUDATA') + ] + except Exception as e: + logger.error(f"Failed to retrieve Student data: {e}") + raise if __name__ == '__main__': main() - -