diff --git a/.gitignore b/.gitignore index 4b6a2f8..9b1d687 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,5 @@ fixed/ #random canvas_tool.py +#Assignment Files +assignmentFiles/ \ No newline at end of file diff --git a/codeval.py b/codeval.py index 49a5214..301f260 100644 --- a/codeval.py +++ b/codeval.py @@ -13,6 +13,7 @@ from file_utils import copy_files_to_submission_dir, \ download_attachment, set_acls, unzip import convertMD2Html +import pullcourse CODEVAL_FOLDER = "course files/CodEval" CODEVAL_SUFFIX = ".codeval" @@ -330,6 +331,31 @@ def cmdargs(): global canvasHandler canvasHandler = CanvasHandler() +@cmdargs.command() +@click.argument("course_name") +@click.argument("assignment_name") +@click.option("--mkdn/--no-mkdn", default=True, show_default=True, help="Export assignment as markdown or leave it as json") +def pull_assignment(course_name, assignment_name, mkdn): + """ + Pulls the assignment from the given course as a file + """ + global path + global canvasHandler + try: + course = canvasHandler.get_course(course_name) + except Exception as e: + errorWithException(f'get_course api failed with following error : {e}') + else: + debug(f'Successfully retrieved the course: {course_name}') + courseParser = pullcourse.CourseParser(course, assignment_name, path + "/", + canvasHandler.parser['SERVER']['url'], canvasHandler.parser['SERVER']['token']) + + if mkdn: + courseParser.export_to_MD() + else: + courseParser.export_to_json() + + @cmdargs.command() @click.argument("course_name") @click.argument("specname") diff --git a/mkdnparser.py b/mkdnparser.py new file mode 100644 index 0000000..5e0bf38 --- /dev/null +++ b/mkdnparser.py @@ -0,0 +1,95 @@ +from markdownify import markdownify as md +import re +import os +import requests + + + +class MkdnParser: + + def __init__(self, assignment_dict, dir_name, api_url, api_key): + # Get api key for making file download requests later + self.assignment_dict = assignment_dict + self.dir_name = dir_name + self.api_url = api_url + self.api_key = api_key + + def clean_brackets(self, s): + """ + Removes directory from filename inside brackets + + Keyword arguments: + s -- re.Match object containing the directory + Return: string + """ + print(s.group(2)) + + return '[' + s.group(2) + ']' + + + def check_links(self, s): + """ + Checks if the given link is a canvas file, if so, downloads the file and gives a path to the file + + Keyword arguments: + s -- re.Match object containing a link + Return: string + """ + link = s.group(1) + if "instructure" in link: + filename = s.group(2) + headers = {'Authorization': 'Bearer ' + self.mParser['CANVAS CLONE']['api_key']} + response = requests.get(link, headers) + if 'Content-Length' in response.headers: + with open('CodEvalTooling/' + self.dir_name + filename, 'wb') as outfile: + outfile.write(response.content) + + relinked_link = s.group().replace(s.group(1), 'URL_OF_HW') + relinked_link = relinked_link.replace(s.group(2), self.dir_name + filename) + + return relinked_link + else: + return s + + + def clean_description(self): + """ + Removes newline characters from the description in the assignment json + + Keyword arguments: + Return: None + """ + self.assignment_dict.update({'description': self.assignment_dict['description'].replace(r'\n', '').replace(r' ', ' ')}) + + + def parse_links_to_files(self): + """ + Takes json structure of assignment and takes files, downloads them and changes the locations in the json + + Keyword arguments: + Return: None + """ + linkPattern = r']*?href="([^"]*)"[^>]*>([^<]*)' + + linkTags = re.sub(linkPattern, self.check_links, self.assignment_dict['description']) + + self.assignment_dict.update({'description': linkTags}) + + def get_mkdn(self): + """ + Opens the file, reads the file and writes it to markdown + + Keyword arguments: + filename -- string, file to write to + Return: None + """ + self.clean_description() + file_mkdn = '' + file_mkdn += 'CRT_HW START ' + self.assignment_dict['name'] + '\n' + file_mkdn += md(self.assignment_dict['description'], autolinks = False, escape_asterisks = False, escape_underscores = False, heading_style = 'ATX', strip = ['ul']) + file_mkdn += '\nCRT_HW END' + + prepared_pattern = r'\[(%filename%)([^]]*)\]'.replace('%filename%', self.dir_name.replace('\\', '/')) + file_mkdn = re.sub(prepared_pattern, self.clean_brackets, file_mkdn) + + return file_mkdn \ No newline at end of file diff --git a/pullcourse.py b/pullcourse.py new file mode 100644 index 0000000..772ae99 --- /dev/null +++ b/pullcourse.py @@ -0,0 +1,92 @@ +from canvasapi import Canvas +from mkdnparser import MkdnParser +import json +import os + +MODULE_REQUIRED_PARAMETERS = {'name'} +MODULE_OPTIONAL_PARAMETERS = {'unlock_at', 'position', 'require_sequential_progress', 'publish_final_grade', 'published'} + +MODULE_ITEM_REQUIRED_PARAMETERS = {'type'} +MODULE_ITEM_OPTIONAL_PARAMETERS = {'title', 'position', 'indent', 'external_url', 'new_tab', 'completion_requirement', 'iframe'} + +ASSIGNMENT_GROUP_REQUIRED_PARAMETERS = {} +ASSIGNMENT_GROUP_OPTIONAL_PARAMETERS = {'name', 'position', 'group_weight', 'integration_data'} + +ASSIGNMENT_REQUIRED_PARAMETERS = {'name'} +ASSIGNMENT_OPTIONAL_PARAMETERS = {'position', 'submission_types', 'allowed_extensions', 'turnitin_enabled', 'vericite_enabled', 'turnitin_settings', + 'integration_data', 'peer_reviews', 'automatic_peer_reviews', 'notify_of_update', 'grade_group_students_individually', 'external_tool_attributes', + 'points_possible', 'grading_type', 'due_at', 'lock_at', 'unlock_at', 'description', 'published', 'omit_from_final_grade', 'quiz_lti', 'moderated_grading', 'grader_count', + 'grader_comments_visible_to_graders', 'graders_anonymous_to_graders', 'graders_names_visible_to_final_grader', 'anonymous_grading', 'allowed_attempts'} + +DISCUSSION_TOPIC_REQUIRED_PARAMETERS = {} +DISCUSSION_TOPIC_OPTIONAL_PARAMETERS = {'title', 'message', 'discussion_type', 'published', 'delayed_post_at', 'allow_rating', 'lock_at', 'podcast_enabled', + 'podcast_has_student_posts', 'require_inital_post', 'is_announcement', 'pinned', 'position_after', 'only_graders_can_rate', + 'sort_by_rating', 'attachment', 'specific_sections'} + +# Update parameters given set, otherwise set to None +def update_parameters(dict_to_update, REQUIRED_PARAMETERS, OPTIONAL_PARAMETERS): + """ + :type dict_to_update: dict + :type REQUIRED_PARAMETERS: set + :rtype: dict + """ + updated_dict = {} + + for parameter in REQUIRED_PARAMETERS: + if parameter in dict_to_update: + updated_dict.update({parameter: dict_to_update[parameter]}) + else: + updated_dict.update({parameter: None}) + for parameter in OPTIONAL_PARAMETERS: + if parameter in dict_to_update and not dict_to_update[parameter] is None: + updated_dict.update({parameter: dict_to_update[parameter]}) + + return updated_dict + +class CourseParser: + def __init__(self, course, assignment_name, dir_name, api_url, api_key): + self.course = course + self.assignment_name = assignment_name + self.dir_name = dir_name + self.api_url = api_url + self.api_key = api_key + + def get_assignments(self): + assignment_list = self.course.get_assignments() + + selected_assignment_list = [] + if self.assignment_name != '*': + for assignment in assignment_list: + assignment_dict = vars(assignment) + if assignment_dict['name'] == self.assignment_name: + selected_assignment_list.append(assignment_dict) + + return selected_assignment_list + + def export_to_MD(self): + selected_assignment_list = self.get_assignments() + for assignment_dict in selected_assignment_list: + assignmentParser = MkdnParser(assignment_dict, self.dir_name, self.api_url, self.api_key) + assignmentParser.parse_links_to_files() + with open(self.dir_name + assignment_dict['name'] + str(assignment_dict['id']), 'wb') as outfile: + outfile.write(bytes(assignmentParser.get_mkdn(), 'utf-8')) + + + def export_to_json(self): + selected_assignment_list = self.get_assignments() + for assignment_dict in selected_assignment_list: + updated_assignment_dict = update_parameters(assignment_dict, ASSIGNMENT_REQUIRED_PARAMETERS, ASSIGNMENT_OPTIONAL_PARAMETERS) + self.write_to_json(self.dir_name + assignment_dict['name'] + str(assignment_dict['id']) + '.json', updated_assignment_dict) + + def write_to_json(self, filename, dict_to_write): + """ + Writes to json file with given filename and dictionary + + Keyword arguments: + filename -- string + dict_to_write -- string + Return: None + """ + dict_json = json.dumps(dict_to_write, indent=4, default=str) + with open(filename, 'w') as outfile: + outfile.write(dict_json) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c710de9..44a5762 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,15 @@ +beautifulsoup4==4.12.2 canvasapi==2.2.0 -certifi==2021.10.8 +certifi==2022.12.7 charset-normalizer==2.0.9 click==8.0.3 configparser==5.2.0 idna==3.3 +markdown==3.4.1 +markdownify==0.11.6 +pymongo==4.3.3 pytz==2021.3 requests==2.27.0 +soupsieve==2.4.1 urllib3==1.26.7 -pymongo==4.3.3 -markdown==3.4.1 +