Austin's assignment pull branch

.gitignore

@@ -62,3 +62,5 @@ fixed/
 #random
 canvas_tool.py
 
+#Assignment Files
+assignmentFiles/

codeval.py

@@ -13,6 +13,7 @@
 from file_utils import copy_files_to_submission_dir, \
     download_attachment, set_acls, unzip
 import convertMD2Html
+import pullcourse
 
 CODEVAL_FOLDER = "course files/CodEval"
 CODEVAL_SUFFIX = ".codeval"
@@ -330,6 +331,31 @@ def cmdargs():
     global canvasHandler
     canvasHandler = CanvasHandler()
 
+@cmdargs.command()
+@click.argument("course_name")
+@click.argument("assignment_name")
+@click.option("--mkdn/--no-mkdn", default=True, show_default=True, help="Export assignment as markdown or leave it as json")
+def pull_assignment(course_name, assignment_name, mkdn):
+    """
+        Pulls the assignment from the given course as a file
+    """
+    global path
+    global canvasHandler
+    try:
+        course = canvasHandler.get_course(course_name)
+    except Exception as e:
+        errorWithException(f'get_course api failed with following error : {e}')
+    else:
+        debug(f'Successfully retrieved the course: {course_name}')
+    courseParser = pullcourse.CourseParser(course, assignment_name, path + "/",
+        canvasHandler.parser['SERVER']['url'], canvasHandler.parser['SERVER']['token'])
+
+    if mkdn:
+        courseParser.export_to_MD()
+    else:
+        courseParser.export_to_json()
+
+
 @cmdargs.command()
 @click.argument("course_name")
 @click.argument("specname")

mkdnparser.py

@@ -0,0 +1,95 @@
+from markdownify import markdownify as md
+import re
+import os
+import requests
+
+
+
+class MkdnParser:
+
+    def __init__(self, assignment_dict, dir_name, api_url, api_key):
+        # Get api key for making file download requests later
+        self.assignment_dict = assignment_dict
+        self.dir_name = dir_name
+        self.api_url = api_url
+        self.api_key = api_key
+
+    def clean_brackets(self, s):
+        """
+        Removes directory from filename inside brackets
+
+        Keyword arguments:
+        s -- re.Match object containing the directory
+        Return: string
+        """
+        print(s.group(2))
+
+        return '[' + s.group(2) + ']'
+
+
+    def check_links(self, s):
+        """
+        Checks if the given link is a canvas file, if so, downloads the file and gives a path to the file
+
+        Keyword arguments:
+        s -- re.Match object containing a link
+        Return: string
+        """
+        link = s.group(1)
+        if "instructure" in link:
+            filename = s.group(2)
+            headers = {'Authorization': 'Bearer ' + self.mParser['CANVAS CLONE']['api_key']}
+            response = requests.get(link, headers)
+            if 'Content-Length' in response.headers:
+                with open('CodEvalTooling/' + self.dir_name + filename, 'wb') as outfile:
+                    outfile.write(response.content)
+
+            relinked_link = s.group().replace(s.group(1), 'URL_OF_HW')
+            relinked_link = relinked_link.replace(s.group(2), self.dir_name + filename)
+
+            return relinked_link
+        else:
+            return s
+
+
+    def clean_description(self):
+        """
+        Removes newline characters from the description in the assignment json
+
+        Keyword arguments:
+        Return: None
+        """
+        self.assignment_dict.update({'description': self.assignment_dict['description'].replace(r'\n', '').replace(r'&nbsp', ' ')})
+
+
+    def parse_links_to_files(self):
+        """
+        Takes json structure of assignment and takes files, downloads them and changes the locations in the json
+
+        Keyword arguments:
+        Return: None
+        """
+        linkPattern = r'<a[^>]*?href="([^"]*)"[^>]*>([^<]*)'
+
+        linkTags = re.sub(linkPattern, self.check_links, self.assignment_dict['description'])
+
+        self.assignment_dict.update({'description': linkTags})
+
+    def get_mkdn(self):
+        """
+        Opens the file, reads the file and writes it to markdown
+
+        Keyword arguments:
+        filename -- string, file to write to
+        Return: None
+        """
+        self.clean_description()
+        file_mkdn = ''
+        file_mkdn += 'CRT_HW START ' + self.assignment_dict['name'] + '\n'
+        file_mkdn += md(self.assignment_dict['description'], autolinks = False, escape_asterisks = False, escape_underscores = False, heading_style = 'ATX', strip = ['ul'])
+        file_mkdn += '\nCRT_HW END'
+
+        prepared_pattern = r'\[(%filename%)([^]]*)\]'.replace('%filename%', self.dir_name.replace('\\', '/'))
+        file_mkdn = re.sub(prepared_pattern, self.clean_brackets, file_mkdn)
+
+        return file_mkdn

pullcourse.py

@@ -0,0 +1,92 @@
+from canvasapi import Canvas
+from mkdnparser import MkdnParser
+import json
+import os
+
+MODULE_REQUIRED_PARAMETERS = {'name'}
+MODULE_OPTIONAL_PARAMETERS = {'unlock_at', 'position', 'require_sequential_progress', 'publish_final_grade', 'published'}
+
+MODULE_ITEM_REQUIRED_PARAMETERS = {'type'}
+MODULE_ITEM_OPTIONAL_PARAMETERS = {'title', 'position', 'indent', 'external_url', 'new_tab', 'completion_requirement', 'iframe'}
+
+ASSIGNMENT_GROUP_REQUIRED_PARAMETERS = {}
+ASSIGNMENT_GROUP_OPTIONAL_PARAMETERS = {'name', 'position', 'group_weight', 'integration_data'}
+
+ASSIGNMENT_REQUIRED_PARAMETERS = {'name'}
+ASSIGNMENT_OPTIONAL_PARAMETERS = {'position', 'submission_types', 'allowed_extensions', 'turnitin_enabled', 'vericite_enabled', 'turnitin_settings',
+    'integration_data', 'peer_reviews', 'automatic_peer_reviews', 'notify_of_update', 'grade_group_students_individually', 'external_tool_attributes',
+    'points_possible', 'grading_type', 'due_at', 'lock_at', 'unlock_at', 'description', 'published', 'omit_from_final_grade', 'quiz_lti', 'moderated_grading', 'grader_count',
+    'grader_comments_visible_to_graders', 'graders_anonymous_to_graders', 'graders_names_visible_to_final_grader', 'anonymous_grading', 'allowed_attempts'}
+
+DISCUSSION_TOPIC_REQUIRED_PARAMETERS = {}
+DISCUSSION_TOPIC_OPTIONAL_PARAMETERS = {'title', 'message', 'discussion_type', 'published', 'delayed_post_at', 'allow_rating', 'lock_at', 'podcast_enabled',
+    'podcast_has_student_posts', 'require_inital_post', 'is_announcement', 'pinned', 'position_after', 'only_graders_can_rate',
+    'sort_by_rating', 'attachment', 'specific_sections'}
+
+# Update parameters given set, otherwise set to None
+def update_parameters(dict_to_update, REQUIRED_PARAMETERS, OPTIONAL_PARAMETERS):
+    """
+    :type dict_to_update: dict
+    :type REQUIRED_PARAMETERS: set
+    :rtype: dict
+    """
+    updated_dict = {}
+
+    for parameter in REQUIRED_PARAMETERS:
+        if parameter in dict_to_update:
+            updated_dict.update({parameter: dict_to_update[parameter]})
+        else:
+            updated_dict.update({parameter: None})
+    for parameter in OPTIONAL_PARAMETERS:
+        if parameter in dict_to_update and not dict_to_update[parameter] is None:
+            updated_dict.update({parameter: dict_to_update[parameter]})
+
+    return updated_dict
+
+class CourseParser:
+    def __init__(self, course, assignment_name, dir_name, api_url, api_key):
+        self.course = course
+        self.assignment_name = assignment_name
+        self.dir_name = dir_name
+        self.api_url = api_url
+        self.api_key = api_key
+
+    def get_assignments(self):
+        assignment_list = self.course.get_assignments()
+
+        selected_assignment_list = []
+        if self.assignment_name != '*':    
+            for assignment in assignment_list:
+                assignment_dict = vars(assignment)
+                if assignment_dict['name'] == self.assignment_name:
+                    selected_assignment_list.append(assignment_dict)
+
+        return selected_assignment_list
+
+    def export_to_MD(self):
+        selected_assignment_list = self.get_assignments()
+        for assignment_dict in selected_assignment_list:
+            assignmentParser = MkdnParser(assignment_dict, self.dir_name, self.api_url, self.api_key)
+            assignmentParser.parse_links_to_files()
+            with open(self.dir_name + assignment_dict['name'] + str(assignment_dict['id']), 'wb') as outfile:
+                outfile.write(bytes(assignmentParser.get_mkdn(), 'utf-8'))
+
+
+    def export_to_json(self):
+        selected_assignment_list = self.get_assignments()
+        for assignment_dict in selected_assignment_list:
+            updated_assignment_dict = update_parameters(assignment_dict, ASSIGNMENT_REQUIRED_PARAMETERS, ASSIGNMENT_OPTIONAL_PARAMETERS)
+            self.write_to_json(self.dir_name + assignment_dict['name'] + str(assignment_dict['id']) + '.json', updated_assignment_dict)
+
+    def write_to_json(self, filename, dict_to_write):
+        """
+        Writes to json file with given filename and dictionary
+
+        Keyword arguments:
+        filename -- string
+        dict_to_write -- string
+        Return: None
+        """
+        dict_json = json.dumps(dict_to_write, indent=4, default=str)
+        with open(filename, 'w') as outfile:
+            outfile.write(dict_json)

requirements.txt

@@ -1,11 +1,15 @@
+beautifulsoup4==4.12.2
 canvasapi==2.2.0
-certifi==2021.10.8
+certifi==2022.12.7
 charset-normalizer==2.0.9
 click==8.0.3
 configparser==5.2.0
 idna==3.3
+markdown==3.4.1
+markdownify==0.11.6
+pymongo==4.3.3
 pytz==2021.3
 requests==2.27.0
+soupsieve==2.4.1
 urllib3==1.26.7
-pymongo==4.3.3
-markdown==3.4.1
+