diff --git a/autobot/actions/banners.py b/autobot/actions/banners.py deleted file mode 100644 index 68edb34..0000000 --- a/autobot/actions/banners.py +++ /dev/null @@ -1,93 +0,0 @@ -from hashlib import sha256 -import io -import warnings - -from jinja2 import Template -from PIL import Image -import requests -import imgkit - -from autobot import get_template -from autobot.concepts import Meeting - - -def render_cover(meeting: Meeting): - """Generates the banner images for each meeting. These should be posted - to the website as well as relevant social media. - - NOTE: This function is destructive. It will overwrite the banner it's - generating - this is intentional behavior, ergo **do not edit banners - directly**. - """ - template_banner = load_setup_template("banners/event.html.j2") - - accepted_content_types = [ - f"image/{x}" for x in ["jpg", "jpeg", "png", "gif", "tiff"] - ] - - extension = meeting.required["cover"].split(".")[-1] - - cover_image_path = paths.site_post_assets(meeting) / "cover.png" - - # snag the image from the URL provided in the syllabus - cover = requests.get( - meeting.required["cover"], headers={"user-agent": "Mozilla/5.0"} - ) - # use this to mute the EXIF data error ~ this seems to be a non-issue based - # on what I've read (@ionlights) ~ circa Sep/Oct 2019 - warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning) - if cover.headers["Content-Type"] in accepted_content_types: - image_as_bytes = io.BytesIO(cover.content) - try: - # noinspection PyTypeChecker - cover_as_bytes = io.BytesIO(open(cover_image_path, "rb").read()) - - # get hashes to check for diff - image_hash = sha256(image_as_bytes.read()).hexdigest() - cover_hash = sha256(cover_as_bytes.read()).hexdigest() - - # clearly, something has changed between what we have and what - # was just downloaded -> update - if cover_hash != image_hash: - image = Image.open(image_as_bytes) - else: - image = Image.open(cover_as_bytes) - except FileNotFoundError: - image = Image.open(image_as_bytes) - finally: - image.save(cover_image_path) - - out = cover_image_path.with_name("banner.png") - banner = template_banner.render(meeting=meeting, cover=cover_image_path.absolute()) - - _imgkit_from_str(banner, out) - - -def render_weekly_instagram_post(meeting: Meeting): - template_instagram = load_setup_template("banners/instagram.html.j2") - raise NotImplementedError() - - -def render_video_background(meeting): - template_videobg = load_setup_template("banners/video-bg.html.j2") - - out = cover_image_path.with_name("banner.png") - banner = template_banner.render(meeting=meeting, cover=cover_image_path.absolute()) - - _imgkit_from_str(banner, out) - raise NotImplementedError() - - -def _imgkit_from_str(to_render, output_file): - imgkit.from_string( - to_render, - output_file, - options={ - # standard flags should be passed as dict keys with empty values... - "quiet": "", - "debug-javascript": "", - "enable-javascript": "", - "javascript-delay": "400", - "no-stop-slow-scripts": "", - }, - ) diff --git a/autobot/actions/meetings.py b/autobot/actions/meetings.py index f3d255f..c347f4d 100644 --- a/autobot/actions/meetings.py +++ b/autobot/actions/meetings.py @@ -1,39 +1,18 @@ -import io -import os -import copy -import datetime -import shutil -import subprocess -import warnings -from pathlib import Path -from hashlib import sha256 -from typing import List, Dict -from itertools import product -from distutils.dir_util import copy_tree -from autobot import load_upkeep_template - -import imgkit +import json + import requests -import yaml -from PIL import Image -import pandas as pd -import numpy as np -from jinja2 import Template -import nbformat as nbf -import nbconvert as nbc -from nbgrader.preprocessors import ClearSolutions, ClearOutput - -from autobot import get_template -from autobot.concepts import Meeting +from tqdm import tqdm + from autobot.actions import paths from autobot.apis import kaggle from autobot.apis.nbconvert import ( SolutionbookToPostExporter, SolutionbookToWorkbookExporter, TemplateNotebookValidator, - ValidateNBGraderPreprocessor, FileExtensions, ) +from autobot.concepts import Meeting +from autobot.pathing import templates, repositories def update_or_create_folders_and_files(meeting: Meeting): @@ -58,7 +37,37 @@ def update_or_create_folders_and_files(meeting: Meeting): # TODO allow for meetings to be moved – this intuitively makes sense to resolve # with the `filename` parameter, but may also need to consider the `date` (you # can get all this from `repr(meeting)`) - raise NotImplementedError() + path = repositories.local_meeting_root(meeting) + + root = repositories.local_semester_root(meeting) + + local_neighbors = list(root.iterdir()) + + for d in local_neighbors: + date, name = d.stem[:10], d.stem[11:] + + # remove placeholder + # rename by date + shares_meet = ("meeting" in name and int(name[-2:]) == meeting.number - 1) + shares_date = (date == repr(meeting)[:10]) + if d.stem != repr(meeting) and (shares_meet or shares_date): + old_path = d + old_soln = d / "".join([d.stem, FileExtensions.Solutionbook]) + + new_path = old_path.with_name(repr(meeting)) + new_soln = old_path / "".join([repr(meeting), FileExtensions.Solutionbook]) + + # NOTE: rename internal files before renaming the folder + tqdm.write(f"renaming: {old_soln} -> {new_soln}") + old_soln.rename(new_soln) + tqdm.write(f"renaming: {old_path} -> {new_path}") + old_path.rename(new_path) + return + # TODO: handle meeting swaps (e.g. meeting02 with meeting 06, etc.) + # TODO: handle meeting renames with swaps + # TODO: allow for renaming to propagate through to platforms like Kaggle + + path.mkdir(exist_ok=True, parents=True) """ Use the following as "inspiration", this feels like it's far too complex to understand @@ -141,18 +150,24 @@ def update_or_create_notebook(meeting: Meeting, overwrite: bool = False): validator = TemplateNotebookValidator() _, _ = validator.from_meeting(meeting) - kernel_metadata = load_upkeep_template("meetings/kernel-metadata.json.j2") + kernel_metadata = templates.load("meeting/kernel-metadata.json.j2") - with open(paths.repo_meeting_folder(meeting) / "kernel-metadata.json", "w") as f: - meeting.optional["kaggle"]["competitions"].insert( - 0, kaggle.slug_competition(meeting) - ) - text = kernel_metadata.render( + meeting.optional["kaggle"]["competitions"].insert( + 0, kaggle.slug_competition(meeting) + ) + + # TODO get this sorted into proper JSON and templating + text = ( + kernel_metadata.render( slug=kaggle.slug_kernel(meeting), notebook=repr(meeting), kaggle=meeting.optional["kaggle"], ) - f.write(text.replace("'", '"')) # JSON doesn't like single-quotes + .replace("'", '"') # JSON doesn't like single-quoted strings + .lower() + ) + + open(paths.repo_meeting_folder(meeting) / "kernel-metadata.json", "w").write(text) # Converts a "Solutionbook" (`.solution.ipynb`) to a "Workbook" (`.ipynb`). # Makes use of Preprocessors and FileWriters to place generate and write the diff --git a/autobot/actions/paths.py b/autobot/actions/paths.py index 03a58c7..51581dc 100644 --- a/autobot/actions/paths.py +++ b/autobot/actions/paths.py @@ -1,5 +1,5 @@ -from pathlib import Path import os +from pathlib import Path from autobot import ORG_NAME from autobot.concepts import Group, Meeting diff --git a/autobot/actions/reader.py b/autobot/actions/reader.py index 5543a90..7f57eb1 100644 --- a/autobot/actions/reader.py +++ b/autobot/actions/reader.py @@ -1,16 +1,16 @@ import os import re +import signal import sys from datetime import datetime -from time import sleep from getpass import getpass -import signal from pathlib import Path +from time import sleep -from termcolor import colored, cprint - -import requests import pandas as pd +import requests + +from termcolor import colored, cprint __author__ = "John Muchovej" diff --git a/autobot/actions/syllabus.py b/autobot/actions/syllabus.py index d7509b4..77ea2a3 100644 --- a/autobot/actions/syllabus.py +++ b/autobot/actions/syllabus.py @@ -1,74 +1,135 @@ import yaml - from tqdm import tqdm +from autobot.apis import ucf +from autobot.pathing import templates, repositories +from autobot.concepts import Coordinator, Group, Meeting + +from . import paths + try: from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper -from autobot.concepts import Group, Coordinator, Meeting -from autobot.apis import ucf +import copy -from . import paths -"""Expected `syllabus.yml` format. You can also find this in `templates/seed/group/syllabus.yml`. - -```yaml -- required: - title: "" - cover: "" - filename: "" - instructors: [] - description: >- - - optional: - date: "" - room: "" - tags: [] - papers: [] - urls: - slides: "" - youtube: "" - kaggle: - datasets: [] - competitions: [] - kernels: [] - gpu: false -``` -""" +# TODO migrate from PyYAML to Confuse: https://github.com/beetbox/confuse + + +def init(group: Group): + path = repositories.local_semester_root(group) + + assert not (path / "syllabus.yml").exists(), "found: syllabus.yml" + assert (path / "overhead.yml").exists(), "found: overhead.yml" + + syllabus = yaml.load(open(templates.get("group/syllabus.yml"), "r"), Loader=Loader) + overhead = yaml.load(open(path / "overhead.yml", "r"), Loader=Loader)["meetings"] + + # assert ( + # overhead["start_offset"] >= 0 + # ), "Need to know the week we start this group to do anything else..." + + try: + schedule = ucf.make_schedule(group, overhead) + except AssertionError: + # don't require `overhead` to be properly filled out + schedule = [None] * ( + ucf.SEMESTER_LEN[group.semester.name.lower()] - overhead["start_offset"] + ) + finally: + meetings = {} + for idx, meeting in tqdm(enumerate(schedule), desc="Initial Meeting Setup"): + # TODO add support for non-standard meeting times + if hasattr(meeting, "date"): + syllabus["optional"]["date"] = meeting.date.isoformat() + + if hasattr(meeting, "room"): + syllabus["optional"]["room"] = meeting.room + + meetings[f"meeting{idx:02d}"] = copy.deepcopy(syllabus) + + yaml.dump( + meetings, + stream=open(path / "syllabus.yml", "w"), + Dumper=Dumper, + width=80, + sort_keys=False, + # default_style='"', + ) + + +def sort(group: Group): + path = repositories.local_semester_root(group) + + assert (path / "syllabus.yml").exists() + assert (path / "overhead.yml").exists() + + overhead = yaml.load(open(path / "overhead.yml", "r"), Loader=Loader)["meetings"] + schedule = ucf.make_schedule(group, overhead) + + syllabus_old = yaml.load(open(path / "syllabus.yml", "r"), Loader=Loader) + syllabus_new = {} + + # resort entries + for idx, previous in tqdm( + enumerate(syllabus_old.values()), desc="Resorting Syllabus" + ): + syllabus_new[f"meeting{idx:02d}"] = copy.deepcopy(previous) + + # re-date the entries + for meeting, info in tqdm(zip(syllabus_new.keys(), schedule), desc="Update Dates"): + syllabus_new[meeting]["optional"]["date"] = info.date.isoformat() + + yaml.dump( + syllabus_new, + stream=open(path / "syllabus.yml", "w"), + Dumper=Dumper, + width=80, + sort_keys=False, + # default_style='"', + ) def parse(group: Group): + path = repositories.local_semester_root(group) + # region 1. Read `overhead.yml` and seed Coordinators - overhead_yml = paths.repo_group_folder(group) / "overhead.yml" - overhead_yml = yaml.load(open(overhead_yml, "r"), Loader=Loader) - coordinators = overhead_yml["coordinators"] - setattr(group, "coords", Coordinator.parse_yaml(coordinators)) + overhead = yaml.load(open(path / "overhead.yml", "r"), Loader=Loader) + setattr(group, "coords", Coordinator.parse_yaml(overhead)) - meeting_overhead = overhead_yml["meetings"] - meeting_schedule = ucf.make_schedule(group, meeting_overhead) + # TODO validate dates follow the meeting pattern and ping Discord if not + overhead = overhead["meetings"] + schedule = ucf.make_schedule(group, overhead) # endregion # region 2. Read `syllabus.yml` and parse Syllabus - syllabus_yml = paths.repo_group_folder(group) / "syllabus.yml" - syllabus_yml = yaml.load(open(syllabus_yml, "r"), Loader=Loader) + syllabus = yaml.load(open(path / "syllabus.yml", "r"), Loader=Loader) - syllabus = [] + meetings = [] # TODO support undecided filenames - for meeting, schedule in tqdm( - zip(syllabus_yml, meeting_schedule), desc="Parsing Meetings" + for (key, meeting), when_where in tqdm( + zip(syllabus.items(), schedule), desc="Parsing Meetings" ): - try: - syllabus.append(Meeting(group, meeting, schedule)) - except AssertionError: - tqdm.write( - f"You're missing `required` fields from the meeting happening on {schedule.date} in {schedule.room}" - ) - continue + # implicitly trust `syllabus.yml` to be correct + if not meeting["optional"].get("room", False): + meeting["optional"]["room"] = when_where.room + + if not meeting["optional"].get("date", False): + meeting["optional"]["date"] = when_when.date.isoformat() + + meetings.append(Meeting(group, meeting, tmpname=key)) + # try: + # meetings.append(Meeting(group, meeting, key)) + # except AssertionError: + # tqdm.write( + # f"You're missing `required` fields from the meeting happening on {meeting.date} in {schedule.room}" + # ) + # continue # endregion - return syllabus + return meetings def write(group: Group): diff --git a/autobot/apis/github.py b/autobot/apis/github.py index 31c7996..ba3e333 100644 --- a/autobot/apis/github.py +++ b/autobot/apis/github.py @@ -1,8 +1,8 @@ +from github.GithubException import GithubException +from github.MainClass import Github from github.NamedUser import NamedUser as User from github.Organization import Organization as Org from github.Team import Team -from github.MainClass import Github -from github.GithubException import GithubException from autobot import ORG_NAME from autobot.concepts import Group diff --git a/autobot/apis/hugo.py b/autobot/apis/hugo.py index 8b13789..e69de29 100644 --- a/autobot/apis/hugo.py +++ b/autobot/apis/hugo.py @@ -1 +0,0 @@ - diff --git a/autobot/apis/kaggle.py b/autobot/apis/kaggle.py index e9eb0ad..0ac2182 100644 --- a/autobot/apis/kaggle.py +++ b/autobot/apis/kaggle.py @@ -1,18 +1,24 @@ +import json import os import shutil import subprocess from hashlib import sha256 from pathlib import Path +from typing import Union from tqdm import tqdm +import requests from autobot import ORG_NAME, KAGGLE_USERNAME from autobot.actions import paths from autobot.concepts import Meeting +from autobot.pathing import repositories from .nbconvert import FileExtensions KAGGLE_CONFIG_DIR = Path(__file__).parent.parent.parent +if os.environ.get("IN_DOCKER", False): + KAGGLE_CONFIG_DIR = "/autobot" def _configure_environment() -> None: @@ -24,9 +30,15 @@ def _configure_environment() -> None: tqdm.write(f"Found `KAGGLE_CONFIG_DIR = {os.environ['KAGGLE_CONFIG_DIR']}`") -def pull_kernel(meeting: Meeting) -> None: - _configure_environment() +def pull_kernel(meeting: Meeting) -> Union[None, Path]: + test_existence = requests.get( + f"https://kaggle.com/{ KAGGLE_USERNAME }/{ slug_kernel(meeting) }" + ) + if test_existence.status_code != requests.codes.OK: + return None + + _configure_environment() subprocess.run( " ".join( [ @@ -47,17 +59,20 @@ def local_and_remote_kernels_diff(meeting: Meeting) -> bool: _configure_environment() remote_kernel = pull_kernel(meeting) - local_kernel = (paths.repo_meeting_folder(meeting) / repr(meeting)).with_suffix( - FileExtensions.Workbook + if not remote_kernel: + return True + + local_kernel = repositories.local_meeting_root(meeting) / "".join( + [repr(meeting), FileExtensions.Workbook] ) - remote_kernel_hash = sha256(open(remote_kernel, "rb").read()).hexdigest() - local_kernel_hash = sha256(open(local_kernel, "rb").read()).hexdigest() + remote_kernel_json = json.dumps(json.load(open(remote_kernel, "r"))).encode("utf-8") + local_kernel_json = json.dumps(json.load(open(local_kernel, "r"))).encode("utf-8") - if str(remote_kernel).startswith(str(paths.repo_meeting_folder(meeting))): - shutil.rmtree(remote_kernel.parent) - else: - os.remove(remote_kernel) + remote_kernel_hash = sha256(bytes(remote_kernel_json)).hexdigest() + local_kernel_hash = sha256(bytes(local_kernel_json)).hexdigest() + + remote_kernel.unlink() # clean-up after diff return remote_kernel_hash != local_kernel_hash @@ -69,7 +84,7 @@ def push_kernel(meeting: Meeting) -> None: subprocess.run( f"kaggle k push -p {paths.repo_meeting_folder(meeting)}", shell=True, - stdout=subprocess.DEVNULL, + # stdout=subprocess.DEVNULL, ) else: tqdm.write(" - Kernels are the same. Skipping.") diff --git a/autobot/apis/nbconvert.py b/autobot/apis/nbconvert.py index 6a6f822..fa0e1d7 100644 --- a/autobot/apis/nbconvert.py +++ b/autobot/apis/nbconvert.py @@ -4,17 +4,14 @@ import nbformat from nbconvert.exporters import MarkdownExporter, NotebookExporter from nbconvert.exporters.exporter import ResourcesDict -from nbconvert.preprocessors import ( - Preprocessor, - TagRemovePreprocessor, -) +from nbconvert.preprocessors import Preprocessor, TagRemovePreprocessor from nbconvert.writers import FilesWriter from nbformat import NotebookNode from nbgrader.preprocessors import ClearOutput, ClearSolutions from autobot.actions import paths from autobot.concepts import Meeting -from autobot.pathing import templates, urlgen +from autobot.pathing import templates, urlgen, repositories class FileExtensions: @@ -26,7 +23,7 @@ def __init__(self): def read_notebook(meeting: Meeting, suffix: str = FileExtensions.Solutionbook): - notebook_path = paths.repo_meeting_folder(meeting) / repr(meeting) + notebook_path = repositories.local_meeting_root(meeting) / repr(meeting) if not notebook_path.with_suffix(suffix).exists(): return nbformat.v4.new_notebook() else: @@ -39,7 +36,7 @@ class SolutionbookToPostExporter(MarkdownExporter): """ def __init__(self, config=None, **kwargs): - self.template_path = [str(templates.get_upkeep("meetings"))] + self.template_path = [str(templates.get("meeting"))] self.template_file = "to-post.md" self.template_extension = ".j2" @@ -71,12 +68,12 @@ def __init__(self, config=None, **kwargs): # self.register_filter("highlight_code", Highlight2HTML(parent=self)) def from_meeting(self, meeting: Meeting): - notebook_path = paths.repo_meeting_folder(meeting) / "".join( + notebook_path = repositories.local_meeting_root(meeting) / "".join( [repr(meeting), FileExtensions.Solutionbook] ) # TODO concatenate front matter to notebook output - front_matter = templates.load_upkeep("meetings/hugo-front-matter.md.j2") + front_matter = templates.load("meeting/hugo-front-matter.md.j2") front_matter = front_matter.render( **{ "group": repr(meeting.group), @@ -89,6 +86,8 @@ def from_meeting(self, meeting: Meeting): "tags": meeting.optional["tags"], "description": meeting.required["description"], "weight": meeting.number, + "room": meeting.meta.room, + "cover": meeting.required["cover"], }, "semester": { "full": str(meeting.group.semester), @@ -136,8 +135,12 @@ def from_meeting(self, meeting: Meeting): resources = {"output_extension": FileExtensions.Workbook} notebook, resources = super().from_notebook_node(nb, resources) - writer = FilesWriter(build_directory=str(paths.repo_meeting_folder(meeting))) - writer.write(json.dumps(notebook), resources, repr(meeting)) + # writer = FilesWriter(build_directory=str(paths.repo_meeting_folder(meeting))) + # writer.write(json.dumps(notebook), resources, repr(meeting)) + filename = ( + repositories.local_meeting_root(meeting) / repr(meeting) + ).with_suffix(FileExtensions.Workbook) + open(filename, "w").write(notebook) return notebook, resources @@ -157,20 +160,25 @@ def from_meeting(self, meeting: Meeting): nb = read_notebook(meeting) resources = {"output_extension": FileExtensions.Solutionbook} - notebook, resources = super(NotebookExporter, self).from_notebook_node( - nb, resources=resources - ) - notebook.cells.insert(0, self._notebook_heading()) + # NotebookExporter.from_notebook_node returns a notebook as a string + notebook, resources = super().from_notebook_node(nb, resources=resources) - writer = FilesWriter(build_directory=str(paths.repo_meeting_folder(meeting))) + # to operate over the notebook, we need it to be a NotebookNode + notebook = nbformat.reads(notebook, as_version=4) + notebook.cells.insert(0, self._notebook_heading()) - writer.write(json.dumps(notebook), resources, repr(meeting)) + # to write to disk, it now needs to be a string + notebook = nbformat.writes(notebook) + filename = ( + repositories.local_meeting_root(meeting) / repr(meeting) + ).with_suffix(FileExtensions.Solutionbook) + open(filename, "w").write(notebook) return notebook, resources def _notebook_heading(self) -> nbformat.NotebookNode: - tpl_heading = templates.load_upkeep("meetings/notebook-heading.html.j2") + tpl_heading = templates.load("meeting/notebook-heading.html.j2") tpl_args = { "group_sem": paths.repo_meeting_folder(self.meeting), diff --git a/autobot/apis/ucf.py b/autobot/apis/ucf.py index e12b2f2..52912ba 100644 --- a/autobot/apis/ucf.py +++ b/autobot/apis/ucf.py @@ -1,13 +1,11 @@ from collections import OrderedDict -from typing import List, Dict +from datetime import datetime +from typing import Dict, List import pandas as pd import requests -from datetime import datetime - -from autobot.concepts import MeetingMeta -from autobot.concepts import Group, Semester +from autobot.concepts import Group, MeetingMeta, Semester # it's unlikely this URL will change, but should be occassionally checked CALENDAR_URL = "https://calendar.ucf.edu" @@ -18,10 +16,13 @@ "fall": ["Veterans Day", "Labor Day", "Thanksgiving"], } +# NOTE make sure to consider 0-indexing here +SEMESTER_LEN = {"spring": 14, "summer": 10, "fall": 15} + def day2index(s: str) -> int: - weekdays = "Mon Tue Wed Thu Fri".split() - return weekdays.index(s) + weekdays = "Mon Tue Wed Thu Fri Sat Sun".lower().split() + return weekdays.index(s.lower()) def make_schedule(group: Group, schedule: Dict): @@ -33,7 +34,8 @@ def make_schedule(group: Group, schedule: Dict): # generate meeting dates, on a weekly basis meeting_dates = pd.Series(date_range) - meeting_start = (offset - 1) * 7 + day2index(wday) + meeting_start = offset * 7 + day2index(wday) + # TODO: support non-standard dates for groups, e.g. like "Supplementary" meeting_dates = meeting_dates[meeting_start::7] if holidays is not None: diff --git a/autobot/concepts/__init__.py b/autobot/concepts/__init__.py index 1697fb6..5201262 100755 --- a/autobot/concepts/__init__.py +++ b/autobot/concepts/__init__.py @@ -2,9 +2,10 @@ MeetingMeta = namedtuple("MeetingMeta", ["date", "room"]) -from .semester import Semester +from .coordinator import Coordinator from .group import Group from .meeting import Meeting -from .coordinator import Coordinator +from .semester import Semester + __all__ = ["Meeting", "Groups", "Coordinator", "Semester", "MeetingMeta"] diff --git a/autobot/concepts/coordinator.py b/autobot/concepts/coordinator.py index 80a489c..8156374 100755 --- a/autobot/concepts/coordinator.py +++ b/autobot/concepts/coordinator.py @@ -14,10 +14,14 @@ def __init__(self, github: str, role: str) -> None: @staticmethod def parse_yaml(d: Dict) -> Dict: - if "coordinators" in d: - d = d["coordinators"] + assert ( + "directors" in d and "coordinators" in d + ), "Currently, we need both director and coordinator sections" - return {c["github"].lower(): Coordinator(**c) for c in d} + ls = [] + ls += d["directors"] + d["coordinators"] + + return {c.lower(): Coordinator(c.lower(), "Coordinator") for c in ls} # def _github_request(self) -> None: # user = get_github_user(self.github) diff --git a/autobot/concepts/group.py b/autobot/concepts/group.py index de8484d..636f0cf 100755 --- a/autobot/concepts/group.py +++ b/autobot/concepts/group.py @@ -8,11 +8,14 @@ log.setLevel(logging.DEBUG) +# https://pyyaml.org/wiki/PyYAMLDocumentation#constructors-representers-resolvers class Group: def __init__(self, name: str, sem_meta: Semester) -> None: self.name = name self.semester = sem_meta self.coords = None + self.push_kaggle = True + self.make_notebooks = True def __str__(self) -> str: return f"{self.name} Group" diff --git a/autobot/concepts/groups.py b/autobot/concepts/groups.py index 8e4c5ca..9cf2952 100755 --- a/autobot/concepts/groups.py +++ b/autobot/concepts/groups.py @@ -2,7 +2,6 @@ new group!""" from autobot.concepts import Semester - from .group import Group diff --git a/autobot/concepts/meeting.py b/autobot/concepts/meeting.py index a94ef0e..63ec213 100755 --- a/autobot/concepts/meeting.py +++ b/autobot/concepts/meeting.py @@ -1,26 +1,24 @@ +import copy +import datetime as dt import io import os -import copy -import datetime import shutil import subprocess -from pathlib import Path +from distutils.dir_util import copy_tree from hashlib import sha256 -from typing import List, Dict from itertools import product -from distutils.dir_util import copy_tree +from pathlib import Path +from typing import Dict, List import imgkit +import nbconvert as nbc +import nbformat as nbf +import pandas as pd import requests import yaml -from PIL import Image -import pandas as pd from jinja2 import Template -import nbformat as nbf -import nbconvert as nbc -from nbgrader.preprocessors import ClearSolutions, ClearOutput - -from autobot import ORG_NAME +from nbgrader.preprocessors import ClearOutput, ClearSolutions +from PIL import Image from . import MeetingMeta from .coordinator import Coordinator @@ -29,9 +27,20 @@ src_dir = Path(__file__).parent.parent +# https://pyyaml.org/wiki/PyYAMLDocumentation#constructors-representers-resolvers class Meeting: - def __init__(self, group: Group, meeting_dict: Dict, meta: MeetingMeta): + def __init__( + self, + group: Group, + meeting_dict: Dict, + tmpname: str = None, + meta: MeetingMeta = None, + ): assert set(["required", "optional"]).intersection(set(meeting_dict.keys())) + self.number = 0 + if tmpname: + self.number = int(tmpname.replace("meeting", "")) + self.number += 1 self.group = group self.required = meeting_dict["required"] @@ -39,6 +48,8 @@ def __init__(self, group: Group, meeting_dict: Dict, meta: MeetingMeta): if "date" in self.optional and self.optional["date"]: date = self.optional["date"] + # mm, dd = self.optional["date"].split("-") + # date = dt.date(int(self.group.semester.year), int(mm), int(dd)) else: date = meta.date @@ -47,14 +58,17 @@ def __init__(self, group: Group, meeting_dict: Dict, meta: MeetingMeta): else: room = meta.room - self.meta = MeetingMeta(date, room) + self.meta = MeetingMeta(pd.to_datetime(date), room) self.required["instructors"] = [x.lower() for x in self.required["instructors"]] - for key in self.required.keys(): - assert self.required[ - key - ], f"You haven't specified `{key}` for this meeting..." + self.required["filename"] = self.required["filename"] or tmpname + self.required["title"] = self.required["title"] or tmpname + + # for key in self.required.keys(): + # assert self.required[ + # key + # ], f"You haven't specified `{key}` for this meeting..." def write_yaml(self) -> Dict: """This prepares the dict to write each entry in `syllabus.yml`.""" diff --git a/autobot/main.py b/autobot/main.py index 03a92b1..8ddb874 100755 --- a/autobot/main.py +++ b/autobot/main.py @@ -2,16 +2,16 @@ import logging import os import sys +import shutil from argparse import ArgumentParser -from distutils.dir_util import copy_tree -from jinja2 import Template from tqdm import tqdm -from autobot import ORG_NAME, get_setup_template +from autobot import ORG_NAME from autobot.actions import meetings, paths, syllabus -from autobot.apis import ucf +from autobot.apis import ucf, kaggle from autobot.concepts import Group, Meeting, Semester, groups +from autobot.pathing import templates, repositories def _argparser(**kwargs): @@ -57,7 +57,16 @@ def main(): if args.action == "semester-setup": semester_setup(group) elif args.action == "semester-upkeep": + try: + syllabus.init(group) + print("Done generating syllabus. Exiting.") + exit(0) + except AssertionError as e: + pass + + syllabus.sort(group) meetings = syllabus.parse(group) + if not args.all: meeting = None if args.date: @@ -89,31 +98,30 @@ def semester_setup(group: Group) -> None: 3. Performs a similar setup with Google Drive & Google Forms. 4. Generates skeleton for the login/management system. """ - if paths.repo_group_folder(group).exists(): - logging.warning(f"{paths.repo_group_folder(group)} exists! Tread carefully.") + path = repositories.local_semester_root(group) + if path.exists(): + logging.warning(f"{path} exists! Tread carefully.") overwrite = input( "The following actions **are destructive**. " "Continue? [y/N] " ) if overwrite.lower() not in ["y", "yes"]: return + path.mkdir() + # region 1. Copy base `yml` files. # 1. env.yml # 2. overhead.yml - # 3. syllabus.yml - # strong preference to use `shutil`, but can't use with existing dirs - # shutil.copytree("autobot/templates/seed/meeting", path.parent) - copy_tree(get_setup_template("group"), str(paths.repo_group_folder(group))) - - env_yml = paths.repo_group_folder(group) / "env.yml" - env = Template(open(env_yml, "r").read()) - - with open(env_yml, "w") as f: + env = templates.load("group/env.yml.j2") + (path / "env.yml").touch() + with open(path / "env.yml", "w") as f: f.write( env.render( org_name=ORG_NAME, group_name=repr(group), semester=repr(group.semester) ) ) + + shutil.copy(str(templates.get("group/overhead.yml")), str(path / "overhead.yml")) # endregion # region 2. Setup Website for this semester @@ -144,12 +152,12 @@ def semester_upkeep(syllabus: List[Meeting], overwrite: bool = False) -> None: tqdm.write(f"{repr(meeting)} ~ {str(meeting)}") # Perform initial directory checks/clean-up - # meetings.update_or_create_folders_and_files(meeting) + meetings.update_or_create_folders_and_files(meeting) # Make edit in the group-specific repo meetings.update_or_create_notebook(meeting, overwrite=overwrite) meetings.download_papers(meeting) - # kaggle.push_kernel(meeting) + kaggle.push_kernel(meeting) # Make edits in the ucfai.org repo # banners.render_cover(meeting) diff --git a/autobot/pathing/hugo.py b/autobot/pathing/hugo.py index 653227d..72fd18d 100644 --- a/autobot/pathing/hugo.py +++ b/autobot/pathing/hugo.py @@ -1,6 +1,6 @@ from pathlib import Path -from autobot.concepts import Group +from autobot.concepts import Coordinator, Group, Meeting CONTENT_ROOT = Path("content/") diff --git a/autobot/pathing/repositories.py b/autobot/pathing/repositories.py index dad5c6c..ed9ab89 100644 --- a/autobot/pathing/repositories.py +++ b/autobot/pathing/repositories.py @@ -1,26 +1,47 @@ +import os from pathlib import Path +from typing import Union from autobot import ORG_NAME from autobot.concepts import Group, Meeting + LOCAL_CONTENT_ROOT = Path("groups") +if os.environ.get("IN_DOCKER", False): + LOCAL_CONTENT_ROOT = Path("/ucfai") + REMOTE_CONTENT_PLATFORM = "https://github.com" REMOTE_CONTENT_ROOT = "/".join([REMOTE_CONTENT_PLATFORM, ORG_NAME]) -def local_group_root(group: Group): +def local_group_root(group: Group) -> Path: return LOCAL_CONTENT_ROOT / repr(group) -def local_semester_root(meeting: Meeting): - return local_group_root(meeting.group) / repr(meeting.group.semester) +def local_semester_root(meeting_or_group: Union[Group, Meeting]) -> Path: + if isinstance(meeting_or_group, Group): + group = meeting_or_group + elif isinstance(meeting_or_group, Meeting): + group = meeting_or_group.group + else: + raise ValueError("Didn't receive a `Meeting` or `Group`.") + + return local_group_root(group) / repr(group.semester) + + +def local_meeting_root(meeting: Meeting): + return local_semester_root(meeting) / repr(meeting) + + +def local_meeting_file(meeting: Meeting): + return lcoal_meeting_root(meeting) / repr(meeting) -def remote_group_root(group: Group): +def remote_group_root(group: Group) -> str: return "/".join([REMOTE_CONTENT_ROOT, repr(group)]) -def remote_semester_root(meeting: Meeting, branch: str = "master"): +def remote_semester_root(meeting: Meeting, branch: str = "master") -> str: return "/".join( [ remote_group_root(meeting.group), # Git URL @@ -31,7 +52,7 @@ def remote_semester_root(meeting: Meeting, branch: str = "master"): ) -def remote_meeting_file(meeting: Meeting): +def remote_meeting_file(meeting: Meeting) -> str: return "/".join( [ remote_semester_root(meeting), # Git URL semeseter root diff --git a/autobot/pathing/urlgen.py b/autobot/pathing/urlgen.py index 4ddafb3..d6daa03 100644 --- a/autobot/pathing/urlgen.py +++ b/autobot/pathing/urlgen.py @@ -10,22 +10,26 @@ def youtube(meeting: Meeting): # YouTube URLs take the following form: # https://www.youtube.com/watch?v=dQw4w9WgXcQ + # https://youtu.be/dQw4w9WgXcQ try: url = meeting.optional["urls"]["youtube"] except KeyError: return "" - # YouTube IDs are likely to state 11-characters, but we'll see: + # YouTube IDs are likely to stay 11-characters, but we'll see: # https://stackoverflow.com/a/6250619 - if len(url) > 11 and "youtube" in url: + if len(url) > 11 and "youtu" in url: # remove the protocol, www, and YouTube's domain name - yt_base_url = "(?:https?://)?(?:www\.)?youtube.com" + proto = "(?:https?://)?" + ln_old = "(?:www\.)?youtube.com/watch?v=" + ln_new = "youtu.be" + yt_full = f"{proto}(?:{ln_old}|{ln_new})" # "...|$" returns the empty string if not a match - url = re.sub(f"{yt_base_url}|$", "", url) - url = re.search("(?<=/watch?v=)([A-Za-z0-9-_]{11})", url).group(0) - url = f"https://youtube.com/watch?v={url}" + url = re.sub(f"{yt_full}|$", "", url) + url = re.search("([A-Za-z0-9-_]{11})", url).group(0) + url = f"https://youtu.be/{url}" if requests.get(url).status_code == requests.codes.ALL_OK: return url @@ -46,7 +50,7 @@ def slides(meeting: Meeting): docs_base_url = "(?:https?://)?docs.google.com/presentation/d/" # "...|$" returns the empty string if not a match - url = re.rub(f"{docs_base_url}|$", "", url) + url = re.sub(f"{docs_base_url}|$", "", url) url = url.split("/", maxsplit=1)[0] url = f"https://docs.google.com/presentation/d/{url}" if requests.get(url).status_code == requests.codes.ALL_OK: diff --git a/autobot/templates/coordinator/_index.md.j2 b/autobot/templates/coordinator/_index.md.j2 new file mode 100644 index 0000000..d7f05e2 --- /dev/null +++ b/autobot/templates/coordinator/_index.md.j2 @@ -0,0 +1,40 @@ +--- +name: {{ github }} + +github: {{ github }} + +ucfai: + roles: + - "" + teams: + - "" + +labs: + - "" + +authors: ['{{ github }}'] + +role: "" + +organizations: + +bio: >- + +interests: + - "" + +education: + courses: + - course: in + institution: University of Central Florida + year: "" + +social: + - icon: "github" + icon_pack: "fab" + url: "https://github.com/{{ github }}" + +user_groups: + - "" + +--- \ No newline at end of file diff --git a/autobot/templates/group/_index.md.j2 b/autobot/templates/group/_index.md.j2 new file mode 100644 index 0000000..8d9ca88 --- /dev/null +++ b/autobot/templates/group/_index.md.j2 @@ -0,0 +1,28 @@ +--- +linktitle: "Core: Fall 2019 Edition" +summary: >- + This semester we continued polishing our material - focusing on cultivating group + interaction and working to solidify your understanding of the topics we're covering. + We also introduced Deep Reinforcement Learning and Computational Cognitive Science as + topics to move us towards a broader understanding of both cutting-edge research and + the begin moving us back to our goals of covering Artificial Intelligence, Data + Science, and Cognitive Science. +weight: 999996 + +title: "Core: Fall 2019 Edition" +date: "2018-09-09T00:00:00Z" +lastmod: "2018-09-09T00:00:00Z" + +draft: false +toc: true +type: docs + +menu: + core_fa19: + name: Fall 2019 + weight: 1 +--- + +{{% alert note %}} +This semester, we met at **5:30pm** on **Wednesdays** in **MSB 359**. +{{% /alert %}} diff --git a/autobot/templates/setup/group/env.yml b/autobot/templates/group/env.yml.j2 similarity index 100% rename from autobot/templates/setup/group/env.yml rename to autobot/templates/group/env.yml.j2 diff --git a/autobot/templates/setup/group/overhead.yml b/autobot/templates/group/overhead.yml similarity index 82% rename from autobot/templates/setup/group/overhead.yml rename to autobot/templates/group/overhead.yml index 81d878a..44dd64b 100644 --- a/autobot/templates/setup/group/overhead.yml +++ b/autobot/templates/group/overhead.yml @@ -16,4 +16,4 @@ meetings: wday: "" # what day of the week? (Mon, Tue, Wed, Thu, Fri, Sat, Sun) time: "" # military time (e.g. 1800-2000) room: "" # e.g. HEC 119 - start_offset: 3 # how many weeks into the semester should we start? \ No newline at end of file + start_offset: 2 # what week of the semester should we start? (0-indexed) \ No newline at end of file diff --git a/autobot/templates/group/syllabus.yml b/autobot/templates/group/syllabus.yml new file mode 100644 index 0000000..7097085 --- /dev/null +++ b/autobot/templates/group/syllabus.yml @@ -0,0 +1,21 @@ +required: + title: "" + cover: "" + filename: "" + instructors: [] + description: >- + +optional: + date: "" + room: "" + tags: [] + papers: # this should read like a dictionary with: : + + urls: + slides: "" + youtube: "" + kaggle: + datasets: [] + competitions: [] + kernels: [] + enable_gpu: false \ No newline at end of file diff --git a/autobot/templates/upkeep/meetings/hugo-front-matter.md.j2 b/autobot/templates/meeting/hugo-front-matter.md.j2 similarity index 89% rename from autobot/templates/upkeep/meetings/hugo-front-matter.md.j2 rename to autobot/templates/meeting/hugo-front-matter.md.j2 index 7a7fe33..e35293b 100644 --- a/autobot/templates/upkeep/meetings/hugo-front-matter.md.j2 +++ b/autobot/templates/meeting/hugo-front-matter.md.j2 @@ -25,8 +25,11 @@ urls: kaggle: "{{ urls['kaggle'] }}" colab: "{{ urls['colab'] }}" +location: "{{ meeting['room'] }}" +cover: "{{ meeting['cover'] }}" + categories: ["{{ semester['short'] }}"] tags: [{% for tag in meeting['tags'] %}"{{ tag }}", {% endfor %}] -description: >- +abstract: >- {{ meeting['description'] }} ---- \ No newline at end of file +--- diff --git a/autobot/templates/upkeep/meetings/jekyll-front-matter.md.j2 b/autobot/templates/meeting/jekyll-front-matter.md.j2 similarity index 100% rename from autobot/templates/upkeep/meetings/jekyll-front-matter.md.j2 rename to autobot/templates/meeting/jekyll-front-matter.md.j2 diff --git a/autobot/templates/setup/meeting/kernel-metadata.json.j2 b/autobot/templates/meeting/kernel-metadata.json.j2 similarity index 87% rename from autobot/templates/setup/meeting/kernel-metadata.json.j2 rename to autobot/templates/meeting/kernel-metadata.json.j2 index 1ef3d6f..861bcb3 100644 --- a/autobot/templates/setup/meeting/kernel-metadata.json.j2 +++ b/autobot/templates/meeting/kernel-metadata.json.j2 @@ -5,7 +5,7 @@ "language": "python", "kernel_type": "notebook", "is_private": false, - "enable_gpu": {{ kaggle.enable_gpu }}, + "enable_gpu": {{ kaggle.enable_gpu | lower }}, "enable_internet": true, "dataset_sources": {{ kaggle.datasets }}, "competition_sources": {{ kaggle.competitions }}, diff --git a/autobot/templates/upkeep/meetings/notebook-heading.html.j2 b/autobot/templates/meeting/notebook-heading.html.j2 similarity index 100% rename from autobot/templates/upkeep/meetings/notebook-heading.html.j2 rename to autobot/templates/meeting/notebook-heading.html.j2 diff --git a/autobot/templates/upkeep/meetings/to-post.md.j2 b/autobot/templates/meeting/to-post.md.j2 similarity index 100% rename from autobot/templates/upkeep/meetings/to-post.md.j2 rename to autobot/templates/meeting/to-post.md.j2 diff --git a/autobot/templates/setup/group/syllabus.yml b/autobot/templates/setup/group/syllabus.yml deleted file mode 100644 index 36290e7..0000000 --- a/autobot/templates/setup/group/syllabus.yml +++ /dev/null @@ -1,23 +0,0 @@ -meeting: - required: - title: "" - cover: "" - filename: "" - instructors: [] - description: >- - - optional: - date: "" - room: "" - tags: [] - slides: "" - papers: # this should read like a dictionary with: : - - urls: - slides: "" - youtube: "" - kaggle: - datasets: [] - competitions: [] - kernels: [] - enable_gpu: false \ No newline at end of file diff --git a/autobot/templates/setup/hugo/author.md.j2 b/autobot/templates/setup/hugo/author.md.j2 deleted file mode 100644 index e69de29..0000000 diff --git a/autobot/templates/setup/hugo/group.md.j2 b/autobot/templates/setup/hugo/group.md.j2 deleted file mode 100644 index e69de29..0000000 diff --git a/autobot/templates/upkeep/banners/event.html.j2 b/autobot/templates/upkeep/banners/event.html.j2 deleted file mode 100755 index 0510645..0000000 --- a/autobot/templates/upkeep/banners/event.html.j2 +++ /dev/null @@ -1,184 +0,0 @@ - - - - - - - - - diff --git a/autobot/templates/upkeep/banners/instagram.html b/autobot/templates/upkeep/banners/instagram.html deleted file mode 100644 index b4bb1b5..0000000 --- a/autobot/templates/upkeep/banners/instagram.html +++ /dev/null @@ -1,248 +0,0 @@ - - - - - - - - - diff --git a/autobot/templates/upkeep/banners/particles-config.js b/autobot/templates/upkeep/banners/particles-config.js deleted file mode 100644 index 717629c..0000000 --- a/autobot/templates/upkeep/banners/particles-config.js +++ /dev/null @@ -1,68 +0,0 @@ -particlesJS("particles-js", { - particles: { - number: { - value: 260, - density: { enable: true, value_area: 481 } - }, - color: { value: "#fff000" }, - shape: { - type: "circle", - stroke: { width: 0, color: "#000000" }, - polygon: { nb_sides: 5 }, - image: { src: "img/github.svg", width: 100, height: 100 } - }, - opacity: { - value: 0.8, - random: true, - anim: { enable: true, speed: 1, opacity_min: 0.1, sync: false } - }, - size: { - value: 4, - random: true, - anim: { enable: true, speed: 4, size_min: 0.4, sync: false } - }, - line_linked: { - enable: true, - distance: 78, - color: "#fdde3c", - opacity: 0.2, - width: 1 - }, - detect_on: "window", - move: { - enable: true, - speed: 2, - direction: "none", - random: true, - straight: false, - out_mode: "out", - bounce: false, - attract: { enable: true, rotateX: 1420, rotateY: 600 } - } - }, - interactivity: { - detect_on: "window", - events: { - onhover: { enable: true, mode: "grab" }, - onclick: { enable: false, mode: "bubble" }, - resize: true - }, - modes: { - grab: { - distance: 100, - line_linked: { opacity: 0.6 } - }, - bubble: { - distance: 146.17389821424212, - size: 10, - duration: 2, - opacity: 0, - speed: 3 - }, - repulse: { distance: 400, duration: 0.4 }, - push: { particles_nb: 4 }, - remove: { particles_nb: 2 } - } - }, - retina_detect: true -}); diff --git a/autobot/templates/upkeep/banners/video.html.j2 b/autobot/templates/upkeep/banners/video.html.j2 deleted file mode 100644 index d0a598d..0000000 --- a/autobot/templates/upkeep/banners/video.html.j2 +++ /dev/null @@ -1,245 +0,0 @@ - - - - - - - - - diff --git a/autobot/templates/upkeep/meetings/kernel-metadata.json.j2 b/autobot/templates/upkeep/meetings/kernel-metadata.json.j2 deleted file mode 100644 index 1ef3d6f..0000000 --- a/autobot/templates/upkeep/meetings/kernel-metadata.json.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{ - "id": "ucfaibot/{{ slug }}", - "title": "{{ slug }}", - "code_file": "{{ notebook }}.ipynb", - "language": "python", - "kernel_type": "notebook", - "is_private": false, - "enable_gpu": {{ kaggle.enable_gpu }}, - "enable_internet": true, - "dataset_sources": {{ kaggle.datasets }}, - "competition_sources": {{ kaggle.competitions }}, - "kernel_sources": {{ kaggle.kernels }} -} diff --git a/docker-compose.yml b/docker-compose.yml index 6bd10c6..88ff812 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,9 @@ services: - ./docker/volume.ucfai.org/:/ucfai.org/ - ./docker/container.autobot:/docker - autobot-production: - container_name: autobot-production - image: docker.pkg.github.com/ucfai/bot/production:latest + # autobot-production: + # container_name: autobot-production + # image: docker.pkg.github.com/ucfai/bot/production:latest autobot-hugo: image: jojomi/hugo:0.60.0 @@ -20,6 +20,11 @@ services: - HUGO_THEME=academic - HUGO_WATCH=true # - HUGO_BASEURL= + command: + - /run.sh # from Dockerfile + - --buildFuture + - --disableFastRender + - --verbose volumes: - ./docker/volume.ucfai.org:/src # use the volume mount below to inspect the hugo output, IFF needed @@ -37,4 +42,4 @@ services: # volumes: # - .:/autobot # - ./docker/container.reader:/docker - # - ./docker/volume.reader:/reader \ No newline at end of file + # - ./docker/volume.reader:/reader diff --git a/docker/container.autobot/env.yml b/docker/container.autobot/env.yml index a07ff8e..2015ded 100755 --- a/docker/container.autobot/env.yml +++ b/docker/container.autobot/env.yml @@ -2,26 +2,19 @@ name: autobot channels: - defaults - conda-forge - - bioconda dependencies: - python=3.7 - - argcomplete - black - jinja2 - - jupyter - nbconvert - nbformat - nbgrader - pandas - numpy - pip - # - pygithub=1.39 - pyyaml - pillow - requests - - termcolor - tqdm - - wkhtmltopdf - pip: - - imgkit - kaggle diff --git a/setup.py b/setup.py index 3c8972f..02208db 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,5 @@ "PyYAML", "requests", "nbconvert", - "pygithub", ], )