Merge branch 'master' into zshkoor/remove-poviding-args

.github/dependabot.yml

@@ -5,3 +5,5 @@ updates:
     directory: "/"
     schedule:
         interval: "weekly"
+    reviewers:
+      - "openedx/arbi-bom"

.github/workflows/ci.yml

@@ -15,9 +15,9 @@ jobs:
         python-version: ['3.8']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: setup python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/pypi-publish.yml

@@ -12,9 +12,9 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
 

.gitignore

@@ -23,5 +23,3 @@ edx_repo_tools.egg-info
 .idea/
 
 build/
-
-fake_repo*

Makefile

@@ -38,6 +38,5 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)  ## update the requirements/*.txt files with
 	pip-compile --upgrade -o requirements/development.txt requirements/development.in
 	for fextra in edx_repo_tools/*/extra.in; do pip-compile --upgrade -o $${fextra%.in}.txt $$fextra; done
 
-lint:				## run pep8 and pylint
-	pep8 || true
-	pylint *.py edx_repo_tools tests || true
+lint:				## run pylint
+	pylint *.py edx_repo_tools tests

barcalendar.py

@@ -19,7 +19,11 @@
 import colorsys
 import datetime
 import itertools
+import re
+import time
 
+import requests
+import yaml
 
 def css_to_rgb(hex):
     assert hex[0] == "#"
@@ -135,14 +139,14 @@ def years_months(self):
                 for (m = 0; m < 12; m++) {{
                     sheet.getRange({monthrow}, {iyear}+m).setValue("JFMAMJJASOND"[m]);
                 }}
-                """);
+                """)
         print(f"""\
             sheet.getRange({yearrow}, 1, 1, {self.width})
                 .setFontWeight("bold")
                 .setHorizontalAlignment("center");
             sheet.getRange({monthrow}, 1, 1, {self.width})
                 .setHorizontalAlignment("center");
-            """);
+            """)
         print(f"""\
             var rules = sheet.getConditionalFormatRules();
             rules.push(
@@ -245,6 +249,52 @@ def column_marker(self, column):
     def write(self):
         self.epilog()
 
+def get_defaults_from_tutor():
+    """
+    Fetches default configurations from tutor repository.
+    Returns:
+        object: Default configurations as Python object
+    """
+    url = "https://raw.githubusercontent.com/overhangio/tutor/master/tutor/templates/config/defaults.yml"
+    while True:
+        try:
+            resp = requests.get(url, timeout=10)
+        except requests.RequestException as exc:
+            print(f"Couldn't fetch {url}: {exc}")
+            raise
+        if resp.status_code == 429:
+            wait = int(resp.headers.get("Retry-After", 10))
+            time.sleep(wait + 1)
+        else:
+            break
+
+    if resp.status_code == 200:
+        return yaml.safe_load(resp.text)
+    resp.raise_for_status()
+
+def parse_version_number(line):
+    """
+    Get version number in line from YAML file.
+    Note that this only captures major and minor version (not patch number).
+    e.g. "docker.io/elasticsearch:7.17.9" -> "7.17"
+    """
+    match = re.search(r'(?P<version_number>\d+(\.\d+)?)', line)
+    if match is not None:
+        version_number = match.group("version_number")
+        return version_number
+    raise ValueError(f"Couldn't get version number from {line!r}")
+
+def parse_version_name(line):
+    """
+    Get openedx version name in line from YAML file.
+    e.g.1 "open-release/palm.1" -> "Palm"
+    e.g.2 "open-release/palm.master" -> "Palm"
+    """
+    match = re.search(r'/(?P<version_name>[A-Za-z]+)\.', line)
+    if match is not None:
+        version_name = match.group("version_name")
+        return version_name.capitalize()
+    raise ValueError(f"Couldn't get version name from {line!r}")
 
 # ==== Editable content ====
 
@@ -253,17 +303,19 @@ def write(self):
 END_YEAR = 2027
 LTS_ONLY = True
 
+versions = get_defaults_from_tutor()
+
 # The current versions of everything.  Use the same strings as the keys in the various sections below.
 CURRENT = {
-    "Open edX": "Palm",
+    "Open edX": parse_version_name(versions['OPENEDX_COMMON_VERSION']),
     "Python": "3.8",
     "Django": "3.2",
     "Ubuntu": "20.04",
     "Node": "16.x",
-    "Mongo": "4.2",
-    "MySQL": "5.7",
-    "Elasticsearch": "7.10",
-    "Redis": "5.6",
+    "Mongo": parse_version_number(versions['DOCKER_IMAGE_MONGODB']),
+    "MySQL": parse_version_number(versions['DOCKER_IMAGE_MYSQL']),
+    "Elasticsearch": parse_version_number(versions['DOCKER_IMAGE_ELASTICSEARCH']),
+    "Redis": parse_version_number(versions['DOCKER_IMAGE_REDIS']),
     "Ruby": "3.0",
 }
 
@@ -440,11 +492,12 @@ def write(self):
     # ('2.4', 2016, 8, 2018, 2),
     # ('5.6', 2017, 9, 2019, 3),
     # ('6.8', 2019, 5, 2020, 11),
-    ('7.8', 2020, 6, 2021, 12),
+    # ('7.8', 2020, 6, 2021, 12),
     ('7.10', 2020, 11, 2022, 5),
     ('7.11', 2021, 2, 2022, 8),
     ('7.12', 2021, 3, 2022, 9),
     ('7.13', 2021, 5, 2022, 11),
+    ('7.17', 2022, 2, 2023, 8),
 ]
 for name, syear, smonth, eyear, emonth in es_releases:
     cal.bar(f"Elasticsearch {name}", start=(syear, smonth), end=(eyear, emonth), color="#4595ba", current=(name==CURRENT["Elasticsearch"]))

edx_repo_tools/conventional_commits/commitstats.py

@@ -3,7 +3,6 @@
 """
 
 import csv
-import datetime
 import fnmatch
 import os
 import os.path
@@ -56,7 +55,7 @@ def load_commits(db, repo_name):
         commit_table = db["commits"]
 
         log = get_cmd_output(GITLOG)
-        for i, commit in enumerate(log.split(SEP + "\n")):
+        for commit in log.split(SEP + "\n"):
             if re.match(r"fatal: your current branch '\w+' does not have any commits yet", commit):
                 # Project-only or uninitialized repos are like this.
                 continue

edx_repo_tools/conventional_commits/extra.txt

@@ -4,25 +4,23 @@
 #
 #    make upgrade
 #
-alembic==1.10.4
-    # via
-    #   -c edx_repo_tools/conventional_commits/../../requirements/constraints.txt
-    #   dataset
+alembic==1.11.1
+    # via dataset
 banal==1.0.6
     # via dataset
 contourpy==1.1.0
     # via matplotlib
 cycler==0.11.0
     # via matplotlib
-dataset==1.6.0
+dataset==1.6.2
     # via -r edx_repo_tools/conventional_commits/extra.in
-fonttools==4.40.0
+fonttools==4.41.1
     # via matplotlib
 greenlet==2.0.2
     # via sqlalchemy
-importlib-metadata==6.7.0
+importlib-metadata==6.8.0
     # via alembic
-importlib-resources==5.12.0
+importlib-resources==6.0.0
     # via
     #   alembic
     #   matplotlib
@@ -32,7 +30,7 @@ mako==1.2.4
     # via alembic
 markupsafe==2.1.3
     # via mako
-matplotlib==3.7.1
+matplotlib==3.7.2
     # via -r edx_repo_tools/conventional_commits/extra.in
 numpy==1.24.4
     # via
@@ -43,9 +41,9 @@ packaging==23.1
     # via matplotlib
 pandas==2.0.3
     # via -r edx_repo_tools/conventional_commits/extra.in
-pillow==9.5.0
+pillow==10.0.0
     # via matplotlib
-pyparsing==3.1.0
+pyparsing==3.0.9
     # via matplotlib
 python-dateutil==2.8.2
     # via
@@ -55,15 +53,15 @@ pytz==2023.3
     # via pandas
 six==1.16.0
     # via python-dateutil
-sqlalchemy==1.4.48
+sqlalchemy==1.4.49
     # via
     #   alembic
     #   dataset
-typing-extensions==4.7.0
+typing-extensions==4.7.1
     # via alembic
 tzdata==2023.3
     # via pandas
-zipp==3.15.0
+zipp==3.16.2
     # via
     #   importlib-metadata
     #   importlib-resources

edx_repo_tools/find_dependencies/README.rst

@@ -24,7 +24,7 @@ Run it with a list of local repo directories.  It will traverse into each direct
 
 - second_party_urls.txt is the subset of repo_urls.txt that come from organizations close enough to the Open edX project, that the repos might need to be moved into the openedx organization.
 
-I run it in a tree of all repos, with these commands to example all the repos branched for Olive::
+I run it in a tree of all repos, with these commands to examine all the repos branched for Olive::
 
     $ export OLIVE_DIRS=$(gittreeif origin/open-release/olive.master -q pwd)
     $ find_dependencies $OLIVE_DIRS

edx_repo_tools/find_dependencies/extra.txt

@@ -4,9 +4,9 @@
 #
 #    make upgrade
 #
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 idna==3.4
     # via requests
@@ -20,7 +20,7 @@ requests==2.31.0
     # via -r edx_repo_tools/find_dependencies/extra.in
 rich==13.4.2
     # via -r edx_repo_tools/find_dependencies/extra.in
-typing-extensions==4.7.0
+typing-extensions==4.7.1
     # via rich
-urllib3==2.0.3
+urllib3==2.0.4
     # via requests

edx_repo_tools/find_dependencies/find_dependencies.py

@@ -278,7 +278,7 @@ def find_py_reqs():
         if possible_req.exists():
             return possible_req
     if any(Path(ind).exists() for ind in PY_INDICATORS):
-        print(f"WARNING: {repo_name} is likely a Python package, but we can't find its dependencies.")
+        print(f"WARNING: {os.getcwd()} is likely a Python package, but we can't find its dependencies.")
     return None
 
 

edx_repo_tools/repo_access_scraper/extra.txt

@@ -6,9 +6,11 @@
 #
 greenlet==2.0.2
     # via playwright
-playwright==1.35.0
+playwright==1.36.0
     # via -r edx_repo_tools/repo_access_scraper/extra.in
 pyee==9.0.4
     # via playwright
-typing-extensions==4.7.0
-    # via playwright
+typing-extensions==4.7.1
+    # via
+    #   playwright
+    #   pyee

edx_repo_tools/repo_access_scraper/repo_access_scraper.py

@@ -55,10 +55,10 @@ def screenshot_pages(page, url, image_prefix):
         page.screenshot(path=f"{IMAGES_DIR}/{image_prefix}-{imgnum}.png", full_page=True)
 
         # If there's a next page, visit it.
-        next = page.locator("a.next_page")
-        if next.count():
+        next_page = page.locator("a.next_page")
+        if next_page.count():
             with page.expect_navigation():
-                next.click()
+                next_page.click()
         else:
             # No next page, we're done here.
             break
@@ -72,7 +72,7 @@ def request_list(url):
     """
     data = []
     while url:
-        resp = requests.get(url, headers=HEADERS)
+        resp = requests.get(url, headers=HEADERS, timeout=60)
         data.extend(resp.json())
         url = None
         if "link" in resp.headers:
@@ -84,7 +84,7 @@ def request_list(url):
 
 def request_dict(url):
     """Get dict data from a GitHub URL."""
-    return requests.get(url, headers=HEADERS).json()
+    return requests.get(url, headers=HEADERS, timeout=60).json()
 
 def counted(things: list, thing_name: str) -> str:
     """

edx_repo_tools/repo_checks/extra.txt

@@ -6,11 +6,11 @@
 #
 cache-to-disk==2.0.0
     # via -r edx_repo_tools/repo_checks/extra.in
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.6
     # via -r edx_repo_tools/repo_checks/extra.in
 fastcore==1.5.29
     # via ghapi
@@ -19,12 +19,14 @@ ghapi==1.0.4
 idna==3.4
     # via requests
 packaging==23.1
-    # via ghapi
-pyyaml==6.0
+    # via
+    #   fastcore
+    #   ghapi
+pyyaml==6.0.1
     # via -r edx_repo_tools/repo_checks/extra.in
 requests==2.31.0
     # via -r edx_repo_tools/repo_checks/extra.in
-urllib3==2.0.3
+urllib3==2.0.4
     # via requests
 
 # The following packages are considered to be unsafe in a requirements file:

edx_repo_tools/repo_checks/repo_checks.py

@@ -817,9 +817,9 @@ def _update_branch_protection(self, params):
             "https://api.github.com"
             + self.api.repos.update_branch_protection.path.format(**params)
         )
-        resp = requests.put(
+        resp = requests.put(  # pylint: disable=missing-timeout
             url, headers=headers, json=params
-        )  # pylint: disable=missing-timeout
+        )
 
         resp.raise_for_status()