Tsp report

.github/workflows/cornflow-client-publish-to-pypi.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Set up Python 3.8
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Install wheel
@@ -27,16 +27,16 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python setup.py sdist bdist_wheel
     - name: Publish distribution 📦 to Test PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.test_pypi_password }}
-        repository_url: https://test.pypi.org/legacy/
-        packages_dir: libs/client/dist/
+        repository-url: https://test.pypi.org/legacy/
+        packages-dir: libs/client/dist/
     - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.pypi_password }}
-        packages_dir: libs/client/dist/
+        packages-dir: libs/client/dist/
     - name: Get version number
       uses: winterjung/split@v2
       id: split

.github/workflows/cornflow-publish-to-pypi.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@master
     - name: Set up Python 3.8
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Install wheel
@@ -27,16 +27,16 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python setup.py sdist bdist_wheel
     - name: Publish distribution 📦 to Test PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.CORNFLOW_TEST_PYPI_TOKEN }}
-        repository_url: https://test.pypi.org/legacy/
-        packages_dir: cornflow-server/dist/
+        repository-url: https://test.pypi.org/legacy/
+        packages-dir: cornflow-server/dist/
     - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.CORNFLOW_PYPI_TOKEN }}
-        packages_dir: cornflow-server/dist/
+        packages-dir: cornflow-server/dist/
     - name: Get version number
       uses: winterjung/split@v2
       id: split

.github/workflows/test_cornflow_client.yml

@@ -92,8 +92,9 @@ jobs:
           CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
           python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
           python -m pip install -U orloge pulp
-          python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
           python -m pip install -U -r airflow_config/requirements.txt
+          python -m pip uninstall cornflow-client -y
+          python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
           airflow db init
           airflow users create -u admin -f admin -l admin -r Admin -p admin -e [email protected]
           airflow webserver -p 8080 &

.github/workflows/test_cornflow_dags.yml

@@ -36,6 +36,13 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Quarto
+      uses: quarto-dev/quarto-actions/setup@v2
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        # To install LaTeX to build PDF book
+        tinytex: true
     - name: Install cbc
       run: |
         sudo apt-get install -y coinor-cbc

.github/workflows/test_cornflow_server.yml

@@ -50,9 +50,16 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Quarto
+      uses: quarto-dev/quarto-actions/setup@v2
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        # To install LaTeX to build PDF book
+        tinytex: true
     - name: Copy DAG files
       run: |
         cd ..
@@ -80,8 +87,9 @@ jobs:
         python -m pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
         python -m pip install -U orloge pulp
         python -m pip uninstall cornflow-client -y
-        python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
         python -m pip install -U -r airflow_config/requirements.txt
+        python -m pip uninstall cornflow-client -y
+        python -m pip install -U "git+https://github.com/baobabsoluciones/cornflow@${CLIENT_BRANCH}#subdirectory=libs/client"
         airflow db init
         airflow users create \
               --username admin \
@@ -101,9 +109,9 @@ jobs:
         AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth
         AIRFLOW__WEBSERVER__SECRET_KEY: e9adafa751fd35adfc1fdd3285019be15eea0758f76e38e1e37a1154fb36
         AIRFLOW__CORE__LOAD_EXAMPLES: 0
-        AIRFLOW_CONN_CF_URI: http://airflow:Airflow_test_password1@localhost:5050
+        AIRFLOW_CONN_CF_URI: http://service_user:Airflow_test_password1@localhost:5050
     - name: Run unit tests
-      run: | 
+      run: |
         coverage run --source=./cornflow/ --rcfile=./.coveragerc -m unittest discover -s cornflow/tests/unit
         coverage report -m
       env:
@@ -115,7 +123,7 @@ jobs:
         AIRFLOW_PWD: notadmin
         CF_ALARMS_ENDPOINT: 1
     - name: Run ldap unit tests
-      run: | 
+      run: |
         coverage run -a --source=./cornflow/ --rcfile=./.coveragerc -m unittest discover -s cornflow/tests/ldap
         coverage report -m
       env:
@@ -143,7 +151,7 @@ jobs:
         LOG_LEVEL: 30
         CORNFLOW_SERVICE_USER: cornflow
     - name: Run postgres unit tests
-      run: | 
+      run: |
         coverage run -a --source=./cornflow/ --rcfile=./.coveragerc -m unittest cornflow/tests/unit/test_commands.py
         coverage report -m
       env:

cornflow-dags/DAG/activate_dags.py

@@ -1,4 +1,5 @@
 import cornflow_client.airflow.dag_utilities as utils
+
 from airflow import DAG
 from airflow.operators.python import PythonOperator
 from airflow.secrets.environment_variables import EnvironmentVariablesBackend
@@ -10,6 +11,9 @@ def create_dag(app):
     def solve(**kwargs):
         return utils.cf_solve_app(app, EnvironmentVariablesBackend(), **kwargs)
 
+    def run(**kwargs):
+      return utils.cf_report(app, EnvironmentVariablesBackend(), **kwargs)
+
     if app.default_args is not None:
         default_args = app.default_args
     else:
@@ -27,18 +31,24 @@ def solve(**kwargs):
         tags=["model"],
         **kwargs
     )
-    with dag:
-        notify = getattr(app, "notify", True)
-        if not notify:
-            t1 = PythonOperator(task_id=app.name, python_callable=solve)
-        else:
-            t1 = PythonOperator(
-                task_id=app.name,
-                python_callable=solve,
-                on_failure_callback=utils.callback_email,
-            )
-
-    return dag
+
+    notify = getattr(app, "notify", True)
+    if not notify:
+        t1 = PythonOperator(task_id=app.name, python_callable=solve, dag=dag)
+    else:
+        t1 = PythonOperator(
+            task_id=app.name,
+            python_callable=solve,
+            on_failure_callback=utils.callback_email,
+            dag=dag
+        )
+
+    t2 = PythonOperator(task_id=f"{app.name}_report", python_callable=run, dag=dag)
+
+
+    t1 >> t2
+
+    return dag       
 
 
 for app in get_new_apps():

cornflow-dags/DAG/dag_timer.py

@@ -1,9 +1,13 @@
+import os.path
+
 import time
 from cornflow_client import get_empty_schema
 from cornflow_client import ApplicationCore, InstanceCore, SolutionCore, ExperimentCore
 from cornflow_client.constants import SOLUTION_STATUS_FEASIBLE, STATUS_OPTIMAL
 import logging
 
+from xml.etree import ElementTree as ET
+
 
 class Instance(InstanceCore):
     schema = get_empty_schema()
@@ -33,6 +37,19 @@ def get_objective(self) -> float:
     def check_solution(self, *args, **kwargs):
         return dict()
 
+    def generate_report(self, report_name="report") -> str:
+        report_path = os.path.abspath("./report.html")
+        html = ET.Element("html")
+        body = ET.Element("body")
+        html.append(body)
+        div = ET.Element("div", attrib={"class": "foo"})
+        body.append(div)
+        span = ET.Element("span", attrib={"class": "bar"})
+        div.append(span)
+        with open(report_path, "w") as f:
+            ET.ElementTree(html).write(f, encoding="unicode", method="html")
+        return report_path
+
 
 class Timer(ApplicationCore):
     name = "timer"

cornflow-dags/DAG/graph_coloring/__init__.py

@@ -1,9 +1,5 @@
-from cornflow_client import (
-    get_empty_schema,
-    ApplicationCore,
-)
+from cornflow_client import get_empty_schema, ApplicationCore, add_reports_to_schema
 from typing import List, Dict
-import pytups as pt
 import os
 
 from .solvers import OrToolsCP
@@ -18,31 +14,22 @@ class GraphColoring(ApplicationCore):
     schema = get_empty_schema(
         properties=dict(timeLimit=dict(type="number")), solvers=list(solvers.keys())
     )
+    schema = add_reports_to_schema(schema, ["report"])
 
     @property
     def test_cases(self) -> List[Dict]:
-        def read_file(filePath):
-            with open(filePath, "r") as f:
-                contents = f.read().splitlines()
-
-            pairs = (
-                pt.TupList(contents[1:])
-                .vapply(lambda v: v.split(" "))
-                .vapply(lambda v: dict(n1=int(v[0]), n2=int(v[1])))
-            )
-            return dict(pairs=pairs)
 
         file_dir = os.path.join(os.path.dirname(__file__), "data")
-
+        get_file = lambda name: os.path.join(file_dir, name)
         return [
             {
                 "name": "gc_4_1",
-                "instance": read_file(os.path.join(file_dir, "gc_4_1")),
+                "instance": Instance.from_txt_file(get_file("gc_4_1")).to_dict(),
                 "description": "Example data with 4 pairs",
             },
             {
                 "name": "gc_50_1",
-                "instance": read_file(os.path.join(file_dir, "gc_50_1")),
+                "instance": Instance.from_txt_file(get_file("gc_50_1")).to_dict(),
                 "description": "Example data with 50 pairs",
             },
         ]

cornflow-dags/DAG/graph_coloring/core/experiment.py

@@ -1,8 +1,10 @@
 from cornflow_client import ExperimentCore
 from cornflow_client.core.tools import load_json
+from pytups import TupList
 from .instance import Instance
 from .solution import Solution
 import os
+import quarto
 
 
 class Experiment(ExperimentCore):
@@ -14,6 +16,12 @@ class Experiment(ExperimentCore):
     def instance(self) -> Instance:
         return super().instance
 
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            Instance.from_dict(data["instance"]), Solution.from_dict(data["solution"])
+        )
+
     @property
     def solution(self) -> Solution:
         return super().solution
@@ -29,7 +37,19 @@ def check_solution(self, *args, **kwargs) -> dict:
         # if a pair of nodes have the same colors: that's a problem
         colors = self.solution.get_assignments()
         pairs = self.instance.get_pairs()
+        nodes = self.instance.get_nodes()
+        missing_colors = TupList(set(nodes) - colors.keys())
         errors = [
-            {"n1": n1, "n2": n2} for (n1, n2) in pairs if colors[n1] == colors[n2]
+            {"n1": n1, "n2": n2}
+            for (n1, n2) in pairs
+            if n1 in colors and n2 in colors and colors[n1] == colors[n2]
         ]
-        return dict(pairs=errors)
+        return dict(pairs=errors, missing=missing_colors)
+
+    def generate_report(self, report_name="report") -> str:
+        if not os.path.isabs(report_name):
+            report_name = os.path.join(
+                os.path.dirname(__file__), "../report/", report_name
+            )
+
+        return self.generate_report_quarto(quarto, report_name=report_name)

cornflow-dags/DAG/graph_coloring/core/instance.py

@@ -2,11 +2,40 @@
 from cornflow_client import InstanceCore, get_empty_schema
 from cornflow_client.core.tools import load_json
 import pytups as pt
+import networkx as nx
 
 
 class Instance(InstanceCore):
     schema = load_json(os.path.join(os.path.dirname(__file__), "../schemas/input.json"))
     schema_checks = get_empty_schema()
 
     def get_pairs(self):
-        return pt.TupList((el["n1"], el["n2"]) for el in self.data["pairs"])
+        return pt.TupList(self.data["pairs"]).take(["n1", "n2"])
+
+    def get_nodes(self):
+        pairs = self.data["pairs"]
+        n1s = pt.TupList(pairs).vapply(lambda v: v["n1"])
+        n2s = pt.TupList(pairs).vapply(lambda v: v["n2"])
+        return (n1s + n2s).unique2()
+
+    @classmethod
+    def from_txt_file(cls, filePath):
+        with open(filePath, "r") as f:
+            contents = f.read().splitlines()
+
+        pairs = (
+            pt.TupList(contents[1:])
+            .vapply(lambda v: v.split(" "))
+            .vapply(lambda v: dict(n1=int(v[0]), n2=int(v[1])))
+        )
+        return Instance.from_dict(dict(pairs=pairs))
+
+    def get_graph(self):
+        nodes = self.get_nodes()
+        arcs = self.get_pairs()
+        G = nx.Graph()
+        for node in nodes:
+            G.add_node(node)
+        for n1, n2 in arcs:
+            G.add_edge(n1, n2)
+        return G

cornflow-dags/DAG/graph_coloring/report/.gitignore

@@ -0,0 +1 @@
+/.quarto/