Feature: Exporter: Datasheets (#112)

Exporter that downloads all PDF datasheets from every (sub)module with the has_datasheet trait.

---------

Co-authored-by: iopapamanoglou <[email protected]>

examples/minimal_led_orderable.py

@@ -12,6 +12,7 @@
 
 import faebryk.library._F as F
 from faebryk.core.module import Module
+from faebryk.exporters.documentation.datasheets import export_datasheets
 from faebryk.exporters.pcb.kicad.artifacts import export_svg
 from faebryk.exporters.pcb.kicad.transformer import PCB_Transformer
 from faebryk.exporters.pcb.layout.absolute import LayoutAbsolute
@@ -149,6 +150,7 @@ def main():
     apply_design_to_pcb(app, transform_pcb)
     export_pcba_artifacts(ARTIFACTS, PCB_FILE, app)
     export_svg(PCB_FILE, ARTIFACTS / Path("pcba.svg"))
+    export_datasheets(app, BUILD_DIR / "documentation" / "datasheets")
 
 
 if __name__ == "__main__":

src/faebryk/exporters/documentation/datasheets.py

@@ -0,0 +1,88 @@
+# This file is part of the faebryk project
+# SPDX-License-Identifier: MIT
+
+import logging
+from pathlib import Path
+
+import requests
+
+import faebryk.library._F as F
+from faebryk.core.module import Module
+
+logger = logging.getLogger(__name__)
+
+
+def export_datasheets(
+    app: Module,
+    path: Path = Path("build/documentation/datasheets"),
+    overwrite: bool = False,
+):
+    """
+    Export all datasheets of all modules (that have a datasheet defined)
+    of the given application.
+    """
+    # Create directories if they don't exist
+    path.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"Exporting datasheets to: {path}")
+    for m in app.get_children_modules(types=Module):
+        if not m.has_trait(F.has_datasheet):
+            continue
+        url = m.get_trait(F.has_datasheet).get_datasheet()
+        if not url:
+            logger.warning(f"Missing datasheet URL for {m}")
+            continue
+        filename = type(m).__name__ + ".pdf"
+        file_path = path / filename
+        if file_path.exists() and not overwrite:
+            logger.debug(
+                f"Datasheet for {m} already exists, skipping download"  # noqa: E501
+            )
+            continue
+        try:
+            _download_datasheet(url, file_path)
+        except DatasheetDownloadException as e:
+            logger.error(f"Failed to download datasheet for {m}: {e}")
+
+        logger.debug(f"Downloaded datasheet for {m}")
+
+
+class DatasheetDownloadException(Exception):
+    pass
+
+
+def _download_datasheet(url: str, path: Path):
+    """
+    Download the datasheet of the given module and save it to the given path.
+    """
+    if not url.endswith(".pdf"):
+        raise DatasheetDownloadException(f"Datasheet URL {url} is probably not a PDF")
+    if not url.startswith(("http://", "https://")):
+        raise DatasheetDownloadException(
+            f"Datasheet URL {url} is probably not a valid URL"
+        )
+
+    try:
+        # TODO probably need something fancier
+        user_agent_headers = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"  # noqa: E501
+        }
+        response = requests.get(url, headers=user_agent_headers)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        raise DatasheetDownloadException(
+            f"Failed to download datasheet from {url}: {e}"
+        ) from e
+
+    # check if content is pdf
+    if not response.content.startswith(b"%PDF"):
+        raise DatasheetDownloadException(
+            f"Downloaded content is not a PDF: {response.content[:100]}"
+        )
+
+    try:
+        path.write_bytes(response.content)
+    except Exception as e:
+        raise DatasheetDownloadException(
+            f"Failed to save datasheet to {path}: {e}"
+        ) from e

src/faebryk/library/_F.py

@@ -33,11 +33,11 @@
 from faebryk.library.has_linked_pad import has_linked_pad
 from faebryk.library.has_reference import has_reference
 from faebryk.library.can_bridge import can_bridge
+from faebryk.library.has_datasheet import has_datasheet
 from faebryk.library.has_designator import has_designator
 from faebryk.library.has_descriptive_properties import has_descriptive_properties
 from faebryk.library.has_simple_value_representation import has_simple_value_representation
 from faebryk.library.has_capacitance import has_capacitance
-from faebryk.library.has_datasheet import has_datasheet
 from faebryk.library.has_footprint_requirement import has_footprint_requirement
 from faebryk.library.has_kicad_ref import has_kicad_ref
 from faebryk.library.has_picker import has_picker
@@ -62,11 +62,11 @@
 from faebryk.library.has_overriden_name_defined import has_overriden_name_defined
 from faebryk.library.has_linked_pad_defined import has_linked_pad_defined
 from faebryk.library.can_bridge_defined import can_bridge_defined
+from faebryk.library.has_datasheet_defined import has_datasheet_defined
 from faebryk.library.has_designator_defined import has_designator_defined
 from faebryk.library.has_descriptive_properties_defined import has_descriptive_properties_defined
 from faebryk.library.has_simple_value_representation_based_on_params import has_simple_value_representation_based_on_params
 from faebryk.library.has_simple_value_representation_defined import has_simple_value_representation_defined
-from faebryk.library.has_datasheet_defined import has_datasheet_defined
 from faebryk.library.has_footprint_requirement_defined import has_footprint_requirement_defined
 from faebryk.library.has_multi_picker import has_multi_picker
 from faebryk.library.has_pcb_layout_defined import has_pcb_layout_defined

src/faebryk/libs/picker/lcsc.py

@@ -11,6 +11,7 @@
     EasyedaFootprintImporter,
     EasyedaSymbolImporter,
 )
+from easyeda2kicad.easyeda.parameters_easyeda import EeSymbol
 from easyeda2kicad.kicad.export_kicad_3d_model import Exporter3dModelKicad
 from easyeda2kicad.kicad.export_kicad_footprint import ExporterFootprintKicad
 from easyeda2kicad.kicad.export_kicad_symbol import ExporterSymbolKicad, KicadVersion
@@ -22,9 +23,14 @@
     PickerOption,
     Supplier,
 )
+from faebryk.libs.util import ConfigFlag
 
 logger = logging.getLogger(__name__)
 
+CRAWL_DATASHEET = ConfigFlag(
+    "LCSC_DATASHEET", default=False, descr="Crawl for datasheet on LCSC"
+)
+
 # TODO dont hardcode relative paths
 BUILD_FOLDER = Path("./build")
 LIB_FOLDER = Path("./src/kicad/libs")
@@ -174,6 +180,33 @@ def download_easyeda_info(partno: str, get_model: bool = True):
     return ki_footprint, ki_model, easyeda_footprint, easyeda_model, easyeda_symbol
 
 
+def get_datasheet_url(part: EeSymbol):
+    # TODO use easyeda2kicad api as soon as works again
+    # return part.info.datasheet
+
+    if not CRAWL_DATASHEET:
+        return None
+
+    import re
+
+    import requests
+
+    url = part.info.datasheet
+    if not url:
+        return None
+    # make requests act like curl
+    lcsc_site = requests.get(url, headers={"User-Agent": "curl/7.81.0"})
+    partno = part.info.lcsc_id
+    # find _{partno}.pdf in html
+    match = re.search(f'href="(https://[^"]+_{partno}.pdf)"', lcsc_site.text)
+    if match:
+        pdfurl = match.group(1)
+        logger.debug(f"Found datasheet for {partno} at {pdfurl}")
+        return pdfurl
+    else:
+        return None
+
+
 def attach(component: Module, partno: str, get_model: bool = True):
     ki_footprint, ki_model, easyeda_footprint, easyeda_model, easyeda_symbol = (
         download_easyeda_info(partno, get_model=get_model)
@@ -215,6 +248,10 @@ def attach(component: Module, partno: str, get_model: bool = True):
 
     component.add(F.has_descriptive_properties_defined({"LCSC": partno}))
 
+    datasheet = get_datasheet_url(easyeda_symbol)
+    if datasheet:
+        component.add(F.has_datasheet_defined(datasheet))
+
     # model done by kicad (in fp)