From fa4f01788fba01019a317cad67483ccef357717e Mon Sep 17 00:00:00 2001 From: ruben-iteng <94007802+ruben-iteng@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:27:45 +0200 Subject: [PATCH 1/5] Add datasheet exporter --- .../exporters/documentation/datasheets.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/faebryk/exporters/documentation/datasheets.py diff --git a/src/faebryk/exporters/documentation/datasheets.py b/src/faebryk/exporters/documentation/datasheets.py new file mode 100644 index 00000000..72897137 --- /dev/null +++ b/src/faebryk/exporters/documentation/datasheets.py @@ -0,0 +1,70 @@ +# This file is part of the faebryk project +# SPDX-License-Identifier: MIT + +import logging +from pathlib import Path + +import requests + +import faebryk.library._F as F +from faebryk.core.module import Module + +logger = logging.getLogger(__name__) + + +def export_datasheets( + app: Module, + path: Path = Path("build/documentation/datasheets"), + overwrite: bool = False, +): + """ + Export all datasheets of all modules (that have a datasheet defined) + of the given application. + """ + # Create directories if they don't exist + path.mkdir(parents=True, exist_ok=True) + + logger.info(f"Exporting datasheets to: {path}") + for m in app.get_children_modules(types=Module): + url = m.get_trait(F.has_datasheet).get_datasheet() + if url: + filename = f"{m.get_full_name(types=False)}.pdf".split(".")[-1].replace( + " ", "_" + ) + file_path = path / filename + if file_path.exists() and not overwrite: + logger.info( + f"Datasheet for {m.get_full_name(types=False)} already exists, skipping download" # noqa: E501 + ) + continue + success = _download_datasheet(url, file_path) + logger.info( + f"Downloaded datasheet for {m.get_full_name(types=False)}: {'\033[92mOK\033[0m' if success else '\033[91mFAILED\033[0m'}" # noqa: E501 + ) + + +def _download_datasheet(url: str, path: Path) -> bool: + """ + Download the datasheet of the given module and save it to the given path. + """ + if not url.endswith(".pdf"): + logger.warning(f"Datasheet URL {url} is probably not a PDF") + return False + if not url.startswith(("http://", "https://")): + logger.warning(f"Datasheet URL {url} is probably not a valid URL") + return False + + try: + response = requests.get(url) + response.raise_for_status() + except requests.RequestException as e: + logger.error(f"Failed to download datasheet from {url}: {e}") + return False + + try: + path.write_bytes(response.content) + except Exception as e: + logger.error(f"Failed to save datasheet to {path}: {e}") + return False + + return True From a736c030e5ddbb76a0772345e4f419e081465119 Mon Sep 17 00:00:00 2001 From: ruben-iteng <94007802+ruben-iteng@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:17:22 +0100 Subject: [PATCH 2/5] Fix missing user-agent info Some downloads fail without it --- src/faebryk/exporters/documentation/datasheets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/faebryk/exporters/documentation/datasheets.py b/src/faebryk/exporters/documentation/datasheets.py index 72897137..e82b355f 100644 --- a/src/faebryk/exporters/documentation/datasheets.py +++ b/src/faebryk/exporters/documentation/datasheets.py @@ -55,7 +55,10 @@ def _download_datasheet(url: str, path: Path) -> bool: return False try: - response = requests.get(url) + user_agent_headers = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" # noqa: E501 + } + response = requests.get(url, headers=user_agent_headers) response.raise_for_status() except requests.RequestException as e: logger.error(f"Failed to download datasheet from {url}: {e}") From 19a018557612b1df96a2de61cc049164ed23639f Mon Sep 17 00:00:00 2001 From: ruben-iteng <94007802+ruben-iteng@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:24:56 +0100 Subject: [PATCH 3/5] Use module type name instead of full name --- src/faebryk/exporters/documentation/datasheets.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/faebryk/exporters/documentation/datasheets.py b/src/faebryk/exporters/documentation/datasheets.py index e82b355f..8891b597 100644 --- a/src/faebryk/exporters/documentation/datasheets.py +++ b/src/faebryk/exporters/documentation/datasheets.py @@ -26,11 +26,11 @@ def export_datasheets( logger.info(f"Exporting datasheets to: {path}") for m in app.get_children_modules(types=Module): + if not m.has_trait(F.has_datasheet): + continue url = m.get_trait(F.has_datasheet).get_datasheet() if url: - filename = f"{m.get_full_name(types=False)}.pdf".split(".")[-1].replace( - " ", "_" - ) + filename = type(m).__name__ + ".pdf" file_path = path / filename if file_path.exists() and not overwrite: logger.info( From 6e9f21a0a55c135d1fbbb0beb0c018558b645018 Mon Sep 17 00:00:00 2001 From: iopapamanoglou Date: Sun, 3 Nov 2024 17:26:02 +0100 Subject: [PATCH 4/5] genF --- src/faebryk/library/_F.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/faebryk/library/_F.py b/src/faebryk/library/_F.py index 12340c67..0ddd5b55 100644 --- a/src/faebryk/library/_F.py +++ b/src/faebryk/library/_F.py @@ -33,11 +33,11 @@ from faebryk.library.has_linked_pad import has_linked_pad from faebryk.library.has_reference import has_reference from faebryk.library.can_bridge import can_bridge +from faebryk.library.has_datasheet import has_datasheet from faebryk.library.has_designator import has_designator from faebryk.library.has_descriptive_properties import has_descriptive_properties from faebryk.library.has_simple_value_representation import has_simple_value_representation from faebryk.library.has_capacitance import has_capacitance -from faebryk.library.has_datasheet import has_datasheet from faebryk.library.has_footprint_requirement import has_footprint_requirement from faebryk.library.has_kicad_ref import has_kicad_ref from faebryk.library.has_picker import has_picker @@ -62,11 +62,11 @@ from faebryk.library.has_overriden_name_defined import has_overriden_name_defined from faebryk.library.has_linked_pad_defined import has_linked_pad_defined from faebryk.library.can_bridge_defined import can_bridge_defined +from faebryk.library.has_datasheet_defined import has_datasheet_defined from faebryk.library.has_designator_defined import has_designator_defined from faebryk.library.has_descriptive_properties_defined import has_descriptive_properties_defined from faebryk.library.has_simple_value_representation_based_on_params import has_simple_value_representation_based_on_params from faebryk.library.has_simple_value_representation_defined import has_simple_value_representation_defined -from faebryk.library.has_datasheet_defined import has_datasheet_defined from faebryk.library.has_footprint_requirement_defined import has_footprint_requirement_defined from faebryk.library.has_multi_picker import has_multi_picker from faebryk.library.has_pcb_layout_defined import has_pcb_layout_defined From 2cb5f44910f85804e9a4661a17a727ada9b877c6 Mon Sep 17 00:00:00 2001 From: iopapamanoglou Date: Sun, 3 Nov 2024 18:13:36 +0100 Subject: [PATCH 5/5] Make more robust; Add lcsc crawl --- examples/minimal_led_orderable.py | 2 + .../exporters/documentation/datasheets.py | 59 ++++++++++++------- src/faebryk/libs/picker/lcsc.py | 37 ++++++++++++ 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/examples/minimal_led_orderable.py b/examples/minimal_led_orderable.py index 5764ff97..bd55ccad 100644 --- a/examples/minimal_led_orderable.py +++ b/examples/minimal_led_orderable.py @@ -12,6 +12,7 @@ import faebryk.library._F as F from faebryk.core.module import Module +from faebryk.exporters.documentation.datasheets import export_datasheets from faebryk.exporters.pcb.kicad.artifacts import export_svg from faebryk.exporters.pcb.kicad.transformer import PCB_Transformer from faebryk.exporters.pcb.layout.absolute import LayoutAbsolute @@ -149,6 +150,7 @@ def main(): apply_design_to_pcb(app, transform_pcb) export_pcba_artifacts(ARTIFACTS, PCB_FILE, app) export_svg(PCB_FILE, ARTIFACTS / Path("pcba.svg")) + export_datasheets(app, BUILD_DIR / "documentation" / "datasheets") if __name__ == "__main__": diff --git a/src/faebryk/exporters/documentation/datasheets.py b/src/faebryk/exporters/documentation/datasheets.py index 8891b597..1b4bc56d 100644 --- a/src/faebryk/exporters/documentation/datasheets.py +++ b/src/faebryk/exporters/documentation/datasheets.py @@ -29,45 +29,60 @@ def export_datasheets( if not m.has_trait(F.has_datasheet): continue url = m.get_trait(F.has_datasheet).get_datasheet() - if url: - filename = type(m).__name__ + ".pdf" - file_path = path / filename - if file_path.exists() and not overwrite: - logger.info( - f"Datasheet for {m.get_full_name(types=False)} already exists, skipping download" # noqa: E501 - ) - continue - success = _download_datasheet(url, file_path) - logger.info( - f"Downloaded datasheet for {m.get_full_name(types=False)}: {'\033[92mOK\033[0m' if success else '\033[91mFAILED\033[0m'}" # noqa: E501 + if not url: + logger.warning(f"Missing datasheet URL for {m}") + continue + filename = type(m).__name__ + ".pdf" + file_path = path / filename + if file_path.exists() and not overwrite: + logger.debug( + f"Datasheet for {m} already exists, skipping download" # noqa: E501 ) + continue + try: + _download_datasheet(url, file_path) + except DatasheetDownloadException as e: + logger.error(f"Failed to download datasheet for {m}: {e}") + + logger.debug(f"Downloaded datasheet for {m}") + +class DatasheetDownloadException(Exception): + pass -def _download_datasheet(url: str, path: Path) -> bool: + +def _download_datasheet(url: str, path: Path): """ Download the datasheet of the given module and save it to the given path. """ if not url.endswith(".pdf"): - logger.warning(f"Datasheet URL {url} is probably not a PDF") - return False + raise DatasheetDownloadException(f"Datasheet URL {url} is probably not a PDF") if not url.startswith(("http://", "https://")): - logger.warning(f"Datasheet URL {url} is probably not a valid URL") - return False + raise DatasheetDownloadException( + f"Datasheet URL {url} is probably not a valid URL" + ) try: + # TODO probably need something fancier user_agent_headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" # noqa: E501 } response = requests.get(url, headers=user_agent_headers) response.raise_for_status() except requests.RequestException as e: - logger.error(f"Failed to download datasheet from {url}: {e}") - return False + raise DatasheetDownloadException( + f"Failed to download datasheet from {url}: {e}" + ) from e + + # check if content is pdf + if not response.content.startswith(b"%PDF"): + raise DatasheetDownloadException( + f"Downloaded content is not a PDF: {response.content[:100]}" + ) try: path.write_bytes(response.content) except Exception as e: - logger.error(f"Failed to save datasheet to {path}: {e}") - return False - - return True + raise DatasheetDownloadException( + f"Failed to save datasheet to {path}: {e}" + ) from e diff --git a/src/faebryk/libs/picker/lcsc.py b/src/faebryk/libs/picker/lcsc.py index 61e28a4e..9cf5838e 100644 --- a/src/faebryk/libs/picker/lcsc.py +++ b/src/faebryk/libs/picker/lcsc.py @@ -11,6 +11,7 @@ EasyedaFootprintImporter, EasyedaSymbolImporter, ) +from easyeda2kicad.easyeda.parameters_easyeda import EeSymbol from easyeda2kicad.kicad.export_kicad_3d_model import Exporter3dModelKicad from easyeda2kicad.kicad.export_kicad_footprint import ExporterFootprintKicad from easyeda2kicad.kicad.export_kicad_symbol import ExporterSymbolKicad, KicadVersion @@ -22,9 +23,14 @@ PickerOption, Supplier, ) +from faebryk.libs.util import ConfigFlag logger = logging.getLogger(__name__) +CRAWL_DATASHEET = ConfigFlag( + "LCSC_DATASHEET", default=False, descr="Crawl for datasheet on LCSC" +) + # TODO dont hardcode relative paths BUILD_FOLDER = Path("./build") LIB_FOLDER = Path("./src/kicad/libs") @@ -174,6 +180,33 @@ def download_easyeda_info(partno: str, get_model: bool = True): return ki_footprint, ki_model, easyeda_footprint, easyeda_model, easyeda_symbol +def get_datasheet_url(part: EeSymbol): + # TODO use easyeda2kicad api as soon as works again + # return part.info.datasheet + + if not CRAWL_DATASHEET: + return None + + import re + + import requests + + url = part.info.datasheet + if not url: + return None + # make requests act like curl + lcsc_site = requests.get(url, headers={"User-Agent": "curl/7.81.0"}) + partno = part.info.lcsc_id + # find _{partno}.pdf in html + match = re.search(f'href="(https://[^"]+_{partno}.pdf)"', lcsc_site.text) + if match: + pdfurl = match.group(1) + logger.debug(f"Found datasheet for {partno} at {pdfurl}") + return pdfurl + else: + return None + + def attach(component: Module, partno: str, get_model: bool = True): ki_footprint, ki_model, easyeda_footprint, easyeda_model, easyeda_symbol = ( download_easyeda_info(partno, get_model=get_model) @@ -215,6 +248,10 @@ def attach(component: Module, partno: str, get_model: bool = True): component.add(F.has_descriptive_properties_defined({"LCSC": partno})) + datasheet = get_datasheet_url(easyeda_symbol) + if datasheet: + component.add(F.has_datasheet_defined(datasheet)) + # model done by kicad (in fp)