Merge pull request #37 from IKZ-Berlin/transmission-backcompatibility

Add new and old transmission schemas

.github/workflows/python-test.yml

@@ -0,0 +1,39 @@
+# This workflow will install Python dependencies, run tests, and lint with multiple version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+env:
+  UV_SYSTEM_PYTHON: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{matrix.python_version}}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{matrix.python_version}}
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+    - name: Install dependencies
+      run: |
+        uv pip install -e '.[dev]'
+    - name: Test with pytest
+      run: |
+        pytest

nomad.yaml

@@ -0,0 +1,3 @@
+plugins:
+  exclude:
+    - nomad_measurements.transmission:parser

pyproject.toml

@@ -39,8 +39,8 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "ruff",
     "pytest",
+    "ruff",
     "structlog",
     "python-logstash>=0.4.6",
 ]
@@ -53,6 +53,7 @@ dev = [
 index-url = "https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple"
 
 [tool.ruff]
+include = ["src/*.py", "tests/*.py"]
 # Exclude a variety of commonly ignored directories.
 exclude = [
     ".bzr",
@@ -111,6 +112,7 @@ ignore = [
     "PLW2901", # redefined-loop-name
     "PLR1714", # consider-using-in
     "PLR5501", # else-if-used
+    "F403", # 'from module import *' used; unable to detect undefined names
 ]
 
 fixable = ["ALL"]
@@ -138,11 +140,11 @@ package-dir = { "" = "src" }
 [tool.setuptools.packages.find]
 where = ["src"]
 
-[tool.setuptools_scm]
-
 [project.entry-points.'nomad.plugin']
 general_schema = "nomad_ikz_plugin.general:schema"
 characterization_schema = "nomad_ikz_plugin.characterization:schema"
+characterization_transmission_parser = "nomad_ikz_plugin.characterization:transmission_parser"
+deprecated_characterization_schema = "nomad_ikz_plugin.deprecated.characterization:schema"
 pld_schema = "nomad_ikz_plugin.pld:schema"
 movpe_schema = "nomad_ikz_plugin.movpe:schema"
 movpe2_growth_excel_parser = "nomad_ikz_plugin.movpe.movpe2.growth_excel:parser"
@@ -158,4 +160,4 @@ movpe_substrate_app = "nomad_ikz_plugin.movpe.movpe_app:movpesubstrateapp"
 movpe_growth_run_app = "nomad_ikz_plugin.movpe.movpe_app:movpegrowthrunapp"
 movpe_layers_app = "nomad_ikz_plugin.movpe.movpe_app:movpelayersapp"
 
-
+[tool.setuptools_scm]

src/nomad_ikz_plugin/characterization/__init__.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-from nomad.config.models.plugins import SchemaPackageEntryPoint
+from nomad.config.models.plugins import ParserEntryPoint, SchemaPackageEntryPoint
 
 
 class CharacterizationEntryPoint(SchemaPackageEntryPoint):
@@ -25,7 +25,25 @@ def load(self):
         return m_package
 
 
+class TransmissionParserEntryPoint(ParserEntryPoint):
+    """
+    Entry point for lazy loading of the TransmissionParser.
+    """
+
+    def load(self):
+        from nomad_ikz_plugin.characterization.parser import TransmissionParser
+
+        return TransmissionParser(**self.dict())
+
+
 schema = CharacterizationEntryPoint(
     name='CharacterizationSchema',
     description='Schema package for general characterization methods used at IKZ.',
 )
+
+transmission_parser = TransmissionParserEntryPoint(
+    name='Transmission Parser',
+    description='Parser for data from Transmission Spectrophotometry.',
+    mainfile_mime_re='text/.*|application/zip',
+    mainfile_name_re=r'^.*\.asc$',
+)

src/nomad_ikz_plugin/characterization/parser.py

@@ -0,0 +1,50 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import TYPE_CHECKING
+
+from nomad.parsing import MatchingParser
+from nomad_measurements.transmission.schema import RawFileTransmissionData
+from nomad_measurements.utils import create_archive
+
+from nomad_ikz_plugin.characterization.schema import IKZELNUVVisNirTransmission
+
+if TYPE_CHECKING:
+    from nomad.datamodel.datamodel import (
+        EntryArchive,
+    )
+
+
+class TransmissionParser(MatchingParser):
+    """
+    Parser for matching files from Transmission Spectrophotometry and
+    creating instances of ELN.
+    """
+
+    def parse(
+        self, mainfile: str, archive: 'EntryArchive', logger=None, child_archives=None
+    ) -> None:
+        data_file = mainfile.split('/')[-1]
+        entry = IKZELNUVVisNirTransmission.m_from_dict(
+            IKZELNUVVisNirTransmission.m_def.a_template
+        )
+        entry.data_file = data_file
+        file_name = f'{".".join(data_file.split(".")[:-1])}.archive.json'
+        archive.data = RawFileTransmissionData(
+            measurement=create_archive(entry, archive, file_name)
+        )
+        archive.metadata.entry_name = f'{data_file} data file'

src/nomad_ikz_plugin/characterization/schema.py

@@ -1,17 +1,38 @@
+from typing import TYPE_CHECKING
+
 import numpy as np
+import plotly.express as px
 from nomad.config import config
 from nomad.datamodel.data import EntryData
+from nomad.datamodel.metainfo.annotations import (
+    ELNAnnotation,
+    Filter,
+    SectionProperties,
+)
 from nomad.datamodel.metainfo.basesections import (
     Measurement,
     MeasurementResult,
 )
+from nomad.datamodel.metainfo.plot import (
+    PlotlyFigure,
+)
 from nomad.metainfo import Datetime, MEnum, Quantity, SchemaPackage, Section, SubSection
+from nomad_measurements.transmission.schema import (
+    ELNUVVisNirTransmission,
+    UVVisNirTransmissionResult,
+    UVVisNirTransmissionSettings,
+)
 
 from nomad_ikz_plugin.general.schema import (
     IKZCategory,
     SubstratePreparationStep,
 )
 
+if TYPE_CHECKING:
+    from nomad.datamodel.datamodel import EntryArchive
+    from structlog.stdlib import BoundLogger
+
+
 configuration = config.get_plugin_entry_point(
     'nomad_ikz_plugin.characterization:schema'
 )
@@ -136,4 +157,182 @@ class LightMicroscope(Measurement, SubstratePreparationStep, EntryData):
     )
 
 
+class IKZUVVisNirTransmissionSettings(UVVisNirTransmissionSettings):
+    """
+    A specialized section for IKZ based on the `UVVisNirTransmissionSettings` section.
+    """
+
+    ordinate_type = Quantity(
+        type=MEnum(['%T', 'A']),
+        description=(
+            'Specifies whether the ordinate (y-axis) of the measurement data is '
+            'percent transmittance (%T) or absorbance (A).'
+        ),
+        a_eln={'component': 'EnumEditQuantity'},
+    )
+
+
+class IKZUVVisNirTransmissionResult(UVVisNirTransmissionResult):
+    """
+    A specialized section for IKZ based on the `UVVisNirTransmissionResult` section.
+    """
+
+    m_def = Section(
+        a_eln=ELNAnnotation(
+            properties=SectionProperties(
+                order=[
+                    'transmittance',
+                    'absorbance',
+                    'wavelength',
+                    'extinction_coefficient',
+                ],
+                visible=Filter(
+                    exclude=[
+                        'array_index',
+                    ],
+                ),
+            )
+        )
+    )
+    extinction_coefficient = Quantity(
+        type=np.float64,
+        description=(
+            'Extinction coefficient calculated from transmittance and sample thickness '
+            'values: -log(T)/L. The coefficient includes the effects of '
+            'absorption, reflection, and scattering.'
+        ),
+        shape=['*'],
+        unit='1/m',
+        a_plot={'x': 'array_index', 'y': 'extinction_coefficient'},
+    )
+
+    def generate_plots(self) -> list[PlotlyFigure]:
+        """
+        Extends UVVisNirTransmissionResult.generate_plots() method to include the plotly
+        figures for the `IKZUVVisNirTransmissionResult` section.
+
+        Returns:
+            list[PlotlyFigure]: The plotly figures.
+        """
+        figures = super().generate_plots()
+        if self.wavelength is None:
+            return figures
+
+        # generate plot for extinction coefficient
+        if self.extinction_coefficient is None:
+            return figures
+
+        x = self.wavelength.to('nm').magnitude
+        x_label = 'Wavelength'
+        xaxis_title = x_label + ' (nm)'
+
+        y = self.extinction_coefficient.to('1/cm').magnitude
+        y_label = 'Extinction coefficient'
+        yaxis_title = y_label + ' (1/cm)'
+
+        line_linear = px.line(x=x, y=y)
+
+        line_linear.update_layout(
+            title=f'{y_label} over {x_label}',
+            xaxis_title=xaxis_title,
+            yaxis_title=yaxis_title,
+            xaxis=dict(
+                fixedrange=False,
+            ),
+            yaxis=dict(
+                fixedrange=False,
+            ),
+            template='plotly_white',
+        )
+
+        figures.append(
+            PlotlyFigure(
+                label=f'{y_label} linear plot',
+                figure=line_linear.to_plotly_json(),
+            ),
+        )
+
+        return figures
+
+    def calculate_extinction_coefficient(self, archive, logger):
+        """
+        Calculate the extinction coefficient from the transmittance and sample
+        thickness. The formula used is: -log( T[%] / 100 ) / L.
+
+        Args:
+            archive (EntryArchive): The archive containing the section.
+            logger (BoundLogger): A structlog logger.
+        """
+        self.extinction_coefficient = None
+        if not archive.data.samples:
+            logger.warning(
+                'Cannot calculate extinction coefficient as sample not found.'
+            )
+            return
+        if not archive.data.samples[0].thickness:
+            logger.warning(
+                'Cannot calculate extinction coefficient as sample thickness not found '
+                'or the value is 0.'
+            )
+            return
+
+        path_length = archive.data.samples[0].thickness
+        if self.transmittance is not None:
+            extinction_coeff = -np.log(self.transmittance) / path_length
+            # TODO: The if-block is a temperary fix to avoid processing of nans in
+            # the archive. The issue will be fixed in the future.
+            if np.any(np.isnan(extinction_coeff)):
+                logger.warning(
+                    'Failed to save extinction coefficient. '
+                    'Encountered NaN values in the calculation.'
+                )
+                return
+            self.extinction_coefficient = extinction_coeff
+
+    def normalize(self, archive: 'EntryArchive', logger: 'BoundLogger') -> None:
+        """
+        The normalizer for the `IKZUVVisNirTransmissionResult` class.
+
+        Args:
+            archive (EntryArchive): The archive containing the section that is being
+            normalized.
+            logger (BoundLogger): A structlog logger.
+        """
+        super().normalize(archive, logger)
+        self.calculate_extinction_coefficient(archive, logger)
+
+
+class IKZELNUVVisNirTransmission(ELNUVVisNirTransmission):
+    """
+    A specialized section for IKZ based on the `ELNUVVisNirTransmission` section.
+    """
+
+    m_def = Section(
+        categories=[IKZCategory],
+        label='IKZ UV-Vis-NIR Transmission',
+        a_template={
+            'measurement_identifiers': {},
+        },
+    )
+    results = SubSection(
+        section_def=IKZUVVisNirTransmissionResult,
+        repeats=True,
+    )
+    transmission_settings = SubSection(
+        section_def=IKZUVVisNirTransmissionSettings,
+    )
+
+    def write_transmission_data(self, transmission, data_dict, archive, logger):
+        """
+        Specialized method to write the transmission data for the IKZ plugin. The method
+        overrides the `write_transmission_data` method of the parent
+        `ELNUVVisNirTransmission` class.
+        """
+        super().write_transmission_data(transmission, data_dict, archive, logger)
+        if data_dict['ordinate_type'] in ['%T', 'A']:
+            transmission.transmission_settings.ordinate_type = data_dict.get(
+                'ordinate_type'
+            )
+
+
 m_package.__init_metainfo__()