From abf533ea81af39991352efbd4ce990b4d036ff35 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:20:51 -0500 Subject: [PATCH 1/7] Add solo phi support to ingestion scripts --- install/helioviewer/jp2parser.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/install/helioviewer/jp2parser.py b/install/helioviewer/jp2parser.py index ec38cf749..9221fc605 100644 --- a/install/helioviewer/jp2parser.py +++ b/install/helioviewer/jp2parser.py @@ -101,6 +101,8 @@ def getData(self): elif image['observatory'] == "Hinode": image['filter1'] = measurement.split("-")[0].replace(" ", "_") image['filter2'] = measurement.split("-")[1].replace(" ", "_") + elif image['observatory'] == 'Solar_Orbiter': + self._IdentifySolarOrbiterData(image, measurement, imageData) else: image['measurement'] = measurement image['date'] = imageData.date @@ -108,6 +110,24 @@ def getData(self): image['header'] = imageData.meta return image + + def _IdentifySolarOrbiterData(self, image, measurement, imageData): + assert image['observatory'] == 'Solar_Orbiter' + try: + if image['detector'] == 'FDT': + btype = imageData.meta['btype'].lower() + if btype == "longitudinal magnetic field": + measurement = "BLOS" + elif btype == "intensity [me]": + measurement = "ICNT" + elif image['detector'] == 'HRT': + measurement = imageData.meta['btype'].upper() + except Exception as e: + # If for whatever reason we can't read the btype from the image data + # then just continue with the default behavior. Which probably + # still won't work... + pass + image['measurement'] = measurement def read_header_only_but_still_use_sunpy_map(self, patch_cunit=False): """ From 9e67a4f869071ffdddbdf3c919707b1934f408a5 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:42:55 -0500 Subject: [PATCH 2/7] Add database properties for phi --- install/database/2024_02_09_solo_phi.sql | 25 ++++++++++++++++++++++++ install/helioviewer/db.py | 20 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 install/database/2024_02_09_solo_phi.sql diff --git a/install/database/2024_02_09_solo_phi.sql b/install/database/2024_02_09_solo_phi.sql new file mode 100644 index 000000000..a90ada8c6 --- /dev/null +++ b/install/database/2024_02_09_solo_phi.sql @@ -0,0 +1,25 @@ +INSERT INTO datasources (id, name, description, units, layeringOrder, enabled, sourceIdGroup, displayOrder) +VALUES +(95, 'SOLO PHI FDT BLOS', 'SOLO PHI FDT BLOS', NULL, 1, 0, '', 0), +(96, 'SOLO PHI FDT ICNT', 'SOLO PHI FDT ICNT', NULL, 1, 0, '', 0), +(97, 'SOLO PHI HRT BLOS', 'SOLO PHI HRT BLOS', NULL, 1, 0, '', 0), +(98, 'SOLO PHI HRT ICNT', 'SOLO PHI HRT ICNT', NULL, 1, 0, '', 0); + +INSERT INTO datasource_property (sourceId, label, name, fitsName, description, uiOrder) +VALUES +(95, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(96, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(97, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(98, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(95, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(96, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(97, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(98, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(95, 'Detector', 'FDT', 'FDT', 'Full Disk Telescope', 3), +(96, 'Detector', 'FDT', 'FDT', 'Full Disk Telescope', 3), +(97, 'Detector', 'HRT', 'HRT', 'High Resolution Telescope', 3), +(98, 'Detector', 'HRT', 'HRT', 'High Resolution Telescope', 3), +(95, 'Measurement', 'BLOS', 'BLOS', 'Line of Sight Magnetic Field', 4), +(96, 'Measurement', 'ICNT', 'ICNT', 'Intensity', 4), +(97, 'Measurement', 'BLOS', 'BLOS', 'Line of Sight Magnetic Field', 4), +(98, 'Measurement', 'ICNT', 'ICNT', 'Intensity', 4); \ No newline at end of file diff --git a/install/helioviewer/db.py b/install/helioviewer/db.py index 61119f4c3..35ae51bce 100644 --- a/install/helioviewer/db.py +++ b/install/helioviewer/db.py @@ -323,6 +323,10 @@ def create_datasource_table(cursor): (92, 'IRIS SJI 2832', 'IRIS SJI 2832', NULL, 1, 0, '', 0, 0, 0, 0), (93, 'IRIS SJI 5000', 'IRIS SJI 5000', NULL, 1, 0, '', 0, 0, 0, 0), (94, 'GONG H-alpha', 'GONG H-Alpha', NULL, 1, 0, '', 0, 0, 0, 0), +(95, 'SOLO PHI FDT BLOS', 'SOLO PHI FDT BLOS', NULL, 1, 0, '', 0, 0, 0, 0), +(96, 'SOLO PHI FDT ICNT', 'SOLO PHI FDT ICNT', NULL, 1, 0, '', 0, 0, 0, 0), +(97, 'SOLO PHI HRT BLOS', 'SOLO PHI HRT BLOS', NULL, 1, 0, '', 0, 0, 0, 0), +(98, 'SOLO PHI HRT ICNT', 'SOLO PHI HRT ICNT', NULL, 1, 0, '', 0, 0, 0, 0), (2000, 'GOES-R SUVI 94', 'GOES-R SUVI 94', NULL, 1, 0, '', 0, 0, 0, 0), (2001, 'GOES-R SUVI 131', 'GOES-R SUVI 131', NULL, 1, 0, '', 0, 0, 0, 0), (2002, 'GOES-R SUVI 171', 'GOES-R SUVI 171', NULL, 1, 0, '', 0, 0, 0, 0), @@ -419,6 +423,10 @@ def create_datasource_property_table(cursor): (92, 'Observatory', 'IRIS', 'IRIS', 'IRIS', 1), (93, 'Observatory', 'IRIS', 'IRIS', 'IRIS', 1), (94, 'Observatory', 'GONG', 'NSO-GONG', 'GONG', 1), +(95, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(96, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(97, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), +(98, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1), (2000, 'Observatory', 'GOES-R', 'GOES-R', 'GOES-R', 1), (2001, 'Observatory', 'GOES-R', 'GOES-R', 'GOES-R', 1), (2002, 'Observatory', 'GOES-R', 'GOES-R', 'GOES-R', 1), @@ -472,6 +480,10 @@ def create_datasource_property_table(cursor): (92, 'Instrument', 'SJI', 'SJI', 'Slit Jaw Imager', 2), (93, 'Instrument', 'SJI', 'SJI', 'Slit Jaw Imager', 2), (94, 'Instrument', 'GONG', 'GONG', 'GONG', 2), +(95, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(96, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(97, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), +(98, 'Instrument', 'PHI', 'PHI', 'Polarimetric and Helioseismic Imager', 2), (2000, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2), (2001, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2), (2002, 'Instrument', 'SUVI', 'SUVI', 'Solar UltraViolet Imager', 2), @@ -497,6 +509,10 @@ def create_datasource_property_table(cursor): (86, 'Detector', 'HRI', 'HRI_EUV', 'High Resolution Imager Extreme Ultraviolet', 3), (87, 'Detector', 'HRI', 'HRI_LYA', 'High Resolution Imager Lyman-a', 3), (94, 'Detector', 'H-alpha', 'H-alpha', 'H-alpha', 3), +(95, 'Detector', 'FDT', 'FDT', 'Full Disk Telescope', 3), +(96, 'Detector', 'FDT', 'FDT', 'Full Disk Telescope', 3), +(97, 'Detector', 'HRT', 'HRT', 'High Resolution Telescope', 3), +(98, 'Detector', 'HRT', 'HRT', 'High Resolution Telescope', 3), (0, 'Measurement', '171', '171', '171 Ångström extreme ultraviolet', 3), (1, 'Measurement', '195', '195', '195 Ångström extreme ultraviolet', 3), (2, 'Measurement', '284', '284', '284 Ångström extreme ultraviolet', 3), @@ -541,6 +557,10 @@ def create_datasource_property_table(cursor): (92, 'Measurement', '2832', '2832', '2832 Ångström', 3), (93, 'Measurement', '5000', '5000', '5000 Ångström', 3), (94, 'Measurement', '6562', '6562', 'H-alpha 6562 angstrom', 4), +(95, 'Measurement', 'BLOS', 'BLOS', 'Line of Sight Magnetic Field', 4), +(96, 'Measurement', 'ICNT', 'ICNT', 'Intensity', 4), +(97, 'Measurement', 'BLOS', 'BLOS', 'Line of Sight Magnetic Field', 4), +(98, 'Measurement', 'ICNT', 'ICNT', 'Intensity', 4), (2000, 'Measurement', '94', '94', '94 Ångström', 3), (2001, 'Measurement', '131', '131', '131 Ångström', 3), (2002, 'Measurement', '171', '171', '171 Ångström', 3), From 37122ca590085bd5b6fd5386854b999fd1aada86 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:47:28 -0500 Subject: [PATCH 3/7] Add grayscale PHIImage.php --- src/Image/ImageType/PHIImage.php | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/Image/ImageType/PHIImage.php diff --git a/src/Image/ImageType/PHIImage.php b/src/Image/ImageType/PHIImage.php new file mode 100644 index 000000000..68ee946ff --- /dev/null +++ b/src/Image/ImageType/PHIImage.php @@ -0,0 +1,47 @@ + + * @license http://www.mozilla.org/MPL/MPL-1.1.html Mozilla Public License 1.1 + * @link https://github.com/Helioviewer-Project + */ +require_once HV_ROOT_DIR.'/../src/Image/HelioviewerImage.php'; + +class Image_ImageType_PHIImage extends Image_HelioviewerImage { + /** + * Creates a new EUIImage + * + * @param string $jp2 Source JP2 image + * @param string $filepath Location to output the file to + * @param array $roi Top-left and bottom-right pixel coordinates on the image + * @param array $uiLabels Datasource label hierarchy + * @param int $offsetX Offset of the sun center from the image center + * @param int $offsetY Offset of the sun center from the iamge center + * @param array $options Optional parameters + * + * @return void + */ + public function __construct($jp2, $filepath, $roi, $uiLabels, $offsetX, $offsetY, $options) { + // Use grayscale for now + $this->setColorTable(false); + + parent::__construct($jp2, $filepath, $roi, $uiLabels, $offsetX, $offsetY, $options); + } + + /** + * Gets a string that will be displayed in the image's watermark + * + * @return string watermark name + */ + public function getWaterMarkName() { + $type = $this->uiLabels[2]['name']; + $measurement = $this->uiLabels[3]['name']; + $watermark = 'PHI '.$type.' '.$measurement."\n"; + return $watermark; + } +} +?> From d974acceb5285052d160970df65c3a82780f4d6e Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Fri, 9 Feb 2024 13:49:58 -0500 Subject: [PATCH 4/7] Fix typo --- src/Image/ImageType/PHIImage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Image/ImageType/PHIImage.php b/src/Image/ImageType/PHIImage.php index 68ee946ff..1564a19ee 100644 --- a/src/Image/ImageType/PHIImage.php +++ b/src/Image/ImageType/PHIImage.php @@ -13,7 +13,7 @@ class Image_ImageType_PHIImage extends Image_HelioviewerImage { /** - * Creates a new EUIImage + * Creates a new PHIImage * * @param string $jp2 Source JP2 image * @param string $filepath Location to output the file to From 19de2039da9b20d3aa2fe247abae5f9d23fe7f38 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:36:56 -0400 Subject: [PATCH 5/7] simplify solo btype detection --- install/helioviewer/jp2parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/helioviewer/jp2parser.py b/install/helioviewer/jp2parser.py index 9221fc605..cef815c3a 100644 --- a/install/helioviewer/jp2parser.py +++ b/install/helioviewer/jp2parser.py @@ -110,15 +110,15 @@ def getData(self): image['header'] = imageData.meta return image - + def _IdentifySolarOrbiterData(self, image, measurement, imageData): assert image['observatory'] == 'Solar_Orbiter' try: if image['detector'] == 'FDT': btype = imageData.meta['btype'].lower() - if btype == "longitudinal magnetic field": + if "magnetic field" in btype: measurement = "BLOS" - elif btype == "intensity [me]": + elif "intensity" in btype: measurement = "ICNT" elif image['detector'] == 'HRT': measurement = imageData.meta['btype'].upper() From 77e88615808e6fb77b6ac4f57e6c7c6e936720bd Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:41:46 -0400 Subject: [PATCH 6/7] Add PHIDataServer --- install/helioviewer/hvpull/net/daemon.py | 3 ++- install/helioviewer/hvpull/servers/phi.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 install/helioviewer/hvpull/servers/phi.py diff --git a/install/helioviewer/hvpull/net/daemon.py b/install/helioviewer/hvpull/net/daemon.py index afd340a6e..ecd93b6dc 100644 --- a/install/helioviewer/hvpull/net/daemon.py +++ b/install/helioviewer/hvpull/net/daemon.py @@ -918,7 +918,8 @@ def get_servers(cls): "suvi": "SUVIDataServer", "iris": "IRISDataServer", "hv_iris": "HvIRISDataServer", - "halpha": "GongDataServer" + "halpha": "GongDataServer", + "phi": "PHIDataServer" } @classmethod diff --git a/install/helioviewer/hvpull/servers/phi.py b/install/helioviewer/hvpull/servers/phi.py new file mode 100644 index 000000000..1813c72d2 --- /dev/null +++ b/install/helioviewer/hvpull/servers/phi.py @@ -0,0 +1,19 @@ +import os +from datetime import datetime + +from helioviewer.hvpull.servers import DataServer + +class PHIDataServer(DataServer): + def __init__(self): + """This assumes that SOLO jp2 files are stored locally. + Note that a full path is required to specify the location of the data.""" + DataServer.__init__(self, "/tmp/phi", "PHI") + + def compute_directories(self, start_date, end_date): + """Computes a list of remote directories expected to contain files""" + return [self.uri] + + def get_datetime_from_file(self, filename): + + t = os.path.basename(filename)[21:36] + return datetime.strptime(t, "%Y%m%dT%H%M%S") From 1fd9ca118bc1eb152f61dceff868eb66174e5026 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Briseno <94071409+dgarciabriseno@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:47:06 -0400 Subject: [PATCH 7/7] Fix localmove import error --- install/helioviewer/hvpull/downloader/localmove.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/helioviewer/hvpull/downloader/localmove.py b/install/helioviewer/hvpull/downloader/localmove.py index 88901084e..e18949e9d 100644 --- a/install/helioviewer/hvpull/downloader/localmove.py +++ b/install/helioviewer/hvpull/downloader/localmove.py @@ -1,10 +1,9 @@ """moves files from one location on the local file system to another""" import os import logging -import threading import time import shutil -from downloader_interface import Downloader +from .downloader_interface import Downloader class LocalFileMove(Downloader): def __init__(self, incoming, queue):