Fix DIPOL bugs, add quad tests, improve parallelization, other improv…

…ements (#39)

* fix quad matching, add tests

* improve dipol polarimetry astrocalibration

* iop4config: set sqlite journal mode to WAL (#22)

  Now after each connection, iop4 configures sqlite with
  synchronous = NORMAL
  journal_mode = WAL

  this improves significantly concurrency during parallel reduction
 
  it also improves speed of the web portal and the admin site

* improve parallel reduction (#22)

  pass clear=True to photopolresult.reducedfits.set

* iop4admin: rebuild preview images if they are outdated

* dipol polarimetry: include handling of BLLac special case

  a close star image falls very close to the left image of BL Lac, skip those

* rawfit.py: fix setting read-only permissions

   previous behavior was equivalent to chmod 'a=r', now 'a-w'.

* implement parallel relative polarimetry

* improve astrometry configuration

* fix bug: mutable default arg value in function

* remove unused code for reduction in a ray cluster

  it was not maintained and it has not been used in a long time, no need for it

config/config.example.yaml

@@ -11,16 +11,9 @@ db_path: ~/.iop4data/iop4.db # Path to iop4 sqlite database file.
 astrometry_cache_path: ~/.astrometry_cache/ # <tr> Path to store the astromery index files.
 max_concurrent_threads: 4 # <int> Number of threads / processes to use (e.g. 4).
 astrometry_timeout: 20 # <int> Timeout in minutes for astrometry solving.
-
-###################
-### RAY CLUSTER ###
-###################
-
-ray_use_cluster: False # <bool> True/False (use ray for parallelization), let it to false if you have not configured it.
-ray_cluster_address: null # <str> Aaddress for the cluster, needs ssh keys for current user; e.g. 'user@address'.
-ray_cluster_config: null # <str> Path to ray cluster config file, e.g. /path/to/iop4/priv.rayconfig.yaml'.
-ray_db_path: null # <str> Path in ray cluster, e.g. '~/iop4data/iop4.db'.
-ray_datadir: null # <str> Path in ray cluster, e.g. '~/iop4data/'.
+astrometry_allsky_allow: false # <bool> Whether to allow all sky searches in the astrometry.net solver. If there are many all-sky images and max_concurrent_threads is too high, it might cause
+# very high memory consumption.
+astrometry_allsky_septhreshold: 25 # <int> Threshold of detected poinint mismatch to trigger an all-sky search.
 
 ################
 ### GRAPHICS ###

iop4admin/modeladmins/fitfile.py

@@ -23,6 +23,9 @@
 logger = logging.getLogger(__name__)
 
 class AdminFitFile(admin.ModelAdmin):
+
+    list_per_page = 25
+    list_max_show_all = 200
 
     @admin.display(description='TELESCOPE', ordering='epoch__telescope')
     def telescope(self, obj):

iop4admin/modeladmins/rawfit.py

@@ -18,7 +18,6 @@ class AdminRawFit(AdminFitFile):
     readonly_fields = [field.name for field in RawFit._meta.fields] 
     search_fields = ['id', 'filename', 'epoch__telescope', 'epoch__night'] 
     ordering = ['-epoch__night','-epoch__telescope']
-    list_per_page = 25
     list_filter = (
             RawFitIdFilter,
             RawFitTelescopeFilter,

iop4admin/modeladmins/reducedfit.py

@@ -22,7 +22,6 @@ class AdminReducedFit(AdminFitFile):
     readonly_fields = [field.name for field in ReducedFit._meta.fields]
     search_fields = ['id', 'filename', 'epoch__telescope', 'epoch__night', 'sources_in_field__name']
     ordering = ['-epoch__night', '-epoch__telescope', '-juliandate']
-    list_per_page = 25
     list_filter = (
         RawFitIdFilter,
         RawFitTelescopeFilter,

iop4admin/templates/iop4admin/view_fitdetails.html

@@ -118,13 +118,13 @@ <h2>Reduction information:</h2>
 		<h3>Calibration Frames</h3>
 		<ul>
 		{% if object.masterbias %}
-			<li><a href="{% url 'iop4admin:iop4api_masterbias_changelist' %}?id={{ object.masterbias.id }}">MasterBias {{ object.masterbias.id }}</a></li>
+			<li><a href="{% url 'iop4admin:iop4api_masterbias_changelist' %}?id={{ object.masterbias.id }}">MasterBias {{ object.masterbias.id }}: <code>{{ object.masterbias }}</code></a></li>
 		{% endif %}
 		{% if object.masterflat %}
-			<li><a href="{% url 'iop4admin:iop4api_masterflat_changelist' %}?id={{ object.masterflat.id }}">MasterFlat {{ object.masterflat.id }}</a></li>
+			<li><a href="{% url 'iop4admin:iop4api_masterflat_changelist' %}?id={{ object.masterflat.id }}">MasterFlat {{ object.masterflat.id }}: <code>{{ object.masterflat }}</code></a></li>
 		{% endif %}
 		{% if object.masterdark %}
-		<li><a href="{% url 'iop4admin:iop4api_masterdark_changelist' %}?id={{ object.masterdark.id }}">MasterDark {{ object.masterdark.id }}</a></li>
+		<li><a href="{% url 'iop4admin:iop4api_masterdark_changelist' %}?id={{ object.masterdark.id }}">MasterDark {{ object.masterdark.id }}: <code>{{ object.masterdark }}</code></a></li>
 		{% endif %}
 		</ul>
 
@@ -315,6 +315,8 @@ <h3>Header {{forloop.counter }}:</h3> <small><i>(scroll down)</i></small> <br>
 
 	#img_preview_range_form label {
 		text-align: right;
+		display: flex;
+  		align-items: center;
 	}
 
 	/* header div */

iop4lib/config.py

@@ -2,9 +2,24 @@
 import os, pathlib, yaml, logging
 
 import matplotlib, matplotlib.pyplot
+
+# Disable matplotlib logging except for warnings and above
+
 matplotlib.pyplot.set_loglevel('warning')
 
+# Set journal_mode=WAL and synchronous=NORMAL for sqlite3 databases on 
+# connection, to improve support for concurrent write access to the 
+# database during parallel reduction
+
+from django.db.backends.signals import connection_created
+from django.dispatch.dispatcher import receiver
 
+@receiver(connection_created)    
+def enable_wal_sqlite(sender, connection, **kwargs) -> None:
+    if connection.vendor == "sqlite":
+        with connection.cursor() as cursor:
+            cursor.execute('PRAGMA synchronous=NORMAL;')
+            cursor.execute('PRAGMA journal_mode=WAL;')
 
 class Config(dict):
     r""" Configuration class for IOP4.

iop4lib/db/aperphotresult.py

@@ -26,6 +26,8 @@ class AperPhotResult(models.Model):
     reducedfit = models.ForeignKey("ReducedFit", on_delete=models.CASCADE, related_name='aperphotresults', help_text="The ReducedFit this AperPhotResult has been computed for.")
     astrosource = models.ForeignKey("AstroSource", on_delete=models.CASCADE, related_name='aperphotresults', help_text="The AstroSource this AperPhotResult has been computed for.")
     aperpix = models.FloatField(null=True, blank=True)
+    r_in = models.FloatField(null=True, blank=True)
+    r_out = models.FloatField(null=True, blank=True)
     pairs = models.TextField(null=True, blank=True, choices=PAIRS.choices, help_text="Whether this AperPhotResult is for the Ordinary or Extraordinary pair.")
 
     ## photometry results
@@ -46,7 +48,7 @@ class Meta:
         verbose_name = 'Aperture Photometry Result'
         verbose_name_plural = 'Aperture Photometry Results'
         constraints = [
-            models.UniqueConstraint(fields=['reducedfit', 'astrosource', 'aperpix', 'pairs'], name='unique_aperphotresult')
+            models.UniqueConstraint(fields=['reducedfit', 'astrosource', 'aperpix', 'r_in', 'r_out', 'pairs'], name='unique_aperphotresult')
         ]
 
   # repr and str

iop4lib/db/epoch.py

@@ -27,7 +27,7 @@
 from iop4lib.instruments import Instrument
 from .fields import FlagChoices, FlagBitField
 from iop4lib.utils import  get_mem_parent_from_child, get_total_mem_from_child, get_mem_current, get_mem_children
-from iop4lib.utils.parallel import epoch_bulkreduce_multiprocesing, epoch_bulkreduce_ray
+from iop4lib.utils.parallel import epoch_bulkreduce_multiprocesing
 
 # logging
 
@@ -478,9 +478,7 @@ def reduce_reducedfits(reduced_L, epoch=None):
         """
 
         if len(reduced_L) > 0:
-            if iop4conf.ray_use_cluster:
-                epoch_bulkreduce_ray(reduced_L)
-            elif iop4conf.max_concurrent_threads > 1:
+            if iop4conf.max_concurrent_threads > 1:
                 epoch_bulkreduce_multiprocesing(reduced_L, epoch=epoch)
             else:
                 epoch_bulkreduce_onebyone(reduced_L, epoch=epoch)
@@ -500,17 +498,29 @@ def by_epochname(cls, epochname):
 
 
 
-    def compute_relative_photometry(self):
+    def compute_relative_photometry(self, redf_qs=None):
+        """ Computes the relative photometry results for this epoch.
+
+        Parameters
+        ----------
+        redf_qs : QuerySet, optional
+            If provided, the relative photometry will be computed only for the ReducedFit objects in the QuerySet.
+            If not provided, the relative photometry will be computed for all ReducedFit objects in the epoch.
+        """
+
         from .reducedfit import ReducedFit
 
-        redf_qs = ReducedFit.objects.filter(epoch=self, obsmode=OBSMODES.PHOTOMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
+        if redf_qs is None:
+            redf_qs = ReducedFit.objects.filter(epoch=self, obsmode=OBSMODES.PHOTOMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
+        else:
+            redf_qs = redf_qs.filter(epoch=self, obsmode=OBSMODES.PHOTOMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
 
         for redf in redf_qs:
             redf.compute_relative_photometry()
 
 
 
-    def make_polarimetry_groups(self):
+    def make_polarimetry_groups(self, redf_qs=None):
         """
         To reduce the polarimetry data, we need to group the reduced fits corresponding to rotated polarization angles.
 
@@ -525,7 +535,10 @@ def make_polarimetry_groups(self):
 
         from .reducedfit import ReducedFit
 
-        redf_qs = ReducedFit.objects.filter(epoch=self, obsmode=OBSMODES.POLARIMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
+        if redf_qs is None:
+            redf_qs = ReducedFit.objects.filter(epoch=self, obsmode=OBSMODES.POLARIMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
+        else:
+            redf_qs = redf_qs.filter(epoch=self, obsmode=OBSMODES.POLARIMETRY, flags__has=ReducedFit.FLAGS.BUILT_REDUCED).order_by('juliandate').all()
 
         # Create a groups with the same keys (object, band, exptime)
 
@@ -592,23 +605,38 @@ def make_polarimetry_groups(self):
         return split_groups, split_groups_keys
 
 
-    def compute_relative_polarimetry(self, *args, **kwargs):
+    def compute_relative_polarimetry(self, redf_qs=None):
+        """Computes the relative polarimetry results for this epoch.
+
+        Parameters
+        ----------
+        redf_qs : QuerySet, optional
+            If provided, the relative polarimetry will be computed only for the ReducedFit objects in the QuerySet.
+            If not provided, the relative polarimetry will be computed for all ReducedFit objects in the epoch.
+        """
+
         from .reducedfit import ReducedFit
 
-        clusters_L, groupkeys_L = self.make_polarimetry_groups()
+        clusters_L, groupkeys_L = self.make_polarimetry_groups(redf_qs=redf_qs)
 
         logger.info(f"{self}: computing relative polarimetry over {len(groupkeys_L)} polarimetry groups.")
         logger.debug(f"{self}: {groupkeys_L=}")
 
-        f = lambda x: Instrument.by_name(x[1]['instrument']).compute_relative_polarimetry(x[0], *args, **kwargs)
-
-        for i, (group, keys) in enumerate(zip(clusters_L, groupkeys_L)):
-            try:
-                Instrument.by_name(keys['instrument']).compute_relative_polarimetry(group, *args, **kwargs)
-            except Exception as e:
-                logger.error(f"{self}: error computing relative polarimetry for group n {i} {keys}: {e}")
-            finally:
-                logger.info(f"{self}: computed relative polarimetry for group n {i} {keys}.")
+        f = lambda x: Instrument.by_name(x[1]['instrument']).compute_relative_polarimetry(x[0])
+
+        if iop4conf.max_concurrent_threads > 1:
+            # parallel
+            from iop4lib.utils.parallel import parallel_relative_polarimetry
+            parallel_relative_polarimetry(groupkeys_L, clusters_L)
+        else: 
+            # one by one
+            for i, (group, keys) in enumerate(zip(clusters_L, groupkeys_L)):
+                try:
+                    Instrument.by_name(keys['instrument']).compute_relative_polarimetry(group)
+                except Exception as e:
+                    logger.error(f"{self}: error computing relative polarimetry for group n {i} {keys}: {e}")
+                finally:
+                    logger.info(f"{self}: computed relative polarimetry for group n {i} {keys}.")
 
 
 

iop4lib/db/fitfilemodel.py

@@ -168,7 +168,7 @@ def get_imgbytes_preview_image(self, force_rebuild=False, **kwargs):
         fpath = os.path.join(self.filedpropdir, "img_preview_image.png")
 
         if len(kwargs) == 0 and not force_rebuild:
-            if os.path.isfile(fpath):
+            if os.path.isfile(fpath) and os.path.getmtime(self.filepath) < os.path.getmtime(fpath):
                 # logger.debug(f"Loading image preview for {self.fileloc} from disk")
                 with open(fpath, 'rb') as f:
                     return f.read()
@@ -191,6 +191,20 @@ def get_imgbytes_preview_image(self, force_rebuild=False, **kwargs):
         fig = mplt.figure.Figure(figsize=(width/100, height/100), dpi=iop4conf.mplt_default_dpi)
         ax = fig.subplots()
         ax.imshow(imgdata, cmap=cmap, origin='lower', norm=norm)
+        try:
+            from iop4lib.db import ReducedFit, AstroSource
+            from iop4lib.enums import SRCTYPES
+            # If it is a astro calibrated reduced fit, mark the src position
+            if self.has_flag(ReducedFit.FLAGS.BUILT_REDUCED):
+                if (target_src := self.sources_in_field.exclude(srctype=SRCTYPES.CALIBRATOR).get()) is not None:
+                    target_pos_px = target_src.coord.to_pixel(self.wcs1)
+                    ax.axhline(y=target_pos_px[1], color='r', linestyle="--", linewidth=1)
+                    ax.axvline(x=target_pos_px[0], color='r', linestyle="--", linewidth=1)
+        except Exception as e: 
+            # it can fail if there is any problem with the fit calibration (e.g. in early versions the wcs
+            # was not saved into key A, but we still want to be able to explore the data in the admin).
+            logger.warning(f"Coudl not mark the target source position on ReducedFit {self.pk}: {e}")
+
         ax.axis('off')
         fig.savefig(buf, format='png', bbox_inches='tight', pad_inches=0)
         fig.clf()
@@ -230,7 +244,7 @@ def get_imgbytes_preview_histogram(self, force_rebuild=False, **kwargs):
         fpath = os.path.join(self.filedpropdir, "img_preview_histogram.png")
 
         if len(kwargs) == 0 and not force_rebuild:
-            if os.path.isfile(fpath):
+            if os.path.isfile(fpath) and os.path.getmtime(self.filepath) < os.path.getmtime(fpath):
                 #logger.debug(f"Loading preview histogram for {self.fileloc} from disk")
                 with open(fpath, 'rb') as f:
                     return f.read()

iop4lib/db/photopolresult.py

@@ -188,7 +188,7 @@ def create(cls, reducedfits, astrosource, reduction, **kwargs):
             logger.debug(f'Creating DB entry photopolresult for {reducedfits=}, {astrosource=} and {reduction=}')
             photopolresult = cls(**kwargs)
             photopolresult.save() # we need to save it to use the manytomany field
-            photopolresult.reducedfits.set(reducedfits, through_defaults={'astrosource': astrosource, 'reduction': reduction})
+            photopolresult.reducedfits.set(reducedfits, through_defaults={'astrosource': astrosource, 'reduction': reduction}, clear=True)
             photopolresult.save()
         else:
             logger.debug(f'Db entry for photopolresult already exists for {reducedfits=}, {astrosource=} and {reduction=}, using it instead.')
@@ -292,4 +292,5 @@ def save(self, *args, **kwargs):
 def reducedfits_on_change(sender, instance, action, **kwargs):
     """ Updates automatically filled fields when -after- reducedfits are altered."""
     if action.startswith('post_'):
-        instance.update_fields()
+        if instance.reducedfits.count() > 0:
+            instance.update_fields()

iop4lib/db/rawfit.py

@@ -282,7 +282,10 @@ def procure_local_file(self):
 
         if self.fileexists:
             if iop4conf.set_rawdata_readonly:
-                os.chmod(self.filepath, stat.S_IREAD)
+                # this will remove the write permission from the file for all users
+                current_perms = os.stat(self.filepath).st_mode
+                new_perms = current_perms & (~stat.S_IWUSR) & (~stat.S_IWGRP) & (~stat.S_IWOTH)
+                os.chmod(self.filepath, new_perms)
 
             if self.auto_classify:
                 self.classify()

iop4lib/db/reducedfit.py

@@ -207,8 +207,8 @@ def header_hintcoord(self):
     def header_hintobject(self):
         return self.rawfit.header_hintobject
 
-    def get_astrometry_position_hint(self, allsky=False, n_field_width=1.5):
-        return Instrument.by_name(self.instrument).get_astrometry_position_hint(self.rawfit, allsky=allsky,  n_field_width=n_field_width)
+    def get_astrometry_position_hint(self, allsky=False, n_field_width=1.5, hintsep=None):
+        return Instrument.by_name(self.instrument).get_astrometry_position_hint(self.rawfit, allsky=allsky,  n_field_width=n_field_width, hintsep=hintsep)
 
     def get_astrometry_size_hint(self):
         return Instrument.by_name(self.instrument).get_astrometry_size_hint(self.rawfit)

iop4lib/instruments/andor_cameras.py

@@ -149,17 +149,28 @@ def get_header_hintcoord(cls, rawfit):
         return hint_coord
 
     @classmethod
-    def get_astrometry_position_hint(cls, rawfit, allsky=False, n_field_width=1.5):
-        """ Get the position hint from the FITS header as an astrometry.PositionHint."""        
+    def get_astrometry_position_hint(cls, rawfit, allsky=False, n_field_width=1.5, hintsep=None):
+        """ Get the position hint from the FITS header as an astrometry.PositionHint.
+
+        Parameters
+        ----------
+            allsky: bool, optional
+                If True, the hint will cover the whole sky, and n_field_width and hintsep will be ignored.
+            n_field_width: float, optional
+                The search radius in units of field width. Default is 1.5.
+            hintsep: Quantity, optional
+                The search radius in units of degrees.
+        """        
 
         hintcoord = cls.get_header_hintcoord(rawfit)
 
         if allsky:
-            hintsep = 180.0
+            hintsep = 180.0 * u.deg
         else:
-            hintsep = (n_field_width * cls.field_width_arcmin*u.Unit("arcmin")).to_value(u.deg)
+            if hintsep is None:
+                hintsep = (n_field_width * cls.field_width_arcmin*u.Unit("arcmin"))
 
-        return astrometry.PositionHint(ra_deg=hintcoord.ra.deg, dec_deg=hintcoord.dec.deg, radius_deg=hintsep)
+        return astrometry.PositionHint(ra_deg=hintcoord.ra.deg, dec_deg=hintcoord.dec.deg, radius_deg=hintsep.to_value("deg"))
 
     @classmethod
     def get_astrometry_size_hint(cls, rawfit):

iop4lib/instruments/cafos.py

@@ -116,17 +116,28 @@ def get_header_hintcoord(cls, rawfit):
         return hint_coord
 
     @classmethod
-    def get_astrometry_position_hint(cls, rawfit, allsky=False, n_field_width=1.5):
-        """ Get the position hint from the FITS header as an astrometry.PositionHint object. """        
+    def get_astrometry_position_hint(cls, rawfit, allsky=False, n_field_width=1.5, hintsep=None):
+        """ Get the position hint from the FITS header as an astrometry.PositionHint object. 
+
+        Parameters
+        ----------
+            allsky: bool, optional
+                If True, the hint will cover the whole sky, and n_field_width and hintsep will be ignored.
+            n_field_width: float, optional
+                The search radius in units of field width. Default is 1.5.
+            hintsep: Quantity, optional
+                The search radius in units of degrees.
+        """        
 
         hintcoord = cls.get_header_hintcoord(rawfit)
 
         if allsky:
-            hintsep = 180
+            hintsep = 180.0 * u.deg
         else:
-            hintsep = n_field_width * u.Quantity("16 arcmin").to_value(u.deg) # 16 arcmin is the full field size of the CAFOS T2.2, our cut is smaller (6.25, 800x800, but the pointing kws might be from anywhere in the full field)
+            if hintsep is None:
+                hintsep = n_field_width * u.Quantity("16 arcmin") # 16 arcmin is the full field size of the CAFOS T2.2, our cut is smaller (6.25, 800x800, but the pointing kws might be from anywhere in the full field)
 
-        return astrometry.PositionHint(ra_deg=hintcoord.ra.deg, dec_deg=hintcoord.dec.deg, radius_deg=hintsep)
+        return astrometry.PositionHint(ra_deg=hintcoord.ra.deg, dec_deg=hintcoord.dec.deg, radius_deg=hintsep.to_value(u.deg))
 
     @classmethod
     def get_astrometry_size_hint(cls, rawfit):