FF_works_paper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of the FF catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "from astropy.table import Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "legacy_url = 'http://archives.esac.esa.int/hsa/legacy/HPDP/SPIRE/SPIRE-S/spectral_feature_catalogue'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "obsid_list = Table.read(f'{legacy_url}/hrSparseObservations.csv')\n",
    "ntot = len(np.unique(obsid_list['obsid']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of unique observations in HR sparse: 868\n"
     ]
    }
   ],
   "source": [
    "print (\"Total number of unique observations in HR sparse:\", ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Column name=&apos;dataUsed&apos; dtype=&apos;str7&apos; length=3&gt;\n",
       "<table>\n",
       "<tr><td>bgs</td></tr>\n",
       "<tr><td>calHpdp</td></tr>\n",
       "<tr><td>spg</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Column name='dataUsed' dtype='str7' length=3>\n",
       "    bgs\n",
       "calHpdp\n",
       "    spg"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(obsid_list['dataUsed'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observations using SPG 618 71.19815668202764\n",
      "Observations using BKG HPDP 86 9.90783410138249\n",
      "Observations using cal HPDP 164 18.89400921658986\n",
      "868 868\n"
     ]
    }
   ],
   "source": [
    "nbkg = len(np.where(obsid_list['dataUsed'] == \"bgs\")[0])\n",
    "ncal = len(np.where(obsid_list['dataUsed'] == \"calHpdp\")[0])\n",
    "nspg = len(np.where(obsid_list['dataUsed'] == \"spg\")[0])\n",
    "print (\"Observations using SPG\", nspg,100*nspg/ntot)\n",
    "print (\"Observations using BKG HPDP\", nbkg,100*nbkg/ntot)\n",
    "print (\"Observations using cal HPDP\", ncal,100*ncal/ntot)\n",
    "print (nbkg + ncal + nspg,ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source is point-like 471 54.26267281105991\n",
      "Source is semi-extended 250 28.80184331797235\n",
      "Source is extended 147 16.93548387096774\n",
      "868 868\n"
     ]
    }
   ],
   "source": [
    "npnt = len(np.where(obsid_list['sourceExt'] == \"pointLike\")[0])\n",
    "nsec = len(np.where(obsid_list['sourceExt'] == \"semiExtended\")[0])\n",
    "nex = len(np.where(obsid_list['sourceExt'] == \"extended\")[0])\n",
    "print (\"Source is point-like\", npnt,100*npnt/ntot)\n",
    "print (\"Source is semi-extended\", nsec,100*nsec/ntot)\n",
    "print (\"Source is extended\", nex,100*nex/ntot)\n",
    "print (npnt + nsec + nex,ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mapping HR observations:  180\n"
     ]
    }
   ],
   "source": [
    "## Mapping\n",
    "obsid_list = Table.read(f'{legacy_url}/hrMappingObservations.csv')\n",
    "ntot = len(np.unique(obsid_list['obsid']))\n",
    "print ('Mapping HR observations: ',ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR sparse observations:  293\n"
     ]
    }
   ],
   "source": [
    "## Mapping\n",
    "obsid_list = Table.read(f'{legacy_url}/lrSparseObservations.csv')\n",
    "ntot = len(np.unique(obsid_list['obsid']))\n",
    "print ('LR sparse observations: ',ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mapping LR observations:  106\n"
     ]
    }
   ],
   "source": [
    "## Mapping\n",
    "obsid_list = Table.read(f'{legacy_url}/lrMappingObservations.csv')\n",
    "ntot = len(np.unique(obsid_list['obsid']))\n",
    "print ('Mapping LR observations: ',ntot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking with the SAFECAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SAFECAT total number of features: 167525\n"
     ]
    }
   ],
   "source": [
    "scat = Table.read(f'{legacy_url}/SpireAutomatedFeatureExtractionCATalogue/SAFECAT_v2_restFrame.fits.gz')\n",
    "print ('SAFECAT total number of features:',len(scat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<TableColumns names=('obsid','opDay','frequency','frequencyError','restFrameFreq','SNR','array','row','column','ra','dec','featureFlag','velocity','velocityError','vFlag','extent','calibration','sampling','hpdp','bgs','nccFlag')>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scat.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique OBSIDs 820\n"
     ]
    }
   ],
   "source": [
    "ntot = len(np.unique(scat['obsid']))\n",
    "print (\"Unique OBSIDs\", ntot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparse: 641\n",
      "Mapping: 179\n"
     ]
    }
   ],
   "source": [
    "# mapping vs sparse\n",
    "i1 = scat['sampling'] == 'sparse'\n",
    "n1 = len(np.unique(scat['obsid'][i1]))\n",
    "print (\"Sparse:\",n1)\n",
    "i2 = (scat['sampling'] == 'full') + (scat['sampling'] == 'intermediate')\n",
    "n2 = len(np.unique(scat['obsid'][i2]))\n",
    "print (\"Mapping:\",n2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flagged: 7691  or  4.590956573645725\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# check the flagged features\n",
    "#\n",
    "np.unique(scat['featureFlag'])\n",
    "i11 = (scat['featureFlag'] >= 1.0)\n",
    "n11 = len(scat['obsid'][i11])\n",
    "print (\"Flagged:\",n11, ' or ', 100*n11/len(scat))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NCCflag: 4781\n",
      "NCCflag, unique obsids: 162\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# check for the NCC flag\n",
    "#\n",
    "incc = scat['nccFlag'] == 1\n",
    "print (\"NCCflag:\", len(scat['obsid'][incc]))\n",
    "print (\"NCCflag, unique obsids:\", len(np.unique(scat['obsid'][incc])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}