Merge remote-tracking branch 'origin/issue17'

inc/fits.h

@@ -1,8 +1,10 @@
 /*
- *  definitions and declarations for nemo's FITS.C
+ *  definitions and declarations for nemo's src/image/fits/fits.c
+ *
  *  22-feb-94  ansi- and C++ safe
  *  10-aug-09  size_t instead of int for 2GB+ files
  *  25-oct-20  add hdu counter
+ *   3-jan-24  more proper usage of size_t instead of int
  */
 
 #ifndef _fits_h_
@@ -129,7 +131,7 @@ static struct arglist {
 typedef struct fits_header {
     int hdu;            /* keep track with HDU this is (1 being the first) */
     /* Optionally we could store the buffer in the first structure too... */
-    char *buffer;       /* pointer to exact header copy */
+    char *buffer;       /* pointer to exact header copy , if present */
     size_t buflen;      /* current length of buffer */
     size_t hlen;        /* length of header (bytes, through the END keyword) */
     size_t dlen;        /* length of data (bytes) */
@@ -200,8 +202,8 @@ typedef struct fits_header {
 } fits_header;
 
 
-size_t fts_rhead    (fits_header *, stream);
-char *fts_shead    (fits_header *, string);
+size_t fts_rhead (fits_header *, stream);
+char *fts_shead  (fits_header *, string);
 int fts_lhead    (fits_header *);
 int fts_chead    (fits_header *, stream);
 int fts_thead    (fits_header *);
@@ -211,7 +213,7 @@ int fts_ihead    (fits_header *, string *);
 int fts_fhead    (fits_header *, string *);
 int fts_phead    (fits_header *, string *);
 int fts_whead    (fits_header *, stream);
-size_t fts_xhead    (fits_header *, stream, int, int, int *, int);
+size_t fts_xhead (fits_header *, stream, int, int, int *, int);
 
 int fts_ptable   (fits_header *, stream, string *, string, int *, int);
 int fts_pgroup   (fits_header *, stream, string *, string, int *, int);
@@ -220,8 +222,8 @@ int fts_buf      (int);
 
 int fts_sdata    (fits_header *, stream);
 int fts_cdata    (fits_header *, stream, stream, bool, bool);
-int fts_dsize    (fits_header *);
-int fts_tsize    (fits_header *);
+size_t fts_dsize (fits_header *);
+size_t fts_tsize (fits_header *);
 int fts_rdata    (fits_header *, stream, int, char *);
 int fts_wdata    (fits_header *, stream, int, char *);
 int fts_rrow     (fits_header *, stream, int, char *);

man/man3/filestruct.3

@@ -1,4 +1,4 @@
-.TH FILESTRUCT 3NEMO "7 April 2012"
+.TH FILESTRUCT 3NEMO "2 January 2024"
 
 .SH "NAME"
 filestruct \- primitives for structured binary file I/O
@@ -274,11 +274,12 @@ https://github.com/syrte/snapio - python implementation of snap I/O
 
 .SH "UPDATE HISTORY"
 .nf
-.ta +2.0i +2.0i
+.ta +1.25i +4.5i
 4-Apr-87	original implementation		JEB
 30-Aug-87	type coersion, deferred input	LPH
 16-Apr-88	new types, operators, etc	JEB
 16-May-92	random access to data   	PJT
 5-mar-94	documented qsf          	PJT
 2-jun-05	added blocked I/O		PJT
+2-jan-2024	fix 64bit problem for big items	PJT
 .fi

man/man5/filestruct.5

@@ -1,7 +1,9 @@
-.TH FILESTRUCT 5NEMO "16 May 1992"
-.SH NAME
+.TH FILESTRUCT 5NEMO "2 January 2024"
+
+.SH "NAME"
 filestruct \- binary structured file format 
-.SH SYNOPSIS
+
+.SH "SYNOPSIS"
 .nf
 \fB#include <stdinc.h>\fP
 \fB#include "filesecret.h"\fP        \fI only for local code\fP
@@ -19,23 +21,26 @@ filestruct \- binary structured file format
 .PP
 \fB struct item {\fP
 \fB    string itemtyp;\fP
-\fB    int    itemlen;\fP
+\fB    size_t itemlen;\fP
 \fB    string itemtag;\fP
 \fB    int   *itemdim;\fP
-\fB    byte  *itemdat;\fP
-\fB    long   itempos;\fP
+\fB    void  *itemdat;\fP
+\fB    off_t  itempos;\fP
+\fB    off_t  itemoff;\fP
 \fB };\fP
 .PP
 \fB struct strstk {\fP
 \fB    stream  ss_str;\fP
 \fB    item   *ss_stk[SetStkLen];\fP
 \fB    int     ss_stp;\fP
 \fB    bool    ss_seek;\fP
-\fB    long    ss_pos;\fP            \fI/* only if RANDOM access allowed */\fP
+\fB    int     ss_mode;\fP           \fI/* only if RANDOM access allowed */\fP
+\fB    off_t   ss_pos;\fP            \fI/* only if RANDOM access allowed */\fP
 \fB    itemptr ss_ran;\fP            \fI/* only if RANDOM access allowed */\fP
 \fB };\fP
 .fi
-.SH DESCRIPTION
+
+.SH "DESCRIPTION"
 \fIfilestruct\fP is a method for storing data files largely
 consisting of character strings, booleans, and possible
 multi-dimensional arrays of data.  This data may be structured in a
@@ -74,7 +79,8 @@ otherwise. This is essentially the reason why pipes cannot be used in
 The actual internal format is governed how the application programmer 
 uses the \fIget_XXX\fP and \fPput_XXX\fP routines (see 
 \fIfilestruct(3NEMO)\fP).
-.SH EXPERIMENTAL FEATURES
+
+.SH "EXPERIMENTAL FEATURES"
 If compiled with \fB-DRANDOM\fP 
 some limited random access to data within a data-item is possible.
 .PP
@@ -83,22 +89,27 @@ little/big endian machines. Otherwise,
 data on disk exist in the host format, and no effort has been made to make
 it machine independant (e.g. IEEE floating points and twos-compliment
 integers). This is however expected in some future release.
-.SH ZENO FORMAT
+
+.SH "ZENO FORMAT"
 The \fIzeno(1NEMO)\fP package also used this format, but there are some
-subtle differences to be described.
-.SH FILES
+subtle differences still to be described here.
+
+.SH "FILES"
 .nf
 .ta +2.0i
 ~/src/kernel/io   	filesecret.[ch] filestruct.h
 ~/inc              	filestruct.h
 .fi
-.SH SEE ALSO
+
+.SH "SEE ALSO"
 tsf(1NEMO), rsf(1NEMO), csf(1NEMO), filestruct(3NEMO), zeno(1NEMO)
-.SH AUTHOR
+
+.SH "AUTHOR"
 Joshua Barnes, Lyman Hurd, Peter Teuben
-.SH UPDATE HISTORY
+
+.SH "HISTORY"
 .nf
-.ta +1.0i +4.0i
+.ta +1.25i +4.5i
 dark-ages	V0.0 precurser (filestr)	JEB
 xx-apr-87	V1.0 basic operators     	JEB
 xx-jul-87	V1.x added f-d coercion, deferred output	Lyman Hurd
@@ -108,4 +119,5 @@ xx-xxx-87	V2.0 new types, operators, external fmt  	JEB
 16-may-92	V3.0 finalized random access                        	PJT
 6-jul -01	documented the new uNEMO   	PJT
 27-dec-2019	documented ZENO		PJT
-.fi
+2-jan-2024	V3.6 fix remaining 64bit issues by using off_t, size_t	PJT
+

src/image/fits/fits.c

@@ -151,7 +151,8 @@ size_t fts_rhead(fits_header *fh, stream instr)
 {
     char buf[FTSLINSIZ+2];                /* buffer to hold one card image */
     char a1[9], a2[2], a3[FTSLINSIZ+1],a4[FTSLINSIZ+1],a5[FTSLINSIZ+1];  /* args */
-    int k, n, i, icard;
+    int k, i, icard;
+    size_t n;
     struct arglist *p;
 
     if (fh->naxis >= 0) {         /* we could force a fts_zero here, but ala */
@@ -175,10 +176,9 @@ size_t fts_rhead(fits_header *fh, stream instr)
             n = FTSLINSIZ;
         } else {               /* this is of course the legal way to get cards */
           if ((n=fread(buf,1,FTSLINSIZ,instr)) != FTSLINSIZ) {     /* get card */
-            if (n==0)
-                return -1;         /* error - nothing read -- never -- */
-            else 
-                return -n-1;       /* return some negative number, but < -1 */
+            if (n != 0)
+	      warning("Incomplete read %ld bytes in header", n);
+	    return 0;
           } else {
             buf[FTSLINSIZ] = 0;      /* be sure it's terminated */
             dprintf(5,"RECORD[%d]%65s\n",icard,buf);
@@ -458,7 +458,7 @@ size_t fts_rhead(fits_header *fh, stream instr)
 
     /* process some additional checks */
     fts_lhead(fh);            /* process a few more sanity checks */
-    return fts_dsize(fh);     /* return the size of upcoming data */
+    return fts_dsize(fh);     /* return the size of upcoming data (could be 0 too) */
 }
 
 /*
@@ -1257,7 +1257,7 @@ int fts_phead(fits_header *fh, string *print)
     printf("headersize = %d bytes = %d %d-records\n",
             (int) fh->hlen, ((int) fh->hlen - 1)/ftsblksiz_i + 1, ftsblksiz_i);
     if (dsize > 0) 
-      printf("datasize = %ld bytes = %ld %d-records\n",
+      printf("datasize = %ld bytes = %ld %ld-records\n",
             dsize, (dsize-1)/ftsblksiz_i + 1, ftsblksiz_i);
     else
       printf("datasize = 0 bytes = 0 %d-records\n", ftsblksiz_i);
@@ -2038,11 +2038,14 @@ int fts_sdata(
 
     if (nskip % ftsblksiz_i) {
         ntail = ftsblksiz_i - nskip % ftsblksiz_i;
-        dprintf(1,"Skipping %ld bytes for empty tail\n",ntail);        
+        dprintf(1,"Skipping %ld bytes for empty tail in data\n",ntail);        
         nskip += ntail;
     }
     nskip -= fh->nread;
-    if (nskip>0) fseek(instr, nskip, 1);
+    if (nskip>0) {
+      dprintf(1,"fseek %ld from %ld\n", nskip, fh->nread);
+      fseek(instr, nskip, 1); // SEEK_CUR=1
+    }
     return 1;
 }
 
@@ -2058,7 +2061,7 @@ int fts_cdata(
 	      bool trailr,			/* (i) need to read trailing end too ? */
 	      bool trailw)			/* (i) need to write trailing end too ? */
 {
-    int  nread, nwrite, n, ntowrite, ntoread, itemlen, nitems;
+    size_t  n, nread, nwrite, ntowrite, ntoread, itemlen, nitems;
     char buffer[CONVBUFLEN];
 
     itemlen = ABS(fh->bitpix)/8;        /* # bytes in an item */
@@ -2073,25 +2076,25 @@ int fts_cdata(
       ntowrite = n;
     if (n==0) ntoread=0;                /* no data? */
     if (fh->flip) dprintf(1,"fts_cdata: Swapping bytes\n");
-    dprintf(2,"fts_data: ntoread=%d   ntowrite=%d\n",ntoread,ntowrite);
+    dprintf(2,"fts_data: ntoread=%ld   ntowrite=%ld\n",ntoread,ntowrite);
     while (ntoread > 0) {
         if (ntoread > CONVBUFLEN)
             n = CONVBUFLEN;
         else
             n = ntoread;
         nread = fread(buffer,sizeof(char),n,instr);
         if (nread != n) /* should be a warning in raw mode */
-            error("Tried to read %d, could only read %d",n,nread);
+            error("Tried to read %ld, could only read %ld",n,nread);
 	fh->nread += nread;
 	if (fh->flip) {
             nitems = nread/itemlen;
-            if (nread%itemlen) error("Bad bufsize %d for byteswap",nread);
+            if (nread%itemlen) error("Bad bufsize %ld for byteswap",nread);
             bswap(buffer,itemlen, nitems);
         }
 /****FIX****/
         nwrite = fwrite(buffer,sizeof(char),n,outstr);
         if (nwrite != n) 
-            error("Tried to write %d, could only write %d\n",n,nwrite);
+            error("Tried to write %ld, could only write %ld\n",n,nwrite);
         ntoread -= nread;
     }
     return 1;
@@ -2146,9 +2149,10 @@ int fts_cdata816(
  *             See also eq. (x.y.z) in the NOST manual
  */
 
-int fts_dsize(fits_header *fh)
+size_t fts_dsize(fits_header *fh)
 {
-    int i, size;
+    int i;
+    size_t size = 0;
 
     if (fh->naxis > 0) {
         if (fh->naxisn[0] == 0) {       /* NAXIS1=0 is for random groups */
@@ -2164,17 +2168,18 @@ int fts_dsize(fits_header *fh)
         size *= fh->gcount * ABS(fh->bitpix) / 8;
     } else
         size = 0;
+    dprintf(1,"fts_dsize:  %ld\n", size);    
     return size;
 }
 
 /*
  *  fts_tsize: return trailing size of useable data portion
  */
 
-int fts_tsize(fits_header *fh)
+size_t fts_tsize(fits_header *fh)
 {
-  int n = fts_dsize(fh);
-  int tail = ROUNDUP(n,ftsblksiz_o);
+  size_t n = fts_dsize(fh);
+  size_t tail = ROUNDUP(n,ftsblksiz_o);
   return tail - n;
 }
 

src/image/fits/scanfits.c

@@ -54,7 +54,7 @@ string defv[] = {			/* Standard NEMO keyword+help */
     "blocking=1,1\n  	   Two blocking factors (blocksize/2880) for i&o",
     "select=header,data\n  Select header, data, ...?",
     "split=f\n             Split fits file into HDU pieces '<out>.#'",
-    "VERSION=1.8d\n        13-jul-2021 PJT",
+    "VERSION=1.8e\n        3-jan-2024 PJT",
     NULL,
 };
 
@@ -67,7 +67,8 @@ extern string *burststring(string, string);
 void nemo_main()
 {
     stream instr, outstr;
-    int    i, n, nfile, blocking[2];
+    int    i,n,nfile, blocking[2];
+    size_t size;
     string outfile, hdselect, *insert, *fix, *deletes, *keep, *print;
     char   basename[128];
     struct fits_header fh;
@@ -115,15 +116,17 @@ void nemo_main()
        fts_zero(&fh);			             /* reset header */
        fh.hdu = i;                                   /* keep track of HDU (1=first) */
 
-       n = fts_rhead(&fh,instr);	              /* read header */
-       if (n<0)				              /* if no data (EOF) .. */
+       size = fts_rhead(&fh,instr);	              /* read header */
+       if (size == 0 ) {        		      /* if no data (EOF) .. */
+	  dprintf(1," no more data\n");
           break;			              /* ... quit */
+       }
        if (outstr) dprintf(1,"Working on FITS file %d\n",i);
-       fts_dhead(&fh,deletes);                     /* delete= headers */
-       fts_khead(&fh,keep);                        /* keep= headers */
-       fts_ihead(&fh,insert);                      /* insert= headers */
-       fts_fhead(&fh,fix);                         /* fix= headers */
-       if (!outstr && !split) fts_phead(&fh,print);  /* ? print header */
+       fts_dhead(&fh,deletes);                        /* delete= headers */
+       fts_khead(&fh,keep);                           /* keep= headers */
+       fts_ihead(&fh,insert);                         /* insert= headers */
+       fts_fhead(&fh,fix);                            /* fix= headers */
+       if (!outstr && !split) fts_phead(&fh,print);   /* ? print header */
        if ((outstr && (nfile==0 || nfile==i)) || split) {
           if (split) {
           	if (strchr(outfile,'%'))

src/image/io/image.c

@@ -33,7 +33,8 @@
  *  13-feb-13   V8.1 added region and sub_image()                        PJT
  *  22-may-21   V8.2 deal with Object
  *  19-mar-22   V8.3 deprecate Axis=0 images
- *  17-dec-22        deal with Telescope/Object/Unit 
+ *  17-dec-22        deal with Telescope/Object/Unit
+ *   2-jan-24        fix alloc large cubes
  *			
  *
  *	  Example of usage: see snapccd.c	for writing
@@ -319,7 +320,9 @@ int read_image (stream instr, imageptr *iptr)
 
          get_set (instr,MapTag);
             if (Frame(*iptr)==NULL) {        /* check if allocated */
-	        nxyz = Nx(*iptr)*Ny(*iptr)*Nz(*iptr);
+	        nxyz  = Nx(*iptr);
+		nxyz *= Ny(*iptr);
+		nxyz *= Nz(*iptr);
                 Frame(*iptr) = (real *) allocate(nxyz * sizeof(real));
                 dprintf (DLEV,"Frame allocated @ %p ",Frame(*iptr));
             } else

src/kernel/io/dprintf.c

@@ -33,7 +33,7 @@
 
 int debug_level=0;	/* needs to be global; see also getparam.c */
 
-static char *nemo_file = "dprintf.c: debugging stuff";
+// static char *nemo_file = "dprintf.c: debugging stuff";
 
 bool nemo_debug(int debug) 
 {