From 6f43a91bb639cdf102bfc72db036fe6af8de55fc Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Thu, 2 Jan 2025 13:09:47 -0500 Subject: [PATCH 1/8] converted all relevant int/long to off_t and size_t code now does not run a simple "tsf" - failure in copydata() --- src/kernel/io/filesecret.c | 115 +++++++++++++++++++------------------ src/kernel/io/filesecret.h | 24 ++++---- 2 files changed, 72 insertions(+), 67 deletions(-) diff --git a/src/kernel/io/filesecret.c b/src/kernel/io/filesecret.c index 2e23d7eff..d939bd205 100644 --- a/src/kernel/io/filesecret.c +++ b/src/kernel/io/filesecret.c @@ -42,7 +42,8 @@ * V 3.3 25-may-07 pjt handle > 2GB objects in memory (Pierre Fortin ) * V 3.4 12-dec-09 pjt support the new halfp type for I/O (see also csf) * 27-Sep-10 jcl MINGW32/WINDOWS support - * 3.5 8-jun-13 pjt eltcnt type fixed for 64bit so it handles > 2B + * 3.5 8-jun-13 pjt eltcnt type fixed for 64bit so it handles > 2GB + * 3.6 2-jan-24 pjt subtle 64bit; now using off_t and size_t * * Although the SWAP test is done on input for every item - for deferred * input it may fail if in the mean time another file was read which was @@ -92,6 +93,7 @@ void copy_item(stream ostr, stream istr, string tag) dims = get_dims(istr, tag); /* find out about shape */ dlen = get_dlen(istr, tag); /* and length in bytes */ if(dlen<0) error("copy_item: %s with dlen=%d",tag,dlen); /* yuck */ + dprintf(0,"copy_item: %ld\n",dlen); buf = (byte *) calloc(dlen,1); /* get space for a buffer */ if (buf == NULL) /* and check for error */ error("copy_item: item %s: not enuf memory", tag); @@ -137,7 +139,8 @@ void copy_item_cvt(stream ostr, stream istr, string tag, string *cvt) if (! streq(type, SetType)) { /* a basic type or array? */ dims = get_dims(istr, tag); /* find out about shape */ dlen = get_dlen(istr, tag); /* and length in bytes */ - if(dlen<0) error("copy_item_cvt: %s with dlen=%d",tag,dlen); + if(dlen<0) error("copy_item_cvt: %s with dlen=%d",tag,dlen); + dprintf(0,"copy_item_cvt: %ld\n",dlen); bufin = (byte *) calloc(dlen,1); /* get space for a buffer */ if (bufin == NULL) /* and check for error */ error("copy_item_cvt: item %s: not enuf memory", tag); @@ -727,14 +730,14 @@ string get_string( dp == NULL || *dp++ == 0 || *dp != 0) /* and shape of data */ error("get_string: item %s: not plural char", tag); dlen = datlen(ipt,0); - if(dlen<0) error("get_string: %s with dlen=%d",tag,dlen); /* yuck */ + if(dlen<0) error("get_string: %s with dlen=%ld",tag,dlen); /* yuck */ dat = (char *) calloc(dlen,1); /* alloc memory for string */ if (dat == NULL) /* did alloc fail? */ error("get_string: item %s: not enuf memory", tag); copydata(dat, 0, dlen, ipt, str); /* copy string from input */ if (sspt->ss_stp == -1) /* item read at top level? */ freeitem(ipt, TRUE); /* yes, so free it up */ - return (dat); /* return string as value */ + return dat; /* return string as value */ } /* @@ -752,10 +755,10 @@ bool get_tag_ok( sspt = findstream(str); /* lookup associated entry */ if (sspt->ss_stp == -1) { /* input from top level? */ ipt = nextitem(sspt); /* look at next item */ - return (ipt != NULL && streq(tag, ItemTag(ipt))); + return ipt != NULL && streq(tag, ItemTag(ipt)); /* test existance and tag */ } else /* input within a set? */ - return (finditem(sspt, tag) != NULL); /* test success of scan */ + return finditem(sspt, tag) != NULL; /* test success of scan */ } /* @@ -773,7 +776,7 @@ stream str if (sspt->ss_stp == -1) { /* input from top level? */ ipt = nextitem(sspt); /* get next item read in */ if (ipt == NULL) /* nothing left in input? */ - return (NULL); /* then return nothing */ + return NULL; /* then return nothing */ *tpt++ = (string) copxstr(ItemTag(ipt), sizeof(char)); /* make copy of item tag */ } else { /* input within a set? */ @@ -784,7 +787,7 @@ stream str /* make copy of item tag */ } *tpt = NULL; /* terminate string of tags */ - return ((string *) copxstr(tags, sizeof(string))); + return (string *) copxstr(tags, sizeof(string)); /* return copy of copies */ } @@ -806,7 +809,7 @@ string get_type( error("get_type: at EOF"); if (sspt->ss_stp == -1) /* was input at top level? */ sspt->ss_stk[0] = ipt; /* put back for next time */ - return ((string) copxstr(ItemTyp(ipt), sizeof(char))); + return (string) copxstr(ItemTyp(ipt), sizeof(char)); /* return copy of type */ } @@ -824,7 +827,7 @@ int *get_dims( if (sspt->ss_stp == -1) /* was input at top level? */ sspt->ss_stk[0] = ipt; /* put back for next time */ if (ItemDim(ipt) != NULL) /* any dimensions to item? */ - return ((int *) copxstr(ItemDim(ipt), sizeof(int))); + return (int *) copxstr(ItemDim(ipt), sizeof(int)); /* return copy of dims */ else return NULL; @@ -884,19 +887,19 @@ local bool writeitem(stream str, itemptr ipt) itemptr *setp, tesp; if (! streq(ItemTyp(ipt), SetType)) /* just a simple item? */ - return (putitem(str, ipt)); /* then write it out */ + return putitem(str, ipt); /* then write it out */ else { /* recursively handle set */ if (! putitem(str, ipt)) /* write out set item */ - return (FALSE); + return FALSE; setp = (itemptr*) ItemDat(ipt); /* point to item string */ while (*setp != NULL) /* loop over the items */ writeitem(str, *setp++); /* write each one out */ tesp = makeitem(TesType, NULL, NULL, NULL); /* create closing item */ if (! putitem(str, tesp)) /* write out tes item */ - return (FALSE); + return FALSE; freeitem(tesp, FALSE); /* and reclaim memory */ - return (TRUE); /* indicate success */ + return TRUE; /* indicate success */ } } @@ -907,12 +910,12 @@ local bool writeitem(stream str, itemptr ipt) local bool putitem(stream str, itemptr ipt) { if (! puthdr(str, ipt)) /* write item header */ - return (FALSE); + return FALSE; if (! streq(ItemTyp(ipt), SetType) && ! streq(ItemTyp(ipt), TesType)) /* an ordinary data item? */ if (! putdat(str, ipt)) /* write item data */ - return (FALSE); - return (TRUE); /* indicate success */ + return FALSE; + return TRUE ; /* indicate success */ } /* @@ -927,22 +930,22 @@ local bool puthdr(stream str, itemptr ipt) num = (ItemDim(ipt) == NULL) ? SingMagic : PlurMagic; /* determine magic number */ if (fwrite((char *)&num, sizeof(short), 1, str) != 1) - return (FALSE); /* return FALSE on failure */ + return FALSE; /* return FALSE on failure */ if (! putxstr(str, ItemTyp(ipt), sizeof(char))) - return (FALSE); + return FALSE; if (ItemTag(ipt) != NULL) { /* is item tagged? */ if (xstrlen(ItemTag(ipt), sizeof(char)) > MaxTagLen) error("puthdr: tag too long"); if (! putxstr(str, ItemTag(ipt), sizeof(char))) - return (FALSE); /* write item tag */ + return FALSE; /* write item tag */ } if (ItemDim(ipt) != NULL) { /* a vectorized item? */ if (xstrlen(ItemDim(ipt), sizeof(int)) > MaxVecDim) error("puthdr: too many dimensions"); if (! putxstr(str, ItemDim(ipt), sizeof(int))) - return (FALSE); /* write vect dims */ + return FALSE; /* write vect dims */ } - return(TRUE); /* indicate success */ + return TRUE; /* indicate success */ } /* @@ -956,7 +959,7 @@ local bool putdat(stream str, itemptr ipt) if (ItemDat(ipt) == NULL) /* no data to write? */ error("putdat: item %s has no data", ItemTag(ipt)); len = datlen(ipt, 0); /* count bytes to output */ - return (fwrite((char*)ItemDat(ipt), sizeof(byte), len, str) == len); + return fwrite((char*)ItemDat(ipt), sizeof(byte), len, str) == len; /* write data to stream */ } @@ -986,7 +989,7 @@ local itemptr scantag(strstkptr sspt, string tag) error("scantag: item %s not found in set %s", tag, ItemTag(sspt->ss_stk[sspt->ss_stp])); } - return (ipt); /* return item found */ + return ipt; /* return item found */ } local itemptr nextitem(strstkptr sspt) @@ -999,7 +1002,7 @@ local itemptr nextitem(strstkptr sspt) ipt = readitem(sspt->ss_str, NULL); /* read next item in */ sspt->ss_stk[0] = ipt; /* and save for later */ } - return (ipt); /* supply item to caller */ + return ipt; /* supply item to caller */ } local itemptr finditem(strstkptr sspt, string tag) @@ -1013,7 +1016,7 @@ local itemptr finditem(strstkptr sspt, string tag) break; /* done with loop */ ivec++; /* on to next item */ } - return (*ivec); /* return item or NULL */ + return *ivec; /* return item or NULL */ } /* @@ -1027,7 +1030,7 @@ local itemptr readitem(stream str, itemptr first) ip = first != NULL ? first : getitem(str); /* use 1st or next item */ if (ip == NULL || /* EOF detected by getitem */ ! streq(ItemTyp(ip), SetType)) /* or item not a set? */ - return (ip); /* just return it */ + return ip; /* just return it */ bufp = &ibuf[0]; /* prepare item buffer */ for ( ; ; ) { /* loop reading items in */ if (bufp >= &ibuf[MaxSetLen]) /* no room for next? */ @@ -1045,7 +1048,7 @@ local itemptr readitem(stream str, itemptr first) /* construct compound item */ freeitem(ip, TRUE); /* reclaim orig. header */ freeitem(np, TRUE); - return (res); /* return compound item */ + return res; /* return compound item */ } /* @@ -1058,11 +1061,11 @@ local itemptr getitem(stream str) ipt = gethdr(str); /* try reading header in */ if (ipt == NULL) /* did gethdr detect EOF? */ - return (NULL); /* return NULL on EOF */ + return NULL; /* return NULL on EOF */ if (! streq(ItemTyp(ipt), SetType) && /* if item does not start */ ! streq(ItemTyp(ipt), TesType)) /* or terminate a set */ getdat(ipt, str); /* try reading data in */ - return (ipt); /* return resulting item */ + return ipt; /* return resulting item */ } /* @@ -1072,7 +1075,7 @@ local itemptr getitem(stream str) local itemptr gethdr(stream str) { short num; - string typ, tag; + string tag, typ = NULL; int *dim, *ip; /* ISSWAP */ permanent bool firsttime = TRUE; @@ -1128,7 +1131,7 @@ local itemptr gethdr(stream str) #endif } else dim = NULL; - return (makeitem(typ, tag, NULL, dim)); /* return item less data */ + return makeitem(typ, tag, NULL, dim); /* return item less data */ } /* gethdr */ /* * GETHDR: read a item header from a stream. @@ -1213,18 +1216,21 @@ local copyproc copyfun(string srctyp, string destyp) local void copydata( void *vdat, - int off, - int len, + off_t off, + size_t len, itemptr ipt, stream str) { char *src, *dat = (char *) vdat; off_t oldpos; + + dprintf(0,"COPYDATA: off=%ld len=%ld\n", off, len); off *= ItemLen(ipt); /* offset bytes from start */ if (ItemDat(ipt) != NULL) { /* data already in core? */ src = (char *) ItemDat(ipt) + off; /* get pointer to source */ len *= ItemLen(ipt); /* number of bytes to copy */ + dprintf(0,"COPYDATA2: len=%ld src=0x%x dat=0x%x\n",len, src, dat); while (--len >= 0) /* loop copying data */ *dat++ = *src++; /* byte by byte */ } else { /* time to read data in */ @@ -1237,8 +1243,8 @@ local void copydata( local void copydata_f2d( double *dat, - int off, - int len, + off_t off, + size_t len, itemptr ipt, stream str) { @@ -1262,8 +1268,8 @@ local void copydata_f2d( local void copydata_d2f( float *dat, - int off, - int len, + off_t off, + size_t len, itemptr ipt, stream str) { @@ -1303,8 +1309,8 @@ local double getdbl(stream str) local void saferead( void *dat, - int siz, - int cnt, + size_t siz, + size_t cnt, stream str) { if (fread(dat, siz, cnt, str) != cnt) @@ -1330,20 +1336,19 @@ local void safeseek( /* * ELTCNT: compute number of basic elements in subspace of item. - * @todo: long vs. size_t */ -local long eltcnt( +local size_t eltcnt( itemptr ipt, /* pointer to item w/ possible vector dims */ - int skp) /* num of dims to skip, starting with dimN */ + int dimskp) /* num of dims to skip, starting with dimN */ { register int *ip; - register long prod; + register size_t prod; prod = 1; /* scalers have one */ if (ItemDim(ipt) != NULL) { /* a vectorized item? */ for (ip = ItemDim(ipt); *ip != 0; ip++) /* loop over dimensions */ - if (--skp < 0) /* past 1st skp dims? */ + if (--dimskp < 0) /* past 1st skp dims? */ prod *= *ip; /* include this dim */ } return prod; /* return product of dims */ @@ -1355,9 +1360,9 @@ local long eltcnt( local size_t datlen( itemptr ipt, /* pointer to item w/ possible vector dims */ - int skp) /* num of dims to skip, starting with dimN */ + int dimskp) /* num of dims to skip, starting with dimN */ { - return (ItemLen(ipt) * eltcnt(ipt, skp)); /* find byte-len needed */ + return ItemLen(ipt) * eltcnt(ipt, dimskp); /* find byte-len needed */ } /* @@ -1384,7 +1389,7 @@ local itemptr makeitem( ItemDim(ipt) = NULL; /* clear out dimensions */ ItemDat(ipt) = dat; /* set pointer to data */ ItemPos(ipt) = 0; /* clear out file position */ - return (ipt); /* return complete item */ + return ipt; /* return complete item */ } /* @@ -1433,15 +1438,15 @@ local typlen tl_tab[] = { { NULL, 0, }, }; -local int baselen(string typ) +local size_t baselen(string typ) { typlenptr tp; for (tp = tl_tab; tp->tl_typ != NULL; tp++) /* loop over basic types */ - if (streq(typ, tp->tl_typ)) /* found type we want? */ - return (tp->tl_len); /* return byte length */ + if (streq(typ, tp->tl_typ)) /* found type we want? */ + return tp->tl_len; /* return byte length */ error("baselen: type %s unknown", typ); /* bad type string */ - return (0); /* will never get here */ + return 0; /* will never get here */ } /************************************************************************/ @@ -1488,13 +1493,13 @@ local strstkptr findstream(stream str) strstkptr stfree, sspt; if (last!=NULL && last->ss_str == str) - return(last); + return last; stfree = NULL; /* remember free slot */ for (sspt = strtable; sspt < strtable+StrTabLen; sspt++) { /* loop over the table */ if (sspt->ss_str == str) { /* found that stream? */ last = sspt; - return (sspt); /* then return slot */ + return sspt; /* then return slot */ } if (stfree == NULL && sspt->ss_str == NULL) /* first empty slot? */ @@ -1512,7 +1517,7 @@ local strstkptr findstream(stream str) stfree->ss_pos = 0L; /* set at start of file */ #endif last = stfree; /* mark for quick access */ - return (stfree); /* return new slot */ + return stfree; /* return new slot */ } local void ss_push(strstkptr sspt, itemptr ipt) diff --git a/src/kernel/io/filesecret.h b/src/kernel/io/filesecret.h index 578f9ce71..07b53d108 100644 --- a/src/kernel/io/filesecret.h +++ b/src/kernel/io/filesecret.h @@ -32,7 +32,7 @@ typedef struct { string itemtyp; /* type string, listed in filestruct.h */ - long itemlen; /* length associated with above type */ + size_t itemlen; /* length associated with above type */ string itemtag; /* name given this item by application */ int *itemdim; /* int-string of dimensions, or NULL */ void *itemdat; /* the real goodies, if any, or NULL */ @@ -74,16 +74,16 @@ typedef struct { typedef struct { string tl_typ; /* type string (see filestruct.h) */ - int tl_len; /* length of elements in bytes */ + size_t tl_len; /* length of elements in bytes */ } typlen, *typlenptr; /* PROC_COPY's : * replaces the old "proc" type unsafe stuff (for * good practice for C, but needed for C++) */ -typedef void (*copyproc) (void *, int, int, itemptr, stream); -typedef void (*copyproc_d)(double *, int, int, itemptr, stream); -typedef void (*copyproc_f)(float *, int, int, itemptr, stream); +typedef void (*copyproc) (void *, off_t, size_t, itemptr, stream); +typedef void (*copyproc_d)(double *, off_t, size_t, itemptr, stream); +typedef void (*copyproc_f)(float *, off_t, size_t, itemptr, stream); /* @@ -104,18 +104,18 @@ local itemptr getitem ( stream str ); local itemptr gethdr ( stream str ); local void getdat ( itemptr ipt, stream str ); local copyproc copyfun ( string srctyp, string destyp ); -local void copydata ( void *dat, int off, int len, itemptr ipt, stream str ); -local void copydata_f2d( double *dat, int off, int len, itemptr ipt, stream str ); -local void copydata_d2f( float *dat, int off, int len, itemptr ipt, stream str ); +local void copydata ( void *dat, off_t off, size_t len, itemptr ipt, stream str ); +local void copydata_f2d( double *dat, off_t off, size_t len, itemptr ipt, stream str ); +local void copydata_d2f( float *dat, off_t off, size_t len, itemptr ipt, stream str ); local float getflt ( stream str ); local double getdbl ( stream str ); -local void saferead ( void *dat, int siz, int cnt, stream str ); +local void saferead ( void *dat, size_t siz, size_t cnt, stream str ); local void safeseek ( stream str, off_t offset, int key ); -local long eltcnt ( itemptr ipt, int skp ); -local size_t datlen ( itemptr ipt, int skp ); +local size_t eltcnt ( itemptr ipt, int dimskp ); +local size_t datlen ( itemptr ipt, int dimskp ); local itemptr makeitem ( string typ, string tag, void *dat, int *dim ); local void freeitem ( itemptr ipt, bool flg); -local int baselen ( string typ ); +local size_t baselen ( string typ ); local strstkptr findstream ( stream str ); local void ss_push ( strstkptr sspt, itemptr ipt ); local void ss_pop ( strstkptr sspt ); From 3fe34ea027abc22c8f51685bc57bd67e1b440a04 Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Thu, 2 Jan 2025 13:11:21 -0500 Subject: [PATCH 2/8] hide unused variable to appease compiler --- src/kernel/io/dprintf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/io/dprintf.c b/src/kernel/io/dprintf.c index d955318b4..21b60060d 100644 --- a/src/kernel/io/dprintf.c +++ b/src/kernel/io/dprintf.c @@ -33,7 +33,7 @@ int debug_level=0; /* needs to be global; see also getparam.c */ -static char *nemo_file = "dprintf.c: debugging stuff"; +// static char *nemo_file = "dprintf.c: debugging stuff"; bool nemo_debug(int debug) { From cd055d95052ff86ec3c934d1ffeaed1e68304f1d Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Thu, 2 Jan 2025 13:39:57 -0500 Subject: [PATCH 3/8] change while() to a for() loop and copydata now works... go figure --- src/kernel/io/filesecret.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/kernel/io/filesecret.c b/src/kernel/io/filesecret.c index d939bd205..19c4d94e6 100644 --- a/src/kernel/io/filesecret.c +++ b/src/kernel/io/filesecret.c @@ -1224,15 +1224,21 @@ local void copydata( char *src, *dat = (char *) vdat; off_t oldpos; - dprintf(0,"COPYDATA: off=%ld len=%ld\n", off, len); - off *= ItemLen(ipt); /* offset bytes from start */ if (ItemDat(ipt) != NULL) { /* data already in core? */ src = (char *) ItemDat(ipt) + off; /* get pointer to source */ len *= ItemLen(ipt); /* number of bytes to copy */ - dprintf(0,"COPYDATA2: len=%ld src=0x%x dat=0x%x\n",len, src, dat); - while (--len >= 0) /* loop copying data */ + // dprintf(0,"COPYDATA2: len=%ld src=0x%x dat=0x%x\n",len, src, dat); +#if 0 + // this while loop will hit through 0, and eventually segfault in the copy + // compiler bug ? + while (--len >= 0) { /* loop copying data */ *dat++ = *src++; /* byte by byte */ + } +#else + for (size_t i=0; i Date: Thu, 2 Jan 2025 14:05:07 -0500 Subject: [PATCH 4/8] document the final 64bit fixes --- man/man3/filestruct.3 | 5 +++-- man/man5/filestruct.5 | 46 +++++++++++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/man/man3/filestruct.3 b/man/man3/filestruct.3 index bbe1402b7..4f408f1b6 100644 --- a/man/man3/filestruct.3 +++ b/man/man3/filestruct.3 @@ -1,4 +1,4 @@ -.TH FILESTRUCT 3NEMO "7 April 2012" +.TH FILESTRUCT 3NEMO "2 January 2024" .SH "NAME" filestruct \- primitives for structured binary file I/O @@ -274,11 +274,12 @@ https://github.com/syrte/snapio - python implementation of snap I/O .SH "UPDATE HISTORY" .nf -.ta +2.0i +2.0i +.ta +1.25i +4.5i 4-Apr-87 original implementation JEB 30-Aug-87 type coersion, deferred input LPH 16-Apr-88 new types, operators, etc JEB 16-May-92 random access to data PJT 5-mar-94 documented qsf PJT 2-jun-05 added blocked I/O PJT +2-jan-2024 fix 64bit problem for big items PJT .fi diff --git a/man/man5/filestruct.5 b/man/man5/filestruct.5 index fc4d9c20d..f1bb02976 100644 --- a/man/man5/filestruct.5 +++ b/man/man5/filestruct.5 @@ -1,7 +1,9 @@ -.TH FILESTRUCT 5NEMO "16 May 1992" -.SH NAME +.TH FILESTRUCT 5NEMO "2 January 2024" + +.SH "NAME" filestruct \- binary structured file format -.SH SYNOPSIS + +.SH "SYNOPSIS" .nf \fB#include \fP \fB#include "filesecret.h"\fP \fI only for local code\fP @@ -19,11 +21,12 @@ filestruct \- binary structured file format .PP \fB struct item {\fP \fB string itemtyp;\fP -\fB int itemlen;\fP +\fB size_t itemlen;\fP \fB string itemtag;\fP \fB int *itemdim;\fP -\fB byte *itemdat;\fP -\fB long itempos;\fP +\fB void *itemdat;\fP +\fB off_t itempos;\fP +\fB off_t itemoff;\fP \fB };\fP .PP \fB struct strstk {\fP @@ -31,11 +34,13 @@ filestruct \- binary structured file format \fB item *ss_stk[SetStkLen];\fP \fB int ss_stp;\fP \fB bool ss_seek;\fP -\fB long ss_pos;\fP \fI/* only if RANDOM access allowed */\fP +\fB int ss_mode;\fP \fI/* only if RANDOM access allowed */\fP +\fB off_t ss_pos;\fP \fI/* only if RANDOM access allowed */\fP \fB itemptr ss_ran;\fP \fI/* only if RANDOM access allowed */\fP \fB };\fP .fi -.SH DESCRIPTION + +.SH "DESCRIPTION" \fIfilestruct\fP is a method for storing data files largely consisting of character strings, booleans, and possible multi-dimensional arrays of data. This data may be structured in a @@ -74,7 +79,8 @@ otherwise. This is essentially the reason why pipes cannot be used in The actual internal format is governed how the application programmer uses the \fIget_XXX\fP and \fPput_XXX\fP routines (see \fIfilestruct(3NEMO)\fP). -.SH EXPERIMENTAL FEATURES + +.SH "EXPERIMENTAL FEATURES" If compiled with \fB-DRANDOM\fP some limited random access to data within a data-item is possible. .PP @@ -83,22 +89,27 @@ little/big endian machines. Otherwise, data on disk exist in the host format, and no effort has been made to make it machine independant (e.g. IEEE floating points and twos-compliment integers). This is however expected in some future release. -.SH ZENO FORMAT + +.SH "ZENO FORMAT" The \fIzeno(1NEMO)\fP package also used this format, but there are some -subtle differences to be described. -.SH FILES +subtle differences still to be described here. + +.SH "FILES" .nf .ta +2.0i ~/src/kernel/io filesecret.[ch] filestruct.h ~/inc filestruct.h .fi -.SH SEE ALSO + +.SH "SEE ALSO" tsf(1NEMO), rsf(1NEMO), csf(1NEMO), filestruct(3NEMO), zeno(1NEMO) -.SH AUTHOR + +.SH "AUTHOR" Joshua Barnes, Lyman Hurd, Peter Teuben -.SH UPDATE HISTORY + +.SH "HISTORY" .nf -.ta +1.0i +4.0i +.ta +1.25i +4.5i dark-ages V0.0 precurser (filestr) JEB xx-apr-87 V1.0 basic operators JEB xx-jul-87 V1.x added f-d coercion, deferred output Lyman Hurd @@ -108,4 +119,5 @@ xx-xxx-87 V2.0 new types, operators, external fmt JEB 16-may-92 V3.0 finalized random access PJT 6-jul -01 documented the new uNEMO PJT 27-dec-2019 documented ZENO PJT -.fi +2-jan-2024 V3.6 fix remaining 64bit issues by using off_t, size_t PJT + From dac0f33e771ddcd39223d1b2fd2ad2a0c52b97a1 Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Thu, 2 Jan 2025 17:27:24 -0500 Subject: [PATCH 5/8] also convert the other while() loops to a for() loop but now mkhomsph fails the bsf test, all others fine. --- src/kernel/io/filesecret.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/kernel/io/filesecret.c b/src/kernel/io/filesecret.c index 19c4d94e6..616b74314 100644 --- a/src/kernel/io/filesecret.c +++ b/src/kernel/io/filesecret.c @@ -43,7 +43,7 @@ * V 3.4 12-dec-09 pjt support the new halfp type for I/O (see also csf) * 27-Sep-10 jcl MINGW32/WINDOWS support * 3.5 8-jun-13 pjt eltcnt type fixed for 64bit so it handles > 2GB - * 3.6 2-jan-24 pjt subtle 64bit; now using off_t and size_t + * 3.6 2-jan-24 pjt subtle fix for items > 64bit; now using off_t and size_t * * Although the SWAP test is done on input for every item - for deferred * input it may fail if in the mean time another file was read which was @@ -140,7 +140,8 @@ void copy_item_cvt(stream ostr, stream istr, string tag, string *cvt) dims = get_dims(istr, tag); /* find out about shape */ dlen = get_dlen(istr, tag); /* and length in bytes */ if(dlen<0) error("copy_item_cvt: %s with dlen=%d",tag,dlen); - dprintf(0,"copy_item_cvt: %ld\n",dlen); + //PJT + dprintf(1,"copy_item_cvt: %ld\n",dlen); bufin = (byte *) calloc(dlen,1); /* get space for a buffer */ if (bufin == NULL) /* and check for error */ error("copy_item_cvt: item %s: not enuf memory", tag); @@ -1228,10 +1229,10 @@ local void copydata( if (ItemDat(ipt) != NULL) { /* data already in core? */ src = (char *) ItemDat(ipt) + off; /* get pointer to source */ len *= ItemLen(ipt); /* number of bytes to copy */ - // dprintf(0,"COPYDATA2: len=%ld src=0x%x dat=0x%x\n",len, src, dat); #if 0 // this while loop will hit through 0, and eventually segfault in the copy // compiler bug ? + // this pattern happened a few times below, where they've all been converted to the for() loop while (--len >= 0) { /* loop copying data */ *dat++ = *src++; /* byte by byte */ } @@ -1261,13 +1262,13 @@ local void copydata_f2d( if (ItemDat(ipt) != NULL) { /* data already in core? */ src = (float *) ItemDat(ipt) + off; /* get pointer to source */ /* len *= ItemLen(ipt); ==> BUG */ - while (--len >= 0) /* loop converting data */ + for (size_t i=0; i= 0) /* loop reading data */ - *dat++ = (double) getflt(str); /* float to double */ + for (size_t i=0; i= 0) /* loop converting data */ - *dat++ = (float) *src++; /* double to float */ + for (size_t i=0; i= 0) /* loop reading data */ - *dat++ = (float) getdbl(str); /* double to float */ + for (size_t i=0; i Date: Thu, 2 Jan 2025 20:34:10 -0500 Subject: [PATCH 6/8] clean code so compiler is happy --- src/nbody/init/mkhomsph.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/nbody/init/mkhomsph.c b/src/nbody/init/mkhomsph.c index f7b71e4ee..eb308ebc9 100644 --- a/src/nbody/init/mkhomsph.c +++ b/src/nbody/init/mkhomsph.c @@ -26,7 +26,7 @@ #include #include -string defv[] = { /* DEFAULT INPUT PARAMETERS */ +string defv[] = { "out=???\n Output file name", "nbody=2048\n Number of particles", "rmin=0\n Inner cutoff radius", @@ -37,14 +37,12 @@ string defv[] = { /* DEFAULT INPUT PARAMETERS */ "seed=0\n Random number seed", "zerocm=t\n Center c.o.m. ?", "headline=\n Text headline for output", - "VERSION=1.4c\n 29-aug-2018 PJT", + "VERSION=1.4d\n 2-jan-2024 PJT", NULL, }; string usage = "create a uniform sphere of equal massive stars"; -string cvsid="$Id$"; - local real rmin, rmax; local real virial, power, vmax; @@ -76,6 +74,7 @@ void nemo_main(void) power = getdparam("power"); if (power >= 3.0) error("Illegal power=%g, must be < 3.0",power); seed = init_xrandom(getparam("seed")); + dprintf(0,"seed=%d\n",seed); zerocm = getbparam("zerocm"); mksphere(); writegalaxy(getparam("out"), getparam("headline")); @@ -107,7 +106,6 @@ void mksphere(void) { Body *bp; real rmin3, rmax3, r_i, v_i, theta_i, phi_i, mass_i, sigma = 0.0; - real sinp, cosp, sint, cost; int i; btab = (Body *) allocate(nbody * sizeof(Body)); @@ -126,7 +124,8 @@ void mksphere(void) dprintf(1,"Gaussian isotropic velocities: sigma=%g\n",sigma); } else if (vmax > 0.0) { dprintf(1,"Vmax isotropic velocities: vmax=%g\n",vmax); - } + } + // @todo use local sin/cos variables to make it more efficient for (bp=btab, i = 0; i < nbody; bp++, i++) { Mass(bp) = mass_i; #ifdef OLD From e629826da8caf024a60725a0eaf09e149e0634ae Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Thu, 2 Jan 2025 21:20:55 -0500 Subject: [PATCH 7/8] fix allocation large cubes - missed a proper size_t cast --- src/image/io/image.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/image/io/image.c b/src/image/io/image.c index a13984b79..6a2259975 100644 --- a/src/image/io/image.c +++ b/src/image/io/image.c @@ -33,7 +33,8 @@ * 13-feb-13 V8.1 added region and sub_image() PJT * 22-may-21 V8.2 deal with Object * 19-mar-22 V8.3 deprecate Axis=0 images - * 17-dec-22 deal with Telescope/Object/Unit + * 17-dec-22 deal with Telescope/Object/Unit + * 2-jan-24 fix alloc large cubes * * * Example of usage: see snapccd.c for writing @@ -319,7 +320,9 @@ int read_image (stream instr, imageptr *iptr) get_set (instr,MapTag); if (Frame(*iptr)==NULL) { /* check if allocated */ - nxyz = Nx(*iptr)*Ny(*iptr)*Nz(*iptr); + nxyz = Nx(*iptr); + nxyz *= Ny(*iptr); + nxyz *= Nz(*iptr); Frame(*iptr) = (real *) allocate(nxyz * sizeof(real)); dprintf (DLEV,"Frame allocated @ %p ",Frame(*iptr)); } else From 2cd0360839f770c9c7764d2a3de708ab1bf161ee Mon Sep 17 00:00:00 2001 From: Peter Teuben Date: Fri, 3 Jan 2025 19:19:05 -0500 Subject: [PATCH 8/8] proper use of size_t instead of int, for large fits files (64bit) --- inc/fits.h | 16 ++++++++------- src/image/fits/fits.c | 43 ++++++++++++++++++++++----------------- src/image/fits/scanfits.c | 21 +++++++++++-------- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/inc/fits.h b/inc/fits.h index 62e0c0a57..c046be540 100644 --- a/inc/fits.h +++ b/inc/fits.h @@ -1,8 +1,10 @@ /* - * definitions and declarations for nemo's FITS.C + * definitions and declarations for nemo's src/image/fits/fits.c + * * 22-feb-94 ansi- and C++ safe * 10-aug-09 size_t instead of int for 2GB+ files * 25-oct-20 add hdu counter + * 3-jan-24 more proper usage of size_t instead of int */ #ifndef _fits_h_ @@ -129,7 +131,7 @@ static struct arglist { typedef struct fits_header { int hdu; /* keep track with HDU this is (1 being the first) */ /* Optionally we could store the buffer in the first structure too... */ - char *buffer; /* pointer to exact header copy */ + char *buffer; /* pointer to exact header copy , if present */ size_t buflen; /* current length of buffer */ size_t hlen; /* length of header (bytes, through the END keyword) */ size_t dlen; /* length of data (bytes) */ @@ -200,8 +202,8 @@ typedef struct fits_header { } fits_header; -size_t fts_rhead (fits_header *, stream); -char *fts_shead (fits_header *, string); +size_t fts_rhead (fits_header *, stream); +char *fts_shead (fits_header *, string); int fts_lhead (fits_header *); int fts_chead (fits_header *, stream); int fts_thead (fits_header *); @@ -211,7 +213,7 @@ int fts_ihead (fits_header *, string *); int fts_fhead (fits_header *, string *); int fts_phead (fits_header *, string *); int fts_whead (fits_header *, stream); -size_t fts_xhead (fits_header *, stream, int, int, int *, int); +size_t fts_xhead (fits_header *, stream, int, int, int *, int); int fts_ptable (fits_header *, stream, string *, string, int *, int); int fts_pgroup (fits_header *, stream, string *, string, int *, int); @@ -220,8 +222,8 @@ int fts_buf (int); int fts_sdata (fits_header *, stream); int fts_cdata (fits_header *, stream, stream, bool, bool); -int fts_dsize (fits_header *); -int fts_tsize (fits_header *); +size_t fts_dsize (fits_header *); +size_t fts_tsize (fits_header *); int fts_rdata (fits_header *, stream, int, char *); int fts_wdata (fits_header *, stream, int, char *); int fts_rrow (fits_header *, stream, int, char *); diff --git a/src/image/fits/fits.c b/src/image/fits/fits.c index f016b9b06..cc4f256ae 100644 --- a/src/image/fits/fits.c +++ b/src/image/fits/fits.c @@ -151,7 +151,8 @@ size_t fts_rhead(fits_header *fh, stream instr) { char buf[FTSLINSIZ+2]; /* buffer to hold one card image */ char a1[9], a2[2], a3[FTSLINSIZ+1],a4[FTSLINSIZ+1],a5[FTSLINSIZ+1]; /* args */ - int k, n, i, icard; + int k, i, icard; + size_t n; struct arglist *p; if (fh->naxis >= 0) { /* we could force a fts_zero here, but ala */ @@ -175,10 +176,9 @@ size_t fts_rhead(fits_header *fh, stream instr) n = FTSLINSIZ; } else { /* this is of course the legal way to get cards */ if ((n=fread(buf,1,FTSLINSIZ,instr)) != FTSLINSIZ) { /* get card */ - if (n==0) - return -1; /* error - nothing read -- never -- */ - else - return -n-1; /* return some negative number, but < -1 */ + if (n != 0) + warning("Incomplete read %ld bytes in header", n); + return 0; } else { buf[FTSLINSIZ] = 0; /* be sure it's terminated */ dprintf(5,"RECORD[%d]%65s\n",icard,buf); @@ -458,7 +458,7 @@ size_t fts_rhead(fits_header *fh, stream instr) /* process some additional checks */ fts_lhead(fh); /* process a few more sanity checks */ - return fts_dsize(fh); /* return the size of upcoming data */ + return fts_dsize(fh); /* return the size of upcoming data (could be 0 too) */ } /* @@ -1257,7 +1257,7 @@ int fts_phead(fits_header *fh, string *print) printf("headersize = %d bytes = %d %d-records\n", (int) fh->hlen, ((int) fh->hlen - 1)/ftsblksiz_i + 1, ftsblksiz_i); if (dsize > 0) - printf("datasize = %ld bytes = %ld %d-records\n", + printf("datasize = %ld bytes = %ld %ld-records\n", dsize, (dsize-1)/ftsblksiz_i + 1, ftsblksiz_i); else printf("datasize = 0 bytes = 0 %d-records\n", ftsblksiz_i); @@ -2038,11 +2038,14 @@ int fts_sdata( if (nskip % ftsblksiz_i) { ntail = ftsblksiz_i - nskip % ftsblksiz_i; - dprintf(1,"Skipping %ld bytes for empty tail\n",ntail); + dprintf(1,"Skipping %ld bytes for empty tail in data\n",ntail); nskip += ntail; } nskip -= fh->nread; - if (nskip>0) fseek(instr, nskip, 1); + if (nskip>0) { + dprintf(1,"fseek %ld from %ld\n", nskip, fh->nread); + fseek(instr, nskip, 1); // SEEK_CUR=1 + } return 1; } @@ -2058,7 +2061,7 @@ int fts_cdata( bool trailr, /* (i) need to read trailing end too ? */ bool trailw) /* (i) need to write trailing end too ? */ { - int nread, nwrite, n, ntowrite, ntoread, itemlen, nitems; + size_t n, nread, nwrite, ntowrite, ntoread, itemlen, nitems; char buffer[CONVBUFLEN]; itemlen = ABS(fh->bitpix)/8; /* # bytes in an item */ @@ -2073,7 +2076,7 @@ int fts_cdata( ntowrite = n; if (n==0) ntoread=0; /* no data? */ if (fh->flip) dprintf(1,"fts_cdata: Swapping bytes\n"); - dprintf(2,"fts_data: ntoread=%d ntowrite=%d\n",ntoread,ntowrite); + dprintf(2,"fts_data: ntoread=%ld ntowrite=%ld\n",ntoread,ntowrite); while (ntoread > 0) { if (ntoread > CONVBUFLEN) n = CONVBUFLEN; @@ -2081,17 +2084,17 @@ int fts_cdata( n = ntoread; nread = fread(buffer,sizeof(char),n,instr); if (nread != n) /* should be a warning in raw mode */ - error("Tried to read %d, could only read %d",n,nread); + error("Tried to read %ld, could only read %ld",n,nread); fh->nread += nread; if (fh->flip) { nitems = nread/itemlen; - if (nread%itemlen) error("Bad bufsize %d for byteswap",nread); + if (nread%itemlen) error("Bad bufsize %ld for byteswap",nread); bswap(buffer,itemlen, nitems); } /****FIX****/ nwrite = fwrite(buffer,sizeof(char),n,outstr); if (nwrite != n) - error("Tried to write %d, could only write %d\n",n,nwrite); + error("Tried to write %ld, could only write %ld\n",n,nwrite); ntoread -= nread; } return 1; @@ -2146,9 +2149,10 @@ int fts_cdata816( * See also eq. (x.y.z) in the NOST manual */ -int fts_dsize(fits_header *fh) +size_t fts_dsize(fits_header *fh) { - int i, size; + int i; + size_t size = 0; if (fh->naxis > 0) { if (fh->naxisn[0] == 0) { /* NAXIS1=0 is for random groups */ @@ -2164,6 +2168,7 @@ int fts_dsize(fits_header *fh) size *= fh->gcount * ABS(fh->bitpix) / 8; } else size = 0; + dprintf(1,"fts_dsize: %ld\n", size); return size; } @@ -2171,10 +2176,10 @@ int fts_dsize(fits_header *fh) * fts_tsize: return trailing size of useable data portion */ -int fts_tsize(fits_header *fh) +size_t fts_tsize(fits_header *fh) { - int n = fts_dsize(fh); - int tail = ROUNDUP(n,ftsblksiz_o); + size_t n = fts_dsize(fh); + size_t tail = ROUNDUP(n,ftsblksiz_o); return tail - n; } diff --git a/src/image/fits/scanfits.c b/src/image/fits/scanfits.c index 6c9fc8913..a4a886fc8 100644 --- a/src/image/fits/scanfits.c +++ b/src/image/fits/scanfits.c @@ -54,7 +54,7 @@ string defv[] = { /* Standard NEMO keyword+help */ "blocking=1,1\n Two blocking factors (blocksize/2880) for i&o", "select=header,data\n Select header, data, ...?", "split=f\n Split fits file into HDU pieces '.#'", - "VERSION=1.8d\n 13-jul-2021 PJT", + "VERSION=1.8e\n 3-jan-2024 PJT", NULL, }; @@ -67,7 +67,8 @@ extern string *burststring(string, string); void nemo_main() { stream instr, outstr; - int i, n, nfile, blocking[2]; + int i,n,nfile, blocking[2]; + size_t size; string outfile, hdselect, *insert, *fix, *deletes, *keep, *print; char basename[128]; struct fits_header fh; @@ -115,15 +116,17 @@ void nemo_main() fts_zero(&fh); /* reset header */ fh.hdu = i; /* keep track of HDU (1=first) */ - n = fts_rhead(&fh,instr); /* read header */ - if (n<0) /* if no data (EOF) .. */ + size = fts_rhead(&fh,instr); /* read header */ + if (size == 0 ) { /* if no data (EOF) .. */ + dprintf(1," no more data\n"); break; /* ... quit */ + } if (outstr) dprintf(1,"Working on FITS file %d\n",i); - fts_dhead(&fh,deletes); /* delete= headers */ - fts_khead(&fh,keep); /* keep= headers */ - fts_ihead(&fh,insert); /* insert= headers */ - fts_fhead(&fh,fix); /* fix= headers */ - if (!outstr && !split) fts_phead(&fh,print); /* ? print header */ + fts_dhead(&fh,deletes); /* delete= headers */ + fts_khead(&fh,keep); /* keep= headers */ + fts_ihead(&fh,insert); /* insert= headers */ + fts_fhead(&fh,fix); /* fix= headers */ + if (!outstr && !split) fts_phead(&fh,print); /* ? print header */ if ((outstr && (nfile==0 || nfile==i)) || split) { if (split) { if (strchr(outfile,'%'))