diff -Naur netcdf-3.5.1-beta13/src/libsrc/nc.c netcdf-3.5.1-beta13-large-offset/src/libsrc/nc.c --- netcdf-3.5.1-beta13/src/libsrc/nc.c Thu Sep 19 17:18:11 2002 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/nc.c Fri Oct 3 17:07:09 2003 @@ -107,7 +107,7 @@ (void) memset(ncp, 0, sizeof(NC)); ncp->xsz = MIN_NC_XSZ; - assert(ncp->xsz == ncx_len_NC(ncp)); + assert(ncp->xsz == ncx_len_NC(ncp,0)); ncp->chunk = chunkp != NULL ? *chunkp : NC_SIZEHINT_DEFAULT; @@ -206,6 +206,7 @@ size_t v_minfree, size_t r_align) { size_t ii; + int sizeof_off_t; off_t index = 0; NC_var **vpp; NC_var *last = NULL; @@ -215,7 +216,13 @@ if(r_align == NC_ALIGN_CHUNK) r_align = ncp->chunk; - ncp->xsz = ncx_len_NC(ncp); + if (fIsSet(ncp->flags, NC_64BIT_OFFSET)) { + sizeof_off_t = 8; + } else { + sizeof_off_t = 4; + } + + ncp->xsz = ncx_len_NC(ncp,sizeof_off_t); if(ncp->vars.nelems == 0) return; @@ -748,6 +755,7 @@ NC *ncp; int status; void *xp = NULL; + int sizeof_off_t = 0; #if ALWAYS_NC_SHARE /* DEBUG */ fSet(ioflags, NC_SHARE); @@ -768,7 +776,17 @@ if(basepe != 0) return NC_EINVAL; #endif - assert(ncp->xsz == ncx_len_NC(ncp)); + + assert(ncp->flags == 0); + + if (fIsSet(ioflags, NC_64BIT_OFFSET)) { + fSet(ncp->flags, NC_64BIT_OFFSET); + sizeof_off_t = 8; + } else { + sizeof_off_t = 4; + } + + assert(ncp->xsz == ncx_len_NC(ncp,sizeof_off_t)); status = ncio_create(path, ioflags, initialsz, @@ -782,7 +800,6 @@ goto unwind_alloc; } - assert(ncp->flags == 0); fSet(ncp->flags, NC_CREAT); if(fIsSet(ncp->nciop->ioflags, NC_SHARE)) @@ -797,7 +814,7 @@ fSet(ncp->flags, NC_NSYNC); } - status = ncx_put_NC(ncp, &xp, 0, ncp->xsz); + status = ncx_put_NC(ncp, &xp, sizeof_off_t, ncp->xsz); if(status != NC_NOERR) goto unwind_ioc; diff -Naur netcdf-3.5.1-beta13/src/libsrc/nc.h netcdf-3.5.1-beta13-large-offset/src/libsrc/nc.h --- netcdf-3.5.1-beta13/src/libsrc/nc.h Wed Dec 23 09:30:32 1998 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/nc.h Fri Oct 3 16:55:12 2003 @@ -342,7 +342,7 @@ /* Begin defined in v1hpg.c */ extern size_t -ncx_len_NC(const NC *ncp); +ncx_len_NC(const NC *ncp, size_t sizeof_off_t); extern int ncx_put_NC(const NC *ncp, void **xpp, off_t offset, size_t extent); diff -Naur netcdf-3.5.1-beta13/src/libsrc/ncx.c netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.c --- netcdf-3.5.1-beta13/src/libsrc/ncx.c Thu Dec 18 15:36:40 1997 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.c Mon Oct 6 15:45:58 2003 @@ -1743,41 +1743,97 @@ /* x_off_t */ -#if SIZEOF_OFF_T < X_SIZEOF_OFF_T -#error "x_off_t implementation" -/* netcdf requires size_t which can hold a values from 0 to 2^31 -1 */ -#endif - int -ncx_put_off_t(void **xpp, const off_t *lp) +ncx_put_off_t(void **xpp, const off_t *lp, size_t sizeof_off_t) { /* similar to put_ix_int() */ uchar *cp = (uchar *) *xpp; /* No negative offsets stored in netcdf */ - assert(*lp >= 0 && *lp <= X_OFF_MAX); - - *cp++ = (uchar)((*lp) >> 24); - *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); - *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); - *cp = (uchar)((*lp) & 0x000000ff); - - *xpp = (void *)((char *)(*xpp) + X_SIZEOF_OFF_T); + if (*lp < 0) { + /* Assume this is an overflow of a 32-bit int... */ + return ERANGE; + } + + assert(sizeof_off_t == 4 || sizeof_off_t == 8); + + if (sizeof_off_t == 4) { + *cp++ = (uchar) ((*lp) >> 24); + *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); + *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); + *cp = (uchar)( (*lp) & 0x000000ff); + } else { +#if SIZEOF_OFF_T == 4 +/* Write a 64-bit offset on a system with only a 32-bit offset */ + *cp++ = (uchar)0; + *cp++ = (uchar)0; + *cp++ = (uchar)0; + *cp++ = (uchar)0; + + *cp++ = (uchar)(((*lp) & 0xff000000) >> 24); + *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); + *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); + *cp = (uchar)( (*lp) & 0x000000ff); +#else + *cp++ = (uchar) ((*lp) >> 56); + *cp++ = (uchar)(((*lp) & 0x00ff000000000000ULL) >> 48); + *cp++ = (uchar)(((*lp) & 0x0000ff0000000000ULL) >> 40); + *cp++ = (uchar)(((*lp) & 0x000000ff00000000ULL) >> 32); + *cp++ = (uchar)(((*lp) & 0x00000000ff000000ULL) >> 24); + *cp++ = (uchar)(((*lp) & 0x0000000000ff0000ULL) >> 16); + *cp++ = (uchar)(((*lp) & 0x000000000000ff00ULL) >> 8); + *cp = (uchar)( (*lp) & 0x00000000000000ffULL); +#endif + } + *xpp = (void *)((char *)(*xpp) + sizeof_off_t); return ENOERR; } int -ncx_get_off_t(const void **xpp, off_t *lp) +ncx_get_off_t(const void **xpp, off_t *lp, size_t sizeof_off_t) { /* similar to get_ix_int() */ const uchar *cp = (const uchar *) *xpp; assert((*cp & 0x80) == 0); /* No negative offsets stored in netcdf */ + assert(sizeof_off_t == 4 || sizeof_off_t == 8); - *lp = *cp++ << 24; - *lp |= (*cp++ << 16); - *lp |= (*cp++ << 8); - *lp |= *cp; - - *xpp = (const void *)((const char *)(*xpp) + X_SIZEOF_OFF_T); + if (sizeof_off_t == 4) { + *lp = *cp++ << 24; + *lp |= (*cp++ << 16); + *lp |= (*cp++ << 8); + *lp |= *cp; + } else { +#if SIZEOF_OFF_T == 4 +/* Read a 64-bit offset on a system with only a 32-bit offset */ +/* If the offset overflows, set an error code and return */ + *lp = ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= ((off_t)(*cp++)); +/* + * lp now contains the upper 32-bits of the 64-bit offset. if lp is + * not zero, then the dataset is larger than can be represented + * on this system. Set an error code and return + */ + if (*lp != 0) { + return ERANGE; + } + + *lp = ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= (off_t)*cp; +#else + *lp = ((off_t)(*cp++) << 56); + *lp |= ((off_t)(*cp++) << 48); + *lp |= ((off_t)(*cp++) << 40); + *lp |= ((off_t)(*cp++) << 32); + *lp |= ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= (off_t)*cp; +#endif + } + *xpp = (const void *)((const char *)(*xpp) + sizeof_off_t); return ENOERR; } diff -Naur netcdf-3.5.1-beta13/src/libsrc/ncx.h netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.h --- netcdf-3.5.1-beta13/src/libsrc/ncx.h Thu Apr 20 14:42:03 2000 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.h Fri Oct 3 16:54:16 2003 @@ -63,10 +63,13 @@ #define X_SIZEOF_DOUBLE 8 /* - * For now, netcdf is limited to 32 bit offsets and sizes, + * For now, netcdf is limited to 32 bit sizes, + * If compiled with support for "large files", then + * netcdf will use a 64 bit off_t and it can then write a file + * using 64 bit offsets. * see also X_SIZE_MAX, X_OFF_MAX below */ -#define X_SIZEOF_OFF_T X_SIZEOF_INT +#define X_SIZEOF_OFF_T (sizeof(off_t)) #define X_SIZEOF_SIZE_T X_SIZEOF_INT /* @@ -361,14 +364,14 @@ ncx_get_size_t(const void **xpp, size_t *ulp); /* ncx_get_int_off_t */ extern int -ncx_get_off_t(const void **xpp, off_t *lp); +ncx_get_off_t(const void **xpp, off_t *lp, size_t sizeof_off_t); /* ncx_put_int_size_t */ extern int ncx_put_size_t(void **xpp, const size_t *ulp); /* ncx_put_int_off_t */ extern int -ncx_put_off_t(void **xpp, const off_t *lp); +ncx_put_off_t(void **xpp, const off_t *lp, size_t sizeof_off_t); /* diff -Naur netcdf-3.5.1-beta13/src/libsrc/ncx.m4 netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.m4 --- netcdf-3.5.1-beta13/src/libsrc/ncx.m4 Thu Dec 18 15:34:04 1997 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/ncx.m4 Mon Oct 6 15:58:21 2003 @@ -1728,41 +1728,106 @@ /* x_off_t */ -#if SIZEOF_OFF_T < X_SIZEOF_OFF_T -#error "x_off_t implementation" -/* netcdf requires size_t which can hold a values from 0 to 2^31 -1 */ -#endif - int -ncx_put_off_t(void **xpp, const off_t *lp) +ncx_put_off_t(void **xpp, const off_t *lp, size_t sizeof_off_t) { /* similar to put_ix_int() */ uchar *cp = (uchar *) *xpp; /* No negative offsets stored in netcdf */ - assert(*lp >= 0 && *lp <= X_OFF_MAX); - - *cp++ = (uchar)((*lp) >> 24); - *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); - *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); - *cp = (uchar)((*lp) & 0x000000ff); - - *xpp = (void *)((char *)(*xpp) + X_SIZEOF_OFF_T); + if (*lp < 0) { + /* Assume this is an overflow of a 32-bit int... */ + return ERANGE; + } + + assert(sizeof_off_t == 4 || sizeof_off_t == 8); + + if (sizeof_off_t == 4) { + *cp++ = (uchar) ((*lp) >> 24); + *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); + *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); + *cp = (uchar)( (*lp) & 0x000000ff); + } else { +#if SIZEOF_OFF_T == 4 +/* Write a 64-bit offset on a system with only a 32-bit offset */ + *cp++ = (uchar)0; + *cp++ = (uchar)0; + *cp++ = (uchar)0; + *cp++ = (uchar)0; + + *cp++ = (uchar)(((*lp) & 0xff000000) >> 24); + *cp++ = (uchar)(((*lp) & 0x00ff0000) >> 16); + *cp++ = (uchar)(((*lp) & 0x0000ff00) >> 8); + *cp = (uchar)( (*lp) & 0x000000ff); +#else + *cp++ = (uchar) ((*lp) >> 56); + *cp++ = (uchar)(((*lp) & 0x00ff000000000000ULL) >> 48); + *cp++ = (uchar)(((*lp) & 0x0000ff0000000000ULL) >> 40); + *cp++ = (uchar)(((*lp) & 0x000000ff00000000ULL) >> 32); + *cp++ = (uchar)(((*lp) & 0x00000000ff000000ULL) >> 24); + *cp++ = (uchar)(((*lp) & 0x0000000000ff0000ULL) >> 16); + *cp++ = (uchar)(((*lp) & 0x000000000000ff00ULL) >> 8); + *cp = (uchar)( (*lp) & 0x00000000000000ffULL); +#endif + } + *xpp = (void *)((char *)(*xpp) + sizeof_off_t); return ENOERR; } int -ncx_get_off_t(const void **xpp, off_t *lp) +ncx_get_off_t(const void **xpp, off_t *lp, size_t sizeof_off_t) { /* similar to get_ix_int() */ const uchar *cp = (const uchar *) *xpp; assert((*cp & 0x80) == 0); /* No negative offsets stored in netcdf */ + assert(sizeof_off_t == 4 || sizeof_off_t == 8); - *lp = *cp++ << 24; - *lp |= (*cp++ << 16); - *lp |= (*cp++ << 8); - *lp |= *cp; - - *xpp = (const void *)((const char *)(*xpp) + X_SIZEOF_OFF_T); + if (sizeof_off_t == 4) { + *lp = *cp++ << 24; + *lp |= (*cp++ << 16); + *lp |= (*cp++ << 8); + *lp |= *cp; + } else { +#if SIZEOF_OFF_T == 4 +/* Read a 64-bit offset on a system with only a 32-bit offset */ +/* If the offset overflows, set an error code and return */ + *lp = ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= ((off_t)(*cp++)); +/* + * lp now contains the upper 32-bits of the 64-bit offset. if lp is + * not zero, then the dataset is larger than can be represented + * on this system. Set an error code and return. + */ + if (*lp != 0) { + return ERANGE; + } + + *lp = ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= (off_t)*cp; + + if (*lp < 0) { + /* + * If this fails, then the offset is >2^31, but less + * than 2^32 which is not allowed, but is not caught + * by the previous check + */ + return ERANGE; + } +#else + *lp = ((off_t)(*cp++) << 56); + *lp |= ((off_t)(*cp++) << 48); + *lp |= ((off_t)(*cp++) << 40); + *lp |= ((off_t)(*cp++) << 32); + *lp |= ((off_t)(*cp++) << 24); + *lp |= ((off_t)(*cp++) << 16); + *lp |= ((off_t)(*cp++) << 8); + *lp |= (off_t)*cp; +#endif + } + *xpp = (const void *)((const char *)(*xpp) + sizeof_off_t); return ENOERR; } diff -Naur netcdf-3.5.1-beta13/src/libsrc/netcdf.h netcdf-3.5.1-beta13-large-offset/src/libsrc/netcdf.h --- netcdf-3.5.1-beta13/src/libsrc/netcdf.h Thu May 10 08:26:40 2001 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/netcdf.h Fri Oct 3 16:46:46 2003 @@ -91,6 +91,7 @@ #define NC_NOFILL 0x100 /* Don't fill data section an records */ #define NC_LOCK 0x0400 /* Use locking if available */ #define NC_SHARE 0x0800 /* Share updates, limit cacheing */ +#define NC_64BIT_OFFSET 0x0200 /* Use large (64-bit) file offsets */ /* * Let nc__create() or nc__open() figure out diff -Naur netcdf-3.5.1-beta13/src/libsrc/v1hpg.c netcdf-3.5.1-beta13-large-offset/src/libsrc/v1hpg.c --- netcdf-3.5.1-beta13/src/libsrc/v1hpg.c Wed Aug 13 09:56:12 2003 +++ netcdf-3.5.1-beta13-large-offset/src/libsrc/v1hpg.c Mon Oct 6 15:15:38 2003 @@ -6,6 +6,7 @@ #include "nc.h" #include +#include #include #include #include "rnd.h" @@ -27,7 +28,8 @@ * "magic number" at beginning of file: 0x43444601 (big endian) * assert(sizeof(ncmagic) % X_ALIGN == 0); */ -static const schar ncmagic[] = {'C', 'D', 'F', 0x01}; +static const schar ncmagic[] = {'C', 'D', 'F', 0x02}; +static const schar ncmagic1[] = {'C', 'D', 'F', 0x01}; /* @@ -44,6 +46,7 @@ off_t offset; /* argument to nciop->get() */ size_t extent; /* argument to nciop->get() */ int flags; /* set to RGN_WRITE for write */ + int version; /* either 1 for normal netcdf or 2 for 8-byte offset version */ void *base; /* beginning of current buffer */ void *pos; /* current position in buffer */ void *end; /* end of current buffer = base + extent */ @@ -125,7 +128,7 @@ static int v1h_put_size_t(v1hs *psp, const size_t *sp) { - int status = check_v1hs(psp, X_SIZEOF_SIZE_T); + int status = check_v1hs(psp, psp->version == 1 ? 4 : 8); if(status != ENOERR) return status; return ncx_put_size_t(&psp->pos, sp); @@ -135,7 +138,7 @@ static int v1h_get_size_t(v1hs *gsp, size_t *sp) { - int status = check_v1hs(gsp, X_SIZEOF_SIZE_T); + int status = check_v1hs(gsp, gsp->version == 1 ? 4 : 8); if(status != ENOERR) return status; return ncx_get_size_t((const void **)(&gsp->pos), sp); @@ -803,11 +806,12 @@ NC_xlen_var(vpp) */ static size_t -ncx_len_NC_var(const NC_var *varp) +ncx_len_NC_var(const NC_var *varp, size_t sizeof_off_t) { size_t sz; assert(varp != NULL); + assert(sizeof_off_t != 0); sz = ncx_len_NC_string(varp->name); sz += X_SIZEOF_SIZE_T; /* ndims */ @@ -815,7 +819,7 @@ sz += ncx_len_NC_attrarray(&varp->attrs); sz += X_SIZEOF_NC_TYPE; /* type */ sz += X_SIZEOF_SIZE_T; /* len */ - sz += X_SIZEOF_OFF_T; /* begin */ + sz += sizeof_off_t; /* begin */ return(sz); } @@ -854,10 +858,10 @@ if(status != ENOERR) return status; - status = check_v1hs(psp, X_SIZEOF_OFF_T); + status = check_v1hs(psp, psp->version == 1 ? 4 : 8); if(status != ENOERR) return status; - status = ncx_put_off_t(&psp->pos, &varp->begin); + status = ncx_put_off_t(&psp->pos, &varp->begin, psp->version == 1 ? 4 : 8); if(status != ENOERR) return status; @@ -908,11 +912,11 @@ if(status != ENOERR) goto unwind_alloc; - status = check_v1hs(gsp, X_SIZEOF_OFF_T); + status = check_v1hs(gsp, gsp->version == 1 ? 4 : 8); if(status != ENOERR) goto unwind_alloc; status = ncx_get_off_t((const void **)&gsp->pos, - &varp->begin); + &varp->begin, gsp->version == 1 ? 4 : 8); if(status != ENOERR) goto unwind_alloc; @@ -930,7 +934,7 @@ static size_t -ncx_len_NC_vararray(const NC_vararray *ncap) +ncx_len_NC_vararray(const NC_vararray *ncap, size_t sizeof_off_t) { size_t xlen = X_SIZEOF_NCTYPE; /* type */ xlen += X_SIZEOF_SIZE_T; /* count */ @@ -942,7 +946,7 @@ const NC_var *const *const end = &vpp[ncap->nelems]; for( /*NADA*/; vpp < end; vpp++) { - xlen += ncx_len_NC_var(*vpp); + xlen += ncx_len_NC_var(*vpp, sizeof_off_t); } } return xlen; @@ -1132,7 +1136,7 @@ size_t -ncx_len_NC(const NC *ncp) +ncx_len_NC(const NC *ncp, size_t sizeof_off_t) { size_t xlen = sizeof(ncmagic); @@ -1141,7 +1145,7 @@ xlen += X_SIZEOF_SIZE_T; /* numrecs */ xlen += ncx_len_NC_dimarray(&ncp->dims); xlen += ncx_len_NC_attrarray(&ncp->attrs); - xlen += ncx_len_NC_vararray(&ncp->vars); + xlen += ncx_len_NC_vararray(&ncp->vars, sizeof_off_t); return xlen; } @@ -1160,6 +1164,11 @@ ps.nciop = ncp->nciop; ps.flags = RGN_WRITE; + if (ncp->flags & NC_64BIT_OFFSET) + ps.version = 2; + else + ps.version = 1; + if(xpp == NULL) { /* @@ -1198,7 +1207,10 @@ ps.end = (char *)ps.base + ps.extent; } - status = ncx_putn_schar_schar(&ps.pos, sizeof(ncmagic), ncmagic); + if (ps.version == 2) + status = ncx_putn_schar_schar(&ps.pos, sizeof(ncmagic), ncmagic); + else + status = ncx_putn_schar_schar(&ps.pos, sizeof(ncmagic1), ncmagic1); if(status != ENOERR) goto release; @@ -1244,6 +1256,7 @@ gs.offset = 0; /* beginning of file */ gs.extent = 0; gs.flags = 0; + gs.version = 0; gs.base = NULL; gs.pos = gs.base; @@ -1291,11 +1304,23 @@ if(status != ENOERR) goto unwind_get; - if(memcmp(magic, ncmagic, sizeof(ncmagic)) != 0) + if(memcmp(magic, ncmagic, sizeof(ncmagic)-1) != 0) { status = NC_ENOTNC; goto unwind_get; } + /* Check version number in last byte of magic */ + if (magic[sizeof(ncmagic)-1] == 0x1) { + gs.version = 1; + } else if (magic[sizeof(ncmagic)-1] == 0x2) { + gs.version = 2; + if (sizeof(off_t) != 8) { + fprintf(stderr, "NETCDF WARNING: Version 2 database on 32-bit system.\n"); + } + } else { + status = NC_ENOTNC; + goto unwind_get; + } } { @@ -1320,7 +1345,7 @@ if(status != ENOERR) goto unwind_get; - ncp->xsz = ncx_len_NC(ncp); + ncp->xsz = ncx_len_NC(ncp, (gs.version == 1) ? 4 : 8); status = NC_computeshapes(ncp);