From 3fda8225649c06a3d78378705a0d5f74c90df78a Mon Sep 17 00:00:00 2001 From: Matthew Petroff Date: Tue, 6 Apr 2021 18:44:46 -0400 Subject: [PATCH] Add read-only support for zipped Dirfiles. Separate from the Dirfile encoding scheme, GetData will read Dirfiles contained in uncompressed Zip files. This functionality is meant for reading archival data, so writing to these Zip files is not supported. Using the Info-ZIP `zip` utility, a Zip file can be created by running `zip -r0 ../dirfile.zip *` from within the root of an existing Dirfile. All encoding schemes are supported by this functionality except for the two encoding schemes that already use Zip files, *zzip* and *zzslim*. The encoding scheme must be specified using the /ENCODING directive, even if the Dirfile is unencoded. For /INCLUDE directives and LINTERP field look up table files, only relative paths are supported and only without `./` and `../` syntax. Although Zip files are most commonly created using _Deflate_ compression, the Zip standard (ISO/IEC 21320-1) also supports _Store_ compression, i.e., no compression at all. GetData's Zip file support requires _Store_ compression for all data files, although either _Store_ compression or _Deflate_ compression can be used for any *format* files or any LINTERP field look up table files. With _Store_ compression, a Zip file effectively concatenates a Dirfile's individual files together into a single file. Since a Zip file contains an offset table, unlike a tarball, random reads are supported without the need to load the entire file from disk. --- .gitignore | 11 ++++ src/Makefile.am | 3 ++ src/ascii.c | 7 +-- src/bzip.c | 2 +- src/close.c | 5 ++ src/common.c | 34 ++++++++---- src/flac.c | 2 +- src/gzip.c | 25 +++++++-- src/include.c | 30 ++++++++--- src/internal.h | 15 ++++++ src/iopos.c | 107 +++++++++++++++++++++++++++++++++++++ src/lzma.c | 6 ++- src/open.c | 47 +++++++++++----- src/raw.c | 29 +++++++--- src/sie.c | 65 +++++++++++++++++----- src/slim.c | 23 +++++++- test/Makefile.am | 19 +++---- test/alloc_callback.c | 4 +- test/alloc_clear.c | 4 +- test/alloc_entry.c | 4 +- test/ascii_int32_zip.c | 27 ++++++++++ test/bzip_int32_zip.c | 31 +++++++++++ test/enc_int32_zip.c | 80 +++++++++++++++++++++++++++ test/flac_int32_zip.c | 31 +++++++++++ test/get_int32_zip.c | 68 +++++++++++++++++++++++ test/get_int32_zip2.c | 69 ++++++++++++++++++++++++ test/get_linterp_zip.c | 74 +++++++++++++++++++++++++ test/get_linterp_zip2.c | 75 ++++++++++++++++++++++++++ test/gzip_int32_zip.c | 31 +++++++++++ test/lzma_xz_int32_zip.c | 31 +++++++++++ test/nframes_nframes_zip.c | 68 +++++++++++++++++++++++ test/sie_get_little_zip.c | 75 ++++++++++++++++++++++++++ 32 files changed, 1024 insertions(+), 78 deletions(-) create mode 100644 test/ascii_int32_zip.c create mode 100644 test/bzip_int32_zip.c create mode 100644 test/enc_int32_zip.c create mode 100644 test/flac_int32_zip.c create mode 100644 test/get_int32_zip.c create mode 100644 test/get_int32_zip2.c create mode 100644 test/get_linterp_zip.c create mode 100644 test/get_linterp_zip2.c create mode 100644 test/gzip_int32_zip.c create mode 100644 test/lzma_xz_int32_zip.c create mode 100644 test/nframes_nframes_zip.c create mode 100644 test/sie_get_little_zip.c diff --git a/.gitignore b/.gitignore index 18d3161a..11dc8786 100644 --- a/.gitignore +++ b/.gitignore @@ -309,6 +309,7 @@ test/ascii_get_here test/ascii_get_sub test/ascii_int16 test/ascii_int32 +test/ascii_int32_zip test/ascii_int64 test/ascii_int8 test/ascii_nframes @@ -348,6 +349,7 @@ test/bzip_get_get2 test/bzip_get_put test/bzip_int16 test/bzip_int32 +test/bzip_int32_zip test/bzip_int64 test/bzip_int8 test/bzip_move_from @@ -642,6 +644,7 @@ test/flac_get_little test/flac_get_long test/flac_int16 test/flac_int32 +test/flac_int32_zip test/flac_int64 test/flac_int8 test/flac_move_from @@ -794,6 +797,8 @@ test/get_indir test/get_indir_typein test/get_int16 test/get_int32 +test/get_int32_zip +test/get_int32_zip2 test/get_int64 test/get_int8 test/get_invalid @@ -817,6 +822,8 @@ test/get_linterp_nodir test/get_linterp_noin test/get_linterp_notab test/get_linterp_sort +test/get_linterp_zip +test/get_linterp_zip2 test/get_mplex test/get_mplex_bof test/get_mplex_complex @@ -902,6 +909,7 @@ test/gzip_get_get2 test/gzip_get_put test/gzip_int16 test/gzip_int32 +test/gzip_int32_zip test/gzip_int64 test/gzip_int8 test/gzip_move_from @@ -993,6 +1001,7 @@ test/lzma_xz_get_get2 test/lzma_xz_get_put test/lzma_xz_int16 test/lzma_xz_int32 +test/lzma_xz_int32_zip test/lzma_xz_int64 test/lzma_xz_int8 test/lzma_xz_move_from @@ -1186,6 +1195,7 @@ test/nframes64 test/nframes_empty test/nframes_invalid test/nframes_nframes +test/nframes_nframes_zip test/nframes_off64 test/nframes_spf test/nmeta @@ -1607,6 +1617,7 @@ test/sie_err_open test/sie_get_big test/sie_get_header test/sie_get_little +test/sie_get_little_zip test/sie_move_from test/sie_move_to test/sie_nframes_big diff --git a/src/Makefile.am b/src/Makefile.am index ca0c2a7a..fc9f0ad4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -80,6 +80,9 @@ MODULE_LIBS=libgetdata.la if USE_MODULES EXPORT_DYNAMIC=-export-dynamic DGETDATA_MODULEDIR=-DGETDATA_MODULEDIR="\"$(moduledir)\"" +EXTERNAL_CPPFLAGS=$(ZZIP_CPPFLAGS) +EXTERNAL_LDFLAGS=$(ZZIP_LDFLAGS) +EXTERNAL_LIBS=$(ZZIP_LIBS) else EXTERNAL_CPPFLAGS=$(SLIM_CPPFLAGS) $(GZIP_CPPFLAGS) $(BZIP2_CPPFLAGS) \ $(LZMA_CPPFLAGS) $(ZZIP_CPPFLAGS) $(FLAC_CPPFLAGS) diff --git a/src/ascii.c b/src/ascii.c index 50338be9..7d7442b7 100644 --- a/src/ascii.c +++ b/src/ascii.c @@ -28,7 +28,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_, dtrace("%i, %p, , , %u", fd, file, mode); if (!(mode & GD_FILE_TEMP)) - file->idata = gd_OpenAt(file->D, fd, file->name, ((mode & GD_FILE_WRITE) + file->idata = gd_openat_wrapper(file->D, fd, file->name, ((mode & GD_FILE_WRITE) ? (O_RDWR | O_CREAT) : O_RDONLY) | O_BINARY, 0666); else file->idata = _GD_MakeTempFile(file->D, fd, file->name); @@ -49,6 +49,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_, file->mode = mode | GD_FILE_READ; file->pos = 0; + file->start_offset = ftello64(file->edata); dreturn("%i", 0); return 0; } @@ -61,7 +62,7 @@ off64_t _GD_AsciiSeek(struct gd_raw_file_* file, off64_t count, dtrace("%p, %" PRId64 ", , 0x%X", file, (int64_t)count, mode); if (count < file->pos) { - rewind((FILE *)file->edata); + fseeko64((FILE *)file->edata, file->start_offset, SEEK_SET); /* rewind */ file->pos = 0; } @@ -285,7 +286,7 @@ off64_t _GD_AsciiSize(int dirfd, struct gd_raw_file_* file, dtrace("%i, %p, , ", dirfd, file); - fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY, 0666); + fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY, 0666); if (fd < 0) { dreturn("%i", -1); return -1; diff --git a/src/bzip.c b/src/bzip.c index 290450fe..293db79d 100644 --- a/src/bzip.c +++ b/src/bzip.c @@ -56,7 +56,7 @@ static struct gd_bzdata *_GD_Bzip2DoOpen(int dirfd, struct gd_raw_file_* file, file->error = BZ_IO_ERROR; if (mode & GD_FILE_READ) { - fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); + fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); } else if (mode & GD_FILE_TEMP) { fd = _GD_MakeTempFile(file->D, dirfd, file->name); fdmode = "wb"; diff --git a/src/close.c b/src/close.c index 063fbe9d..881b1bbd 100644 --- a/src/close.c +++ b/src/close.c @@ -76,6 +76,11 @@ static void _GD_FreeD(DIRFILE *D, int keep_dirfile) } free(D->dir); +#ifdef HAVE_ZZIP_LIB_H + if (D->zzip_dir) + zzip_dir_close(D->zzip_dir); +#endif + if (!keep_dirfile) free(D); diff --git a/src/common.c b/src/common.c index bf7684c0..8dbdd6fa 100644 --- a/src/common.c +++ b/src/common.c @@ -346,7 +346,7 @@ static int lutcmp(const void* a, const void* b) */ int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E) { - FILE *fp; + FILE *fp = NULL; struct gd_lut_ *ptr; int i, fd; int dir = -1; @@ -364,15 +364,23 @@ int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E) return 1; } - fd = gd_OpenAt(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file, - O_RDONLY, 0666); - if (fd == -1) { - _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL); - dreturn("%i", 1); - return 1; + if (!D->zzip_dir) { + fd = gd_openat_wrapper(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file, + O_RDONLY, 0666); + if (fd == -1) { + _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL); + dreturn("%i", 1); + return 1; + } + fp = fdopen(fd, "rb"); + } else { + if (!gd_zip_read_file(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file, &fp)) { + _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL); + dreturn("%i", 1); + return 1; + } } - fp = fdopen(fd, "rb"); if (fp == NULL) { _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL); dreturn("%i", 1); @@ -1430,9 +1438,13 @@ int _GD_GrabDir(DIRFILE *D, int dirfd, const char *name, int canonical) free(path); if (D->dir[D->ndir].fd == -1) { - free(D->dir[D->ndir].path); - dreturn("%i", -1); - return -1; + if (D->zzip_dir) { + D->dir[D->ndir].fd = dup(dirfd); + } else { + free(D->dir[D->ndir].path); + dreturn("%i", -1); + return -1; + } } #endif D->ndir++; diff --git a/src/flac.c b/src/flac.c index 4edd8265..a89057e7 100644 --- a/src/flac.c +++ b/src/flac.c @@ -151,7 +151,7 @@ static struct gd_flacdata *_GD_FlacDoOpen(int dirfd, struct gd_raw_file_* file, dtrace("%i, %p, 0x%X, %i, 0x%X", dirfd, file, data_type, swap, mode); if (mode & GD_FILE_READ) { - fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); + fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); } else if (mode & GD_FILE_TEMP) { fd = _GD_MakeTempFile(file->D, dirfd, file->name); fdmode = "wb"; diff --git a/src/gzip.c b/src/gzip.c index f5e45df6..3f9ce1b2 100644 --- a/src/gzip.c +++ b/src/gzip.c @@ -46,7 +46,7 @@ int _GD_GzipOpen(int fd, struct gd_raw_file_* file, dtrace("%i, %p, , , 0x%X", fd, file, mode); if (mode & GD_FILE_READ) { - file->idata = gd_OpenAt(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666); + file->idata = gd_openat_wrapper(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666); gzmode = "r"; } else if (mode & GD_FILE_TEMP) { file->idata = _GD_MakeTempFile(file->D, fd, file->name); @@ -185,17 +185,32 @@ off64_t _GD_GzipSize(int dirfd, struct gd_raw_file_ *file, gd_type_t data_type, dtrace("%i, %p, 0x%X, ", dirfd, file, data_type); - fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); + fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); if (fd < 0) { dreturn("%i", -1); return -1; } /* seek to the end */ - if (lseek64(fd, -4, SEEK_END) == -1) { - dreturn("%i", -1); - return -1; +#ifdef HAVE_ZZIP_LIB_H + if (file->D->zzip_dir) { + ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, O_RDONLY | O_BINARY); + if (zzip_file && zzip_file->method == 0) { + lseek64(dirfd, zzip_file->dataoffset + zzip_file->csize - 4, SEEK_SET); + zzip_file_close(zzip_file); + } else { + dreturn("%i", -1); + return -1; + } + } else { +#endif + if (lseek64(fd, -4, SEEK_END) == -1) { + dreturn("%i", -1); + return -1; + } +#ifdef HAVE_ZZIP_LIB_H } +#endif if (read(fd, &size, 4) < 4) { dreturn("%i", -1); return -1; diff --git a/src/include.c b/src/include.c index f0d16dfa..af1f0af3 100644 --- a/src/include.c +++ b/src/include.c @@ -253,14 +253,14 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename, /* Open the containing directory */ dirfd = _GD_GrabDir(D, D->fragment[parent].dirfd, temp_buf1, 1); - if (dirfd == -1 && D->error == GD_E_OK) + if (dirfd == -1 && D->error == GD_E_OK && !D->zzip_dir) _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, ename); if (D->error) goto include_error; /* Reject weird stuff */ if (gd_StatAt(D, dirfd, base, &statbuf, 0)) { - if (!(p->flags & GD_CREAT)) { + if (!(p->flags & GD_CREAT) && !D->zzip_dir) { _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1); _GD_ReleaseDir(D, dirfd); goto include_error; @@ -280,11 +280,15 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename, } /* Try to open the file */ - i = gd_OpenAt(D, dirfd, base, - ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) | - ((p->flags & GD_CREAT) ? O_CREAT : 0) | - ((p->flags & GD_TRUNC) ? O_TRUNC : 0) | - ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666); + if (D->zzip_dir) { + i = open(D->name, O_RDONLY | O_BINARY); + } else { + i = gd_OpenAt(D, dirfd, base, + ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) | + ((p->flags & GD_CREAT) ? O_CREAT : 0) | + ((p->flags & GD_TRUNC) ? O_TRUNC : 0) | + ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666); + } if (i < 0) { _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1); @@ -292,7 +296,17 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename, goto include_error; } - new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb"); + if (!D->zzip_dir) { + new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb"); + } else { + if (!gd_zip_read_file(D, dirfd, base, &new_fp)) { + _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1); + _GD_ReleaseDir(D, dirfd); + close(i); + goto include_error; + } + close(i); + } /* If opening the file failed, set the error code and abort parsing. */ if (new_fp == NULL) { diff --git a/src/internal.h b/src/internal.h index b127ea00..b203484c 100644 --- a/src/internal.h +++ b/src/internal.h @@ -113,6 +113,10 @@ #include #endif +#ifdef HAVE_ZZIP_LIB_H +#include +#include +#endif /* MSCVRT defines size_t but not ssize_t */ #ifdef __MSVCRT__ @@ -994,6 +998,7 @@ struct gd_raw_file_ { DIRFILE *D; unsigned int mode; off64_t pos; + off64_t start_offset; }; /* linterp table datum */ @@ -1283,6 +1288,13 @@ struct gd_dirfile_ { /* syntax error callback */ gd_parser_callback_t sehandler; void* sehandler_extra; + + /* for zipped Dirfile support */ +#ifdef HAVE_ZZIP_LIB_H + ZZIP_DIR *zzip_dir; +#else + void *zzip_dir; +#endif }; /* The caller's preferred memory manager */ @@ -1434,6 +1446,9 @@ gd_entry_t *_GD_ParseFieldSpec(DIRFILE *restrict, char *_GD_ParseFragment(FILE *restrict, DIRFILE*, struct parser_state *restrict, int, int); void _GD_PerformRename(DIRFILE *restrict, struct gd_rename_data_ *restrict); +int gd_openat_wrapper(const DIRFILE *D, int dirfd, const char *name, int flags, + mode_t mode); +int gd_zip_read_file(const DIRFILE *D, int dirfd, const char *name, FILE **fp); struct gd_rename_data_ *_GD_PrepareRename(DIRFILE *restrict, char *restrict, size_t, gd_entry_t *restrict, int, unsigned); int _GD_ReadLinterpFile(DIRFILE *restrict, gd_entry_t *restrict); diff --git a/src/iopos.c b/src/iopos.c index 000197e9..a4b16667 100644 --- a/src/iopos.c +++ b/src/iopos.c @@ -415,5 +415,112 @@ off_t gd_seek(DIRFILE *D, const char *field_code, off_t frame_num, { return (off_t)gd_seek64(D, field_code, frame_num, sample_num, whence); } + +int gd_openat_wrapper(const DIRFILE *D, int dirfd, const char *name, int flags, + mode_t mode) +{ + int ret; + const char *dir = NULL; + const char *p1, *p2, *p; + char *newp; +#ifdef HAVE_ZZIP_LIB_H + ZZIP_FILE *zzip_file; +#endif + + dtrace("%p, %i, \"%s\", 0x%X, 0%o", D, dirfd, name, flags, mode); + +#ifdef HAVE_ZZIP_LIB_H + if (!D->zzip_dir) { +#endif + ret = gd_OpenAt(D, dirfd, name, flags, mode); +#ifdef HAVE_ZZIP_LIB_H + } else { + dir = _GD_DirName(D, dirfd); + /* find where zip file and desired file's absolute paths differ */ + for (p1 = dir, p2 = D->dir[0].path; *p1 && *p1 == *p2; p1++, p2++); + p = *(p1) && *(p1+1) ? p1+1 : NULL; + if (p) { + /* if paths differ (i.e. not root of zip), construct relative path */ + newp = malloc(strlen(p) + 1 + strlen(name) + 1); + strcpy(newp, p); + newp[strlen(p)] = '/'; + strcpy(newp + strlen(p) + 1, name); + zzip_file = zzip_file_open(D->zzip_dir, newp, flags | O_BINARY); + free(newp); + } else { + /* root of zip */ + zzip_file = zzip_file_open(D->zzip_dir, name, flags | O_BINARY); + } + if (zzip_file && zzip_file->method == 0) { + lseek64(dirfd, zzip_file->dataoffset, SEEK_SET); + zzip_file_close(zzip_file); + ret = dup(dirfd); + } else { + ret = EACCES; + } + } +#endif + + dreturn("%i", ret); + return ret; +} + +int gd_zip_read_file(const DIRFILE *D, int dirfd, const char *name, FILE **fp) +{ + int ret; + const char *p1, *p2, *p; + char *newp; +#ifdef HAVE_ZZIP_LIB_H + ZZIP_FILE *zzip_file; +#endif + const char *dir = NULL; + + dtrace("%p, %i, \"%s\", %p", D, dirfd, name, fp); + +#ifdef HAVE_ZZIP_LIB_H + dir = _GD_DirName(D, dirfd); + /* find where zip file and desired file's absolute paths differ */ + if (D->ndir > 0) { + for (p1 = dir, p2 = D->dir[0].path; *p1 && *p1 == *p2; p1++, p2++); + p = *(p1) && *(p1+1) ? p1+1 : NULL; + } else { + p = NULL; + } + if (p) { + /* if paths differ (i.e. not root of zip), construct relative path */ + newp = malloc(strlen(p) + 1 + strlen(name) + 1); + strcpy(newp, p); + newp[strlen(p)] = '/'; + strcpy(newp + strlen(p) + 1, name); + zzip_file = zzip_file_open(D->zzip_dir, newp, O_RDONLY | O_BINARY); + free(newp); + } else { + /* root of zip */ + zzip_file = zzip_file_open(D->zzip_dir, name, O_RDONLY | O_BINARY); + } + if (zzip_file /*&& zzip_file->method == 0*/) { + char *contents = malloc(zzip_file->usize + 1); + if (!zzip_file_read(zzip_file, contents, zzip_file->usize)) { + dreturn("%i", 0); + return 0; + } + contents[zzip_file->usize] = 0; + *fp = fmemopen(NULL, zzip_file->usize + 1, "r+"); + fwrite(contents, 1, zzip_file->usize + 1, *fp); + rewind(*fp); + free(contents); + zzip_file_close(zzip_file); + ret = 1; + } else { + ret = 0; + } + + dreturn("%i", ret); + return ret; +#else + dreturn("%i", 0); + return 0; +#endif +} /* vim: ts=2 sw=2 et tw=80 */ diff --git a/src/lzma.c b/src/lzma.c index b85d68e6..d18dda89 100644 --- a/src/lzma.c +++ b/src/lzma.c @@ -70,7 +70,7 @@ static struct gd_lzmadata *_GD_LzmaDoOpen(int dirfd, struct gd_raw_file_* file, dtrace("%i, %p, %i", dirfd, file, mode); if (mode & GD_FILE_READ) { - fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); + fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666); } else if (mode & GD_FILE_TEMP) { fd = _GD_MakeTempFile(file->D, dirfd, file->name); fdmode = "wb"; @@ -91,6 +91,8 @@ static struct gd_lzmadata *_GD_LzmaDoOpen(int dirfd, struct gd_raw_file_* file, return NULL; } + file->start_offset = ftello64(stream); + if ((lzd = (struct gd_lzmadata *)malloc(sizeof(struct gd_lzmadata))) == NULL) { fclose(stream); @@ -327,7 +329,7 @@ off64_t _GD_LzmaSeek(struct gd_raw_file_* file, off64_t count, dreturn("%i", 1); return 1; } - rewind(lzd->stream); + fseeko64(lzd->stream, file->start_offset, SEEK_SET); /* rewind */ lzd->input_eof = lzd->stream_end = 0; } diff --git a/src/open.c b/src/open.c index ab4b596f..959eccf3 100644 --- a/src/open.c +++ b/src/open.c @@ -210,18 +210,28 @@ static FILE *_GD_CreateDirfile(DIRFILE *restrict D, int dirfd, int dir_error, /* naively try to open the format file */ if (dirfd < 0) ; /* Directory error */ - else if ((fd = gd_OpenAt(D, dirfd, "format", O_RDONLY | O_BINARY, 0666)) < 0) + else if ((fd = gd_openat_wrapper(D, dirfd, "format", O_RDONLY | O_BINARY, 0666)) < 0) { - format_error = errno; + if (!(D->flags & GD_CREAT)) + D->zzip_dir = NULL; +#ifdef HAVE_ZZIP_LIB_H + if (!D->zzip_dir && !(D->flags & GD_CREAT)) + D->zzip_dir = zzip_dir_fdopen(dup(dirfd), 0); +#endif + if (D->zzip_dir) { + fd = open(D->name, O_RDONLY | O_BINARY); + } else { + format_error = errno; - /* in the non-POSIX case, this has already been done. */ + /* in the non-POSIX case, this has already been done. */ #ifndef GD_NO_DIR_OPEN - /* open failed, try to stat the directory itself */ - if (fstat(dirfd, &statbuf)) - dir_error = errno; - else if (!S_ISDIR(statbuf.st_mode)) - dir_error = ENOTDIR; + /* open failed, try to stat the directory itself */ + if (fstat(dirfd, &statbuf)) + dir_error = errno; + else if (!S_ISDIR(statbuf.st_mode)) + dir_error = ENOTDIR; #endif + } } else dir_error = 0; @@ -340,8 +350,12 @@ static FILE *_GD_CreateDirfile(DIRFILE *restrict D, int dirfd, int dir_error, D->flags = (D->flags & ~GD_ENCODING) | GD_UNENCODED; } - /* associate a stream with the format file */ - if ((fp = fdopen(fd, "rb")) == NULL) { + if (!D->zzip_dir) { + /* associate a stream with the format file */ + fp = fdopen(fd, "rb"); + } + + if ((D->zzip_dir && !gd_zip_read_file(D, dirfd, "format", &fp)) || fp == NULL) { char *format_file = malloc(strlen(dirfile) + 8); sprintf(format_file, "%s/format", dirfile); _GD_SetError(D, GD_E_CREAT, GD_E_CREAT_FORMAT, format_file, 0, NULL); @@ -367,6 +381,9 @@ static FILE *_GD_CreateDirfile(DIRFILE *restrict D, int dirfd, int dir_error, if (fstat(fd, &statbuf) == 0) *mtime = statbuf.st_mtime; + if (D->zzip_dir) + close(fd); + dreturn("%p", fp); return fp; } @@ -430,7 +447,10 @@ static DIRFILE *_GD_Open(DIRFILE *D, int dirfd, const char *filedir, if (gd_stat64(dirfile, &statbuf)) dirfd_error = errno; else if (!S_ISDIR(statbuf.st_mode)) - dirfd_error = ENOTDIR; +#ifdef HAVE_ZZIP_LIB_H + if (!zzip_dir_fdopen(dup(dirfd), 0)) +#endif + dirfd_error = ENOTDIR; else dirfd = 0; #else @@ -544,7 +564,10 @@ static DIRFILE *_GD_Open(DIRFILE *D, int dirfd, const char *filedir, return D; } - D->fragment[0].cname = _GD_CanonicalPath(dirfile, "format"); + if (D->zzip_dir) + D->fragment[0].cname = strdup(dirfile); + else + D->fragment[0].cname = _GD_CanonicalPath(dirfile, "format"); if (D->fragment[0].cname == NULL) { if (errno == ENOMEM) _GD_SetError(D, GD_E_ALLOC, 0, NULL, 0, NULL); diff --git a/src/raw.c b/src/raw.c index 677d3864..3e60f0f6 100644 --- a/src/raw.c +++ b/src/raw.c @@ -32,8 +32,8 @@ int _GD_RawOpen(int fd, struct gd_raw_file_* file, } else if (file->idata >= 0) close(file->idata); - file->idata = gd_OpenAt(file->D, fd, file->name, ((mode & GD_FILE_WRITE) ? - (O_RDWR | O_CREAT) : O_RDONLY) | O_BINARY, 0666); + file->idata = gd_openat_wrapper(file->D, fd, file->name, O_BINARY + | ((mode & GD_FILE_WRITE) ? (O_RDWR | O_CREAT) : O_RDONLY), 0666); } else { file->idata = _GD_MakeTempFile(file->D, fd, file->name); @@ -46,6 +46,7 @@ int _GD_RawOpen(int fd, struct gd_raw_file_* file, file->pos = 0; file->mode = mode | GD_FILE_READ; + file->start_offset = lseek64(file->idata, 0, SEEK_CUR); dreturn("%i", 0); return 0; @@ -64,7 +65,8 @@ off64_t _GD_RawSeek(struct gd_raw_file_* file, off64_t count, return count; } - pos = lseek64(file->idata, count * GD_SIZE(data_type), SEEK_SET); + pos = lseek64(file->idata, file->start_offset + count * GD_SIZE(data_type), + SEEK_SET) - file->start_offset; /* If we've landed in the middle of a sample, we have to back up */ if (pos > 0 && (pos % GD_SIZE(data_type))) @@ -153,10 +155,25 @@ off64_t _GD_RawSize(int dirfd, struct gd_raw_file_ *file, gd_type_t data_type, dtrace("%i, %p, 0x%X, ", dirfd, file, data_type); - if (gd_StatAt64(file->D, dirfd, file->name, &statbuf, 0) < 0) { - dreturn("%i", -1); - return -1; +#ifdef HAVE_ZZIP_LIB_H + if (file->D->zzip_dir) { + ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, 0); + if (zzip_file && zzip_file->method == 0) { + statbuf.st_size = zzip_file->csize; + zzip_file_close(zzip_file); + } else { + dreturn("%i", -1); + return -1; + } + } else { +#endif + if (gd_StatAt64(file->D, dirfd, file->name, &statbuf, 0) < 0) { + dreturn("%i", -1); + return -1; + } +#ifdef HAVE_ZZIP_LIB_H } +#endif dreturn("%" PRId64, (int64_t)statbuf.st_size); return statbuf.st_size / GD_SIZE(data_type); diff --git a/src/sie.c b/src/sie.c index 360189e2..e58d2012 100644 --- a/src/sie.c +++ b/src/sie.c @@ -66,7 +66,7 @@ struct gd_siedata { /* correct for byte sex */ #define FIXSEX(swap,v) ((swap) ? (int64_t)gd_swap64(v) : (v)) -static int _GD_SampIndDiscardHeader(FILE *stream) +static int _GD_SampIndDiscardHeader(FILE *stream, struct gd_raw_file_ *file) { int have = 0; unsigned char header[HEADSIZE]; @@ -96,7 +96,7 @@ static int _GD_SampIndDiscardHeader(FILE *stream) dreturn("%i", -1); return -1; } /* not a header; rewind so we can read it again */ - rewind(stream); + fseeko64(stream, file->start_offset, SEEK_SET); /* rewind */ } else { /* trailing data, unget it so it's available for the first read */ if (ungetc(c, stream) == EOF) { dreturn("%i", -1); @@ -105,7 +105,7 @@ static int _GD_SampIndDiscardHeader(FILE *stream) have = 1; } } else /* Bad header magic = no header: rewind */ - rewind(stream); + fseeko64(stream, file->start_offset, SEEK_SET); /* rewind */ dreturn("%i", have); return have; @@ -120,7 +120,7 @@ static int _GD_SampIndDoOpen(int fdin, struct gd_raw_file_ *file, dtrace("%i, %p, %i, 0x%X", fdin, file, swap, mode); if (!(mode & GD_FILE_TEMP)) { - fd = gd_OpenAt(file->D, fdin, file->name, ((mode & GD_FILE_WRITE) ? + fd = gd_openat_wrapper(file->D, fdin, file->name, ((mode & GD_FILE_WRITE) ? (O_RDWR | O_CREAT) : O_RDONLY) | O_BINARY, 0666); } else { fd = _GD_MakeTempFile(file->D, fdin, file->name); @@ -138,8 +138,10 @@ static int _GD_SampIndDoOpen(int fdin, struct gd_raw_file_ *file, return -1; } + file->start_offset = ftello64(stream); + if (!(mode & GD_FILE_WRITE)) { - f->header = _GD_SampIndDiscardHeader(stream); + f->header = _GD_SampIndDiscardHeader(stream, file); if (f->header < 0) { fclose(stream); dreturn("%i", -1); @@ -248,7 +250,7 @@ off64_t _GD_SampIndSeek(struct gd_raw_file_ *file, off64_t sample, if (sample < f->p) { /* seek backwards -- reading a file backwards doesn't necessarily work * that well. So, let's just rewind to the beginning and try again. */ - rewind(f->fp); + fseeko64(f->fp, file->start_offset, SEEK_SET); /* rewind */ /* Advance past header if necessary */ if (f->header) { @@ -440,6 +442,27 @@ static ssize_t _GD_GetNRec(struct gd_siedata *f, size_t size) dreturn("%" PRIdSIZE, (ssize_t)(statbuf.st_size / size)); return (ssize_t)(statbuf.st_size / size); } +static ssize_t _GD_GetNRec_zip(struct gd_raw_file_ *restrict file, size_t size) +{ + gd_stat64_t statbuf; + dtrace("%p, %" PRIuSIZE, file, size); + +#ifdef HAVE_ZZIP_LIB_H + ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, 0); + if (zzip_file && zzip_file->method == 0) { + statbuf.st_size = zzip_file->csize; + zzip_file_close(zzip_file); + } else { +#endif + dreturn("%i", -1); + return -1; +#ifdef HAVE_ZZIP_LIB_H + } + + dreturn("%" PRIdSIZE, (ssize_t)(statbuf.st_size / size)); + return (ssize_t)(statbuf.st_size / size); +#endif +} ssize_t _GD_SampIndWrite(struct gd_raw_file_ *restrict file, const void *restrict ptr, gd_type_t data_type, size_t nelem) @@ -458,7 +481,8 @@ ssize_t _GD_SampIndWrite(struct gd_raw_file_ *restrict file, const size_t size = sizeof(int64_t) + dlen; dtrace("%p, %p, 0x%03x, %" PRIuSIZE, file, ptr, data_type, nelem); - if ((nrec = _GD_GetNRec(f, size)) < 0) { + if ((nrec = file->D->zzip_dir ? _GD_GetNRec_zip(file, size) + : _GD_GetNRec(f, size)) < 0) { dreturn("%i", -1); return -1; } @@ -736,14 +760,31 @@ off64_t _GD_SampIndSize(int dirfd, struct gd_raw_file_* file, } /* find the last record */ - last_rec = _GD_GetNRec(&f, size) - 1; + last_rec = (file->D->zzip_dir ? _GD_GetNRec_zip(file, size) + : _GD_GetNRec(&f, size)) - 1; /* seek to this record */ - if (fseeko64(f.fp, last_rec * size, SEEK_SET)) { - fclose(f.fp); - dreturn("%i", -1); - return -1; +#ifdef HAVE_ZZIP_LIB_H + if (file->D->zzip_dir) { + ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, 0); + if (zzip_file && zzip_file->method == 0) { + fseeko64(f.fp, zzip_file->dataoffset + last_rec * size, SEEK_SET); + zzip_file_close(zzip_file); + } else { + fclose(f.fp); + dreturn("%i", -1); + return -1; + } + } else { +#endif + if (fseeko64(f.fp, last_rec * size, SEEK_SET)) { + fclose(f.fp); + dreturn("%i", -1); + return -1; + } +#ifdef HAVE_ZZIP_LIB_H } +#endif /* read the sample index */ if (fread(&n, sizeof(uint64_t), 1, f.fp) != 1) { diff --git a/src/slim.c b/src/slim.c index 049d89e7..9022d498 100644 --- a/src/slim.c +++ b/src/slim.c @@ -69,7 +69,15 @@ static struct gd_slimdata *GD_SLIM(DoOpen)(int dirfd, struct gd_raw_file_* file) } /* this is easily broken, but the best we can do in this case */ - gdsl->filepath = gd_MakeFullPathOnly(file->D, dirfd, file->name); + if (!file->D->zzip_dir) { + gdsl->filepath = gd_MakeFullPathOnly(file->D, dirfd, file->name); + } else { + /* this assumes the zip file has a ".zip" extension */ + gdsl->filepath = malloc(strlen(file->D->dir[0].path) - 4 + 1 + strlen(file->name) + 1); + strcpy(gdsl->filepath, file->D->dir[0].path); + gdsl->filepath[strlen(file->D->dir[0].path) - 4] = '/'; + strcpy(gdsl->filepath + strlen(file->D->dir[0].path) - 4 + 1, file->name); + } if (gdsl->filepath == NULL) { file->error = errno; free(gdsl); @@ -193,11 +201,22 @@ off64_t GD_SLIM(Size)(int dirfd, struct gd_raw_file_ *file, gd_type_t data_type, #else { /* this is easily broken, but the best we can do in this case */ - char *filepath = gd_MakeFullPathOnly(file->D, dirfd, file->name); + char *filepath; + if (!file->D->zzip_dir) { + filepath = gd_MakeFullPathOnly(file->D, dirfd, file->name); + } else { + /* this assumes the zip file has a ".zip" extension */ + filepath = malloc(strlen(file->D->dir[0].path) - 4 + 1 + strlen(file->name) + 1); + strcpy(filepath, file->D->dir[0].path); + filepath[strlen(file->D->dir[0].path) - 4] = '/'; + strcpy(filepath + strlen(file->D->dir[0].path) - 4 + 1, file->name); + } + if (filepath == NULL) { dreturn("%i", -1); return -1; } + dtrace("\"%s\"", filepath); size = slimrawsize(filepath); free(filepath); diff --git a/test/Makefile.am b/test/Makefile.am index 02ea9fd5..5062badb 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -29,7 +29,7 @@ EXTRA_DIST=test.h enc_add.c enc_complex64.c enc_complex128.c enc_del.c \ enc_enoent.c enc_float32.c enc_float64.c enc_get_cont.c enc_int8.c \ enc_int16.c enc_int32.c enc_int64.c enc_move_from.c enc_nframes.c \ enc_put_offs.c enc_seek.c enc_uint8.c enc_uint16.c enc_uint32.c \ - enc_uint64.c + enc_uint64.c enc_int32_zip.c ADD_TESTS=add_add add_affix add_alias add_alias_affix add_alias_index \ add_alias_meta add_alias_name add_alias_ns add_alias_prot \ @@ -100,7 +100,7 @@ ASCII_TESTS=ascii_add ascii_complex64 ascii_complex128 ascii_get \ ascii_get_here ascii_get_sub ascii_int8 ascii_int16 ascii_int32 \ ascii_int64 ascii_nframes ascii_put ascii_put_here ascii_seek \ ascii_seek_far ascii_sync ascii_uint8 ascii_uint16 ascii_uint32 \ - ascii_uint64 + ascii_uint64 ascii_int32_zip BOF_TESTS=bof bof_bit bof_bit_code bof_code bof_const bof_index bof_lincom \ bof_lincom_code bof_phase bof_phase_code bof_phase_neg bof_recurse @@ -111,7 +111,7 @@ BZIP_TESTS=bzip_add bzip_complex64 bzip_complex128 bzip_del bzip_enoent \ bzip_int32 bzip_int64 bzip_move_from bzip_move_to bzip_nframes \ bzip_put bzip_put_back bzip_put_endian bzip_put_get bzip_put_offs \ bzip_put_pad bzip_put_sub bzip_seek bzip_seek_far bzip_sync \ - bzip_uint8 bzip_uint16 bzip_uint32 bzip_uint64 + bzip_uint8 bzip_uint16 bzip_uint32 bzip_uint64 bzip_int32_zip CALIST_TESTS=calist calist0 calist_free calist_hidden calist_long calist_meta \ calist_meta0 calist_meta_free calist_meta_hidden calist_meta_meta \ @@ -219,7 +219,7 @@ FLAC_TESTS=flac_add flac_complex64 flac_complex128 flac_del flac_enoent \ flac_put_big flac_put_complex128 flac_put_float64 \ flac_put_int32 flac_put_little flac_put_offs flac_seek \ flac_seek_far flac_sync flac_uint8 flac_uint16 flac_uint32 \ - flac_uint64 + flac_uint64 flac_int32_zip FLIST_TESTS=flist flist0 flist2 flist_hidden flist_invalid flist_meta \ flist_meta2 flist_meta_hidden flist_meta_invalid flist_type \ @@ -284,7 +284,8 @@ GET_TESTS=get64 get_affix get_bad_code get_bit get_carray get_carray_bad \ get_type get_uint16 get_uint32 get_uint64 get_window get_window_clr \ get_window_complex get_window_ge get_window_gt get_window_le \ get_window_lt get_window_ne get_window_s get_window_set get_zero \ - get_zero_complex get_zero_float + get_zero_complex get_zero_float get_int32_zip get_int32_zip2 \ + get_linterp_zip get_linterp_zip2 GLOBAL_TESTS=global_flags global_name @@ -295,7 +296,7 @@ GZIP_TESTS=gzip_add gzip_complex64 gzip_complex128 gzip_del gzip_enoent \ gzip_put gzip_put_back gzip_put_endian gzip_put_get \ gzip_put_nframes gzip_put_off gzip_put_offs gzip_put_pad \ gzip_put_sub gzip_seek gzip_seek_far gzip_seek_put gzip_sync \ - gzip_uint8 gzip_uint16 gzip_uint32 gzip_uint64 + gzip_uint8 gzip_uint16 gzip_uint32 gzip_uint64 gzip_int32_zip HEADER_TESTS=header_complex header_off64t @@ -331,7 +332,7 @@ LZMA_TESTS=lzma_enoent lzma_get lzma_nframes lzma_put lzma_xz_add \ lzma_xz_put_back lzma_xz_put_endian lzma_xz_put_get \ lzma_xz_put_offs lzma_xz_put_pad lzma_xz_seek lzma_xz_seek_far \ lzma_xz_sync lzma_xz_uint8 lzma_xz_uint16 lzma_xz_uint32 \ - lzma_xz_uint64 + lzma_xz_uint64 lzma_xz_int32_zip MADD_TESTS=madd madd_affix madd_alias madd_aliasmeta madd_alias_affix \ madd_alias_alias madd_alias_subsub madd_bit madd_bit_invalid \ @@ -384,7 +385,7 @@ NFIELDS_TESTS=nfields_hidden nfields_invalid nfields_nfields nfields_type \ nfields_vector_hidden nfields_vector_invalid NFRAMES_TESTS=nframes64 nframes_empty nframes_invalid nframes_nframes \ - nframes_off64 nframes_spf + nframes_off64 nframes_spf nframes_nframes_zip NMETA_TESTS=nmeta nmeta_hidden nmeta_invalid nmeta_parent nmeta_type \ nmeta_type_hidden nmeta_type_invalid nmeta_type_parent \ @@ -511,7 +512,7 @@ SIE_TESTS=sie_err_open sie_get_big sie_get_header sie_get_little sie_move_from \ sie_put_append2 sie_put_back sie_put_big sie_put_header \ sie_put_little sie_put_many sie_put_newo sie_put_newo0 sie_put_pad \ sie_put_pad0 sie_put_trunc sie_put_trunc2 sie_put_trunc_nf sie_seek \ - sie_seek_far sie_sync + sie_seek_far sie_sync sie_get_little_zip SLIM_TESTS=slim_get slim_nframes slim_seek slim_seek_far diff --git a/test/alloc_callback.c b/test/alloc_callback.c index a52a679e..b6f9708d 100644 --- a/test/alloc_callback.c +++ b/test/alloc_callback.c @@ -23,7 +23,7 @@ void *cb_ptr = NULL; int good_ptr = 0; -void free_func(void *ptr) +void free_function(void *ptr) { if (ptr == cb_ptr) good_ptr = 1; @@ -46,7 +46,7 @@ int main(void) rmdirfile(); mkdir(filedir, 0700); - gd_alloc_funcs(NULL, free_func); + gd_alloc_funcs(NULL, free_function); MAKEFORMATFILE(format, "Syntax error\n"); diff --git a/test/alloc_clear.c b/test/alloc_clear.c index 832f2867..f29d1021 100644 --- a/test/alloc_clear.c +++ b/test/alloc_clear.c @@ -37,7 +37,7 @@ static void *malloc_func(size_t len) return malloc_ptr[malloc_count++]; } -static void free_func(void *ptr) +static void free_function(void *ptr) { ptr = ptr; free_count++; @@ -65,7 +65,7 @@ int main(void) "window WINDOW dolor magna EQ aliqua.\n" ); - gd_alloc_funcs(malloc_func, free_func); + gd_alloc_funcs(malloc_func, free_function); D = gd_open(filedir, GD_RDONLY | GD_VERBOSE); diff --git a/test/alloc_entry.c b/test/alloc_entry.c index 35ab0c5b..50d5851d 100644 --- a/test/alloc_entry.c +++ b/test/alloc_entry.c @@ -37,7 +37,7 @@ static void *malloc_func(size_t len) return malloc_ptr[malloc_count++]; } -static void free_func(void *ptr) +static void free_function(void *ptr) { ptr = ptr; free_count++; @@ -71,7 +71,7 @@ int main(void) "mplex MPLEX Ut enim as minim\n" ); - gd_alloc_funcs(malloc_func, free_func); + gd_alloc_funcs(malloc_func, free_function); D = gd_open(filedir, GD_RDONLY | GD_VERBOSE); diff --git a/test/ascii_int32_zip.c b/test/ascii_int32_zip.c new file mode 100644 index 00000000..95904920 --- /dev/null +++ b/test/ascii_int32_zip.c @@ -0,0 +1,27 @@ +/* Copyright (C) 2016 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +#define ENC_SUFFIX ".txt" +#define GD_ENC_ENCODED GD_TEXT_ENCODED + +#include "enc_int32_zip.c" diff --git a/test/bzip_int32_zip.c b/test/bzip_int32_zip.c new file mode 100644 index 00000000..8909a114 --- /dev/null +++ b/test/bzip_int32_zip.c @@ -0,0 +1,31 @@ +/* Copyright (C) 2016 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +#if !defined TEST_BZIP2 || !defined USE_BZIP2 +#define ENC_SKIP_TEST 1 +#endif + +#define ENC_SUFFIX ".bz2" +#define GD_ENC_ENCODED GD_BZIP2_ENCODED + +#include "enc_int32_zip.c" diff --git a/test/enc_int32_zip.c b/test/enc_int32_zip.c new file mode 100644 index 00000000..09b6ae60 --- /dev/null +++ b/test/enc_int32_zip.c @@ -0,0 +1,80 @@ +/* Copyright (C) 2016, 2017 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H +#ifdef ENC_SKIP_TEST + return 77; +#else + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data" ENC_SUFFIX; + const char *command = "zip -jq0 dirfile.zip dirfile/format dirfile/data" ENC_SUFFIX; + int e1, e2, e3, r = 0; + DIRFILE *D1, *D2; + const int32_t data_in[8] = { -5, 6, -10, 100, 0, 1, -100, 3 }; + int32_t data_out[8]; + int i = 0; + + rmdirfile(); + unlink(filedirzip); + + D1 = gd_open(filedir, GD_RDWR | GD_CREAT | GD_EXCL | GD_ENC_ENCODED + | GD_VERBOSE); + + e1 = gd_add_spec(D1, "data RAW INT32 1", 0); + CHECKI(e1, 0); + + e2 = gd_putdata(D1, "data", 0, 0, 0, 8, GD_INT32, data_in); + CHECKI(e2, 8); + + gd_close(D1); + + if (gd_system(command)) + return 1; + + unlink(format); + unlink(data); + rmdir(filedir); + + D2 = gd_open(filedirzip, GD_RDONLY | GD_VERBOSE); + + e3 = gd_getdata(D2, "data", 0, 0, 0, 8, GD_INT32, data_out); + CHECKI(e3, 8); + + if (e3 > 8) + e3 = 8; + for (i = 0; i < e3; ++i) + CHECKIi(i, data_out[i], data_in[i]); + + gd_discard(D2); + unlink(filedirzip); + + return r; +#endif +#else + return 77; +#endif +} diff --git a/test/flac_int32_zip.c b/test/flac_int32_zip.c new file mode 100644 index 00000000..a5b96676 --- /dev/null +++ b/test/flac_int32_zip.c @@ -0,0 +1,31 @@ +/* Copyright (C) 2016 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +#if !defined TEST_FLAC || !defined USE_FLAC +#define ENC_SKIP_TEST 1 +#endif + +#define ENC_SUFFIX ".flac" +#define GD_ENC_ENCODED GD_FLAC_ENCODED + +#include "enc_int32_zip.c" diff --git a/test/get_int32_zip.c b/test/get_int32_zip.c new file mode 100644 index 00000000..3d62d3a2 --- /dev/null +++ b/test/get_int32_zip.c @@ -0,0 +1,68 @@ +/* Copyright (C) 2008-2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data"; + const char *command = "zip -jq0 dirfile.zip dirfile/format dirfile/data"; + int32_t c[8]; + int i, n, error, r = 0; + DIRFILE *D; + + memset(c, 0, 8 * sizeof(*c)); + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "/ENCODING none\ndata RAW INT32 8\n"); + MAKEDATAFILE(data, int32_t, i * (0x02000001) * (2 * (i % 2) - 1), 64); + + if (gd_system(command)) + return 1; + + D = gd_open(filedirzip, GD_RDONLY | GD_VERBOSE); + n = gd_getdata(D, "data", 5, 0, 1, 0, GD_INT32, c); + + error = gd_error(D); + CHECKI(error, 0); + CHECKI(n, 8); + + for (i = 0; i < 8; ++i) + CHECKIi(i, c[i], (0x50000028 + i * 0x02000001) * (2 * (i % 2) - 1)); + + gd_discard(D); + + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + return r; +#else + return 77; +#endif +} diff --git a/test/get_int32_zip2.c b/test/get_int32_zip2.c new file mode 100644 index 00000000..57166ff9 --- /dev/null +++ b/test/get_int32_zip2.c @@ -0,0 +1,69 @@ +/* Copyright (C) 2008-2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data"; + /* DEFLATE compress format file but STORE data */ + const char *command = "zip -jqn data dirfile.zip dirfile/format dirfile/data"; + int32_t c[8]; + int i, n, error, r = 0; + DIRFILE *D; + + memset(c, 0, 8 * sizeof(*c)); + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "/ENCODING none\ndata RAW INT32 8\n"); + MAKEDATAFILE(data, int32_t, i * (0x02000001) * (2 * (i % 2) - 1), 64); + + if (gd_system(command)) + return 1; + + D = gd_open(filedirzip, GD_RDONLY | GD_VERBOSE); + n = gd_getdata(D, "data", 5, 0, 1, 0, GD_INT32, c); + + error = gd_error(D); + CHECKI(error, 0); + CHECKI(n, 8); + + for (i = 0; i < 8; ++i) + CHECKIi(i, c[i], (0x50000028 + i * 0x02000001) * (2 * (i % 2) - 1)); + + gd_discard(D); + + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + return r; +#else + return 77; +#endif +} diff --git a/test/get_linterp_zip.c b/test/get_linterp_zip.c new file mode 100644 index 00000000..d1da86a9 --- /dev/null +++ b/test/get_linterp_zip.c @@ -0,0 +1,74 @@ +/* Copyright (C) 2008-2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +/* Attempt to read LINTERP */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data"; + const char *table = "dirfile/table"; + const char *command = "zip -jq0 dirfile.zip dirfile/format dirfile/data dirfile/table"; + unsigned char c = 0; + int i, n, error, r = 0; + DIRFILE *D; + FILE *t; + + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "/ENCODING none\nlinterp LINTERP data table\ndata RAW UINT8 1\n"); + MAKEDATAFILE(data, unsigned char, i, 64); + + t = fopen(table, "wt"); + for (i = 0; i < 2 * GD_LUT_CHUNK; ++i) + fprintf(t, "%i %i\n", i * 6, i * 12); + fclose(t); + + if (gd_system(command)) + return 1; + + D = gd_open(filedirzip, GD_RDONLY | GD_VERBOSE); + n = gd_getdata(D, "linterp", 5, 0, 1, 0, GD_UINT8, &c); + error = gd_error(D); + + gd_discard(D); + + unlink(table); + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + CHECKI(error, 0); + CHECKI(n, 1); + CHECKU(c, 10); + + return r; +#else + return 77; +#endif +} diff --git a/test/get_linterp_zip2.c b/test/get_linterp_zip2.c new file mode 100644 index 00000000..f180ec6c --- /dev/null +++ b/test/get_linterp_zip2.c @@ -0,0 +1,75 @@ +/* Copyright (C) 2008-2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +/* Attempt to read LINTERP */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data"; + const char *table = "dirfile/table"; + /* DEFLATE compress table but STORE data and format file*/ + const char *command = "zip -jqn table dirfile.zip dirfile/format dirfile/data dirfile/table"; + unsigned char c = 0; + int i, n, error, r = 0; + DIRFILE *D; + FILE *t; + + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "/ENCODING none\nlinterp LINTERP data table\ndata RAW UINT8 1\n"); + MAKEDATAFILE(data, unsigned char, i, 64); + + t = fopen(table, "wt"); + for (i = 0; i < 2 * GD_LUT_CHUNK; ++i) + fprintf(t, "%i %i\n", i * 6, i * 12); + fclose(t); + + if (gd_system(command)) + return 1; + + D = gd_open(filedirzip, GD_RDONLY | GD_VERBOSE); + n = gd_getdata(D, "linterp", 5, 0, 1, 0, GD_UINT8, &c); + error = gd_error(D); + + gd_discard(D); + + unlink(table); + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + CHECKI(error, 0); + CHECKI(n, 1); + CHECKU(c, 10); + + return r; +#else + return 77; +#endif +} diff --git a/test/gzip_int32_zip.c b/test/gzip_int32_zip.c new file mode 100644 index 00000000..a3a3a741 --- /dev/null +++ b/test/gzip_int32_zip.c @@ -0,0 +1,31 @@ +/* Copyright (C) 2016 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +#if !defined TEST_GZIP || !defined USE_GZIP +#define ENC_SKIP_TEST 1 +#endif + +#define ENC_SUFFIX ".gz" +#define GD_ENC_ENCODED GD_GZIP_ENCODED + +#include "enc_int32_zip.c" diff --git a/test/lzma_xz_int32_zip.c b/test/lzma_xz_int32_zip.c new file mode 100644 index 00000000..08befec4 --- /dev/null +++ b/test/lzma_xz_int32_zip.c @@ -0,0 +1,31 @@ +/* Copyright (C) 2016 D. V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "test.h" + +#if !defined TEST_LZMA || !defined USE_LZMA +#define ENC_SKIP_TEST 1 +#endif + +#define ENC_SUFFIX ".xz" +#define GD_ENC_ENCODED GD_LZMA_ENCODED + +#include "enc_int32_zip.c" diff --git a/test/nframes_nframes_zip.c b/test/nframes_nframes_zip.c new file mode 100644 index 00000000..443c58a7 --- /dev/null +++ b/test/nframes_nframes_zip.c @@ -0,0 +1,68 @@ +/* Copyright (C) 2008-2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +/* Retreiving the number of frames should succeed cleanly */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data"; + const char *command = "zip -jq0 dirfile.zip dirfile/format dirfile/data"; + int fd, error, r = 0; + const size_t len = strlen(data); + off_t n; + DIRFILE *D; + + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "/ENCODING none\ndata RAW UINT16 1\n"); + + fd = open(data, O_CREAT | O_EXCL | O_WRONLY | O_BINARY, 0666); + write(fd, data, len); + close(fd); + + if (gd_system(command)) + return 1; + + D = gd_open(filedir, GD_RDONLY | GD_VERBOSE); + n = gd_nframes(D); + error = gd_error(D); + gd_discard(D); + + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + CHECKI(error, 0); + CHECKI(n, (off_t)len / 2); + + return r; +#else + return 77; +#endif +} diff --git a/test/sie_get_little_zip.c b/test/sie_get_little_zip.c new file mode 100644 index 00000000..71667343 --- /dev/null +++ b/test/sie_get_little_zip.c @@ -0,0 +1,75 @@ +/* Copyright (C) 2011, 2013, 2017 D.V. Wiebe + * Copyright (C) 2019 Matthew Petroff + * + *************************************************************************** + * + * This file is part of the GetData project. + * + * GetData is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the + * Free Software Foundation; either version 2.1 of the License, or (at your + * option) any later version. + * + * GetData is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with GetData; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +/* Attempt to read little-endian SIE data */ +#include "test.h" + +int main(void) +{ +#ifdef HAVE_ZZIP_LIB_H + const char *filedir = "dirfile"; + const char *filedirzip = "dirfile.zip"; + const char *format = "dirfile/format"; + const char *data = "dirfile/data.sie"; + const char *command = "zip -jq0 dirfile.zip dirfile/format dirfile/data.sie"; + unsigned char c[16]; + const uint8_t data_data[] = { + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x22, + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x32 + }; + DIRFILE *D; + int fd, i, n, error, r = 0; + + rmdirfile(); + unlink(filedirzip); + mkdir(filedir, 0700); + + MAKEFORMATFILE(format, "data RAW UINT8 8\n/ENCODING sie\n/ENDIAN little\n"); + + fd = open(data, O_CREAT | O_EXCL | O_WRONLY | O_BINARY, 0666); + write(fd, data_data, 3 * 9 * sizeof(unsigned char)); + close(fd); + + if (gd_system(command)) + return 1; + + D = gd_open(filedir, GD_RDONLY | GD_VERBOSE); + n = gd_getdata(D, "data", 3, 0, 2, 0, GD_UINT8, c); + error = gd_error(D); + + gd_discard(D); + + unlink(data); + unlink(format); + rmdir(filedir); + unlink(filedirzip); + + CHECKI(error, 0); + CHECKI(n, 16); + for (i = 0; i < 16; ++i) + CHECKIi(i,c[i], (i <= 8) ? 0x22 : 0x32); + + return r; +#else + return 77; +#endif +}