Add read-only support for zipped Dirfiles

.gitignore

@@ -309,6 +309,7 @@ test/ascii_get_here
 test/ascii_get_sub
 test/ascii_int16
 test/ascii_int32
+test/ascii_int32_zip
 test/ascii_int64
 test/ascii_int8
 test/ascii_nframes
@@ -348,6 +349,7 @@ test/bzip_get_get2
 test/bzip_get_put
 test/bzip_int16
 test/bzip_int32
+test/bzip_int32_zip
 test/bzip_int64
 test/bzip_int8
 test/bzip_move_from
@@ -642,6 +644,7 @@ test/flac_get_little
 test/flac_get_long
 test/flac_int16
 test/flac_int32
+test/flac_int32_zip
 test/flac_int64
 test/flac_int8
 test/flac_move_from
@@ -794,6 +797,8 @@ test/get_indir
 test/get_indir_typein
 test/get_int16
 test/get_int32
+test/get_int32_zip
+test/get_int32_zip2
 test/get_int64
 test/get_int8
 test/get_invalid
@@ -817,6 +822,8 @@ test/get_linterp_nodir
 test/get_linterp_noin
 test/get_linterp_notab
 test/get_linterp_sort
+test/get_linterp_zip
+test/get_linterp_zip2
 test/get_mplex
 test/get_mplex_bof
 test/get_mplex_complex
@@ -902,6 +909,7 @@ test/gzip_get_get2
 test/gzip_get_put
 test/gzip_int16
 test/gzip_int32
+test/gzip_int32_zip
 test/gzip_int64
 test/gzip_int8
 test/gzip_move_from
@@ -993,6 +1001,7 @@ test/lzma_xz_get_get2
 test/lzma_xz_get_put
 test/lzma_xz_int16
 test/lzma_xz_int32
+test/lzma_xz_int32_zip
 test/lzma_xz_int64
 test/lzma_xz_int8
 test/lzma_xz_move_from
@@ -1186,6 +1195,7 @@ test/nframes64
 test/nframes_empty
 test/nframes_invalid
 test/nframes_nframes
+test/nframes_nframes_zip
 test/nframes_off64
 test/nframes_spf
 test/nmeta
@@ -1607,6 +1617,7 @@ test/sie_err_open
 test/sie_get_big
 test/sie_get_header
 test/sie_get_little
+test/sie_get_little_zip
 test/sie_move_from
 test/sie_move_to
 test/sie_nframes_big

src/Makefile.am

@@ -80,6 +80,9 @@ MODULE_LIBS=libgetdata.la
 if USE_MODULES
 EXPORT_DYNAMIC=-export-dynamic
 DGETDATA_MODULEDIR=-DGETDATA_MODULEDIR="\"$(moduledir)\""
+EXTERNAL_CPPFLAGS=$(ZZIP_CPPFLAGS)
+EXTERNAL_LDFLAGS=$(ZZIP_LDFLAGS)
+EXTERNAL_LIBS=$(ZZIP_LIBS)
 else
 EXTERNAL_CPPFLAGS=$(SLIM_CPPFLAGS) $(GZIP_CPPFLAGS) $(BZIP2_CPPFLAGS) \
 									$(LZMA_CPPFLAGS) $(ZZIP_CPPFLAGS) $(FLAC_CPPFLAGS)

src/ascii.c

@@ -28,7 +28,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_,
   dtrace("%i, %p, <unused>, <unused>, %u", fd, file, mode);
 
   if (!(mode & GD_FILE_TEMP))
-    file->idata = gd_OpenAt(file->D, fd, file->name, ((mode & GD_FILE_WRITE)
+    file->idata = gd_openat_wrapper(file->D, fd, file->name, ((mode & GD_FILE_WRITE)
           ? (O_RDWR | O_CREAT) : O_RDONLY) | O_BINARY, 0666);
   else
     file->idata = _GD_MakeTempFile(file->D, fd, file->name);
@@ -49,6 +49,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_,
 
   file->mode = mode | GD_FILE_READ;
   file->pos = 0;
+  file->start_offset = ftello64(file->edata);
   dreturn("%i", 0);
   return 0;
 }
@@ -61,7 +62,7 @@ off64_t _GD_AsciiSeek(struct gd_raw_file_* file, off64_t count,
   dtrace("%p, %" PRId64 ", <unused>, 0x%X", file, (int64_t)count, mode);
 
   if (count < file->pos) {
-    rewind((FILE *)file->edata);
+    fseeko64((FILE *)file->edata, file->start_offset, SEEK_SET); /* rewind */
     file->pos = 0;
   }
 
@@ -285,7 +286,7 @@ off64_t _GD_AsciiSize(int dirfd, struct gd_raw_file_* file,
 
   dtrace("%i, %p, <unused>, <unused>", dirfd, file);
 
-  fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY, 0666);
+  fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY, 0666);
   if (fd < 0) {
     dreturn("%i", -1);
     return -1;

src/bzip.c

@@ -56,7 +56,7 @@ static struct gd_bzdata *_GD_Bzip2DoOpen(int dirfd, struct gd_raw_file_* file,
   file->error = BZ_IO_ERROR;
 
   if (mode & GD_FILE_READ) {
-    fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+    fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   } else if (mode & GD_FILE_TEMP) {
     fd = _GD_MakeTempFile(file->D, dirfd, file->name);
     fdmode = "wb";

src/close.c

@@ -76,6 +76,11 @@ static void _GD_FreeD(DIRFILE *D, int keep_dirfile)
   }
   free(D->dir);
 
+#ifdef HAVE_ZZIP_LIB_H
+  if (D->zzip_dir)
+    zzip_dir_close(D->zzip_dir);
+#endif
+
   if (!keep_dirfile)
     free(D);
 

src/common.c

@@ -346,7 +346,7 @@ static int lutcmp(const void* a, const void* b)
 */
 int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E)
 {
-  FILE *fp;
+  FILE *fp = NULL;
   struct gd_lut_ *ptr;
   int i, fd;
   int dir = -1;
@@ -364,15 +364,23 @@ int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E)
       return 1;
     }
 
-  fd = gd_OpenAt(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file,
-      O_RDONLY, 0666);
-  if (fd == -1) {
-    _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
-    dreturn("%i", 1);
-    return 1;
+  if (!D->zzip_dir) {
+    fd = gd_openat_wrapper(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file,
+        O_RDONLY, 0666);
+    if (fd == -1) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
+      dreturn("%i", 1);
+      return 1;
+    }
+    fp = fdopen(fd, "rb");
+  } else {
+    if (!gd_zip_read_file(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file, &fp)) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
+      dreturn("%i", 1);
+      return 1;
+    }
   }
 
-  fp = fdopen(fd, "rb");
   if (fp == NULL) {
     _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
     dreturn("%i", 1);
@@ -1430,9 +1438,13 @@ int _GD_GrabDir(DIRFILE *D, int dirfd, const char *name, int canonical)
   free(path);
 
   if (D->dir[D->ndir].fd == -1) {
-    free(D->dir[D->ndir].path);
-    dreturn("%i", -1);
-    return -1;
+    if (D->zzip_dir) {
+      D->dir[D->ndir].fd = dup(dirfd);
+    } else {
+      free(D->dir[D->ndir].path);
+      dreturn("%i", -1);
+      return -1;
+    }
   }
 #endif
   D->ndir++;

src/flac.c

@@ -151,7 +151,7 @@ static struct gd_flacdata *_GD_FlacDoOpen(int dirfd, struct gd_raw_file_* file,
   dtrace("%i, %p, 0x%X, %i, 0x%X", dirfd, file, data_type, swap, mode);
 
   if (mode & GD_FILE_READ) {
-    fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+    fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   } else if (mode & GD_FILE_TEMP) {
     fd = _GD_MakeTempFile(file->D, dirfd, file->name);
     fdmode = "wb";

src/gzip.c

@@ -46,7 +46,7 @@ int _GD_GzipOpen(int fd, struct gd_raw_file_* file,
   dtrace("%i, %p, <unused>, <unused>, 0x%X", fd, file, mode);
 
   if (mode & GD_FILE_READ) {
-    file->idata = gd_OpenAt(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666);
+    file->idata = gd_openat_wrapper(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666);
     gzmode = "r";
   } else if (mode & GD_FILE_TEMP) {
     file->idata = _GD_MakeTempFile(file->D, fd, file->name);
@@ -185,17 +185,32 @@ off64_t _GD_GzipSize(int dirfd, struct gd_raw_file_ *file, gd_type_t data_type,
 
   dtrace("%i, %p, 0x%X, <unused>", dirfd, file, data_type);
 
-  fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+  fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   if (fd < 0) {
     dreturn("%i", -1);
     return -1;
   }
 
   /* seek to the end */
-  if (lseek64(fd, -4, SEEK_END) == -1) {
-    dreturn("%i", -1);
-    return -1;
+#ifdef HAVE_ZZIP_LIB_H
+  if (file->D->zzip_dir) {
+    ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, O_RDONLY | O_BINARY);
+    if (zzip_file && zzip_file->method == 0) {
+      lseek64(dirfd, zzip_file->dataoffset + zzip_file->csize - 4, SEEK_SET);
+      zzip_file_close(zzip_file);
+    } else {
+      dreturn("%i", -1);
+      return -1;
+    }
+  } else {
+#endif
+    if (lseek64(fd, -4, SEEK_END) == -1) {
+      dreturn("%i", -1);
+      return -1;
+    }
+#ifdef HAVE_ZZIP_LIB_H
   }
+#endif
   if (read(fd, &size, 4) < 4) {
     dreturn("%i", -1);
     return -1;

src/include.c

@@ -253,14 +253,14 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename,
 
   /* Open the containing directory */
   dirfd = _GD_GrabDir(D, D->fragment[parent].dirfd, temp_buf1, 1);
-  if (dirfd == -1 && D->error == GD_E_OK)
+  if (dirfd == -1 && D->error == GD_E_OK && !D->zzip_dir)
     _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, ename);
   if (D->error)
     goto include_error;
 
   /* Reject weird stuff */
   if (gd_StatAt(D, dirfd, base, &statbuf, 0)) {
-    if (!(p->flags & GD_CREAT)) {
+    if (!(p->flags & GD_CREAT) && !D->zzip_dir) {
       _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
       _GD_ReleaseDir(D, dirfd);
       goto include_error;
@@ -280,19 +280,33 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename,
   }
 
   /* Try to open the file */
-  i = gd_OpenAt(D, dirfd, base,
-      ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) |
-      ((p->flags & GD_CREAT) ? O_CREAT : 0) |
-      ((p->flags & GD_TRUNC) ? O_TRUNC : 0) |
-      ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666);
+  if (D->zzip_dir) {
+    i = open(D->name, O_RDONLY | O_BINARY);
+  } else {
+    i = gd_OpenAt(D, dirfd, base,
+        ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) |
+        ((p->flags & GD_CREAT) ? O_CREAT : 0) |
+        ((p->flags & GD_TRUNC) ? O_TRUNC : 0) |
+        ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666);
+  }
 
   if (i < 0) {
     _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
     _GD_ReleaseDir(D, dirfd);
     goto include_error;
   }
 
-  new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb");
+  if (!D->zzip_dir) {
+    new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb");
+  } else {
+    if (!gd_zip_read_file(D, dirfd, base, &new_fp)) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
+      _GD_ReleaseDir(D, dirfd);
+      close(i);
+      goto include_error;
+    }
+    close(i);
+  }
 
   /* If opening the file failed, set the error code and abort parsing. */
   if (new_fp == NULL) {

src/internal.h

@@ -113,6 +113,10 @@
 #include <regex.h>
 #endif
 
+#ifdef HAVE_ZZIP_LIB_H
+#include <zzip/lib.h>
+#include <zzip/file.h>
+#endif
 
 /* MSCVRT defines size_t but not ssize_t */
 #ifdef __MSVCRT__
@@ -994,6 +998,7 @@ struct gd_raw_file_ {
   DIRFILE *D;
   unsigned int mode;
   off64_t pos;
+  off64_t start_offset;
 };
 
 /* linterp table datum */
@@ -1283,6 +1288,13 @@ struct gd_dirfile_ {
   /* syntax error callback */
   gd_parser_callback_t sehandler;
   void* sehandler_extra;
+
+  /* for zipped Dirfile support */
+#ifdef HAVE_ZZIP_LIB_H
+  ZZIP_DIR *zzip_dir;
+#else
+  void *zzip_dir;
+#endif
 };
 
 /* The caller's preferred memory manager */
@@ -1434,6 +1446,9 @@ gd_entry_t *_GD_ParseFieldSpec(DIRFILE *restrict,
 char *_GD_ParseFragment(FILE *restrict, DIRFILE*, struct parser_state *restrict,
     int, int);
 void _GD_PerformRename(DIRFILE *restrict, struct gd_rename_data_ *restrict);
+int gd_openat_wrapper(const DIRFILE *D, int dirfd, const char *name, int flags,
+    mode_t mode);
+int gd_zip_read_file(const DIRFILE *D, int dirfd, const char *name, FILE **fp);
 struct gd_rename_data_ *_GD_PrepareRename(DIRFILE *restrict, char *restrict,
     size_t, gd_entry_t *restrict, int, unsigned);
 int _GD_ReadLinterpFile(DIRFILE *restrict, gd_entry_t *restrict);