Remove grib_file from file_pool if no more grib_index holds it

ecmwf · Oct 30, 2024 · 19bacb5 · 19bacb5
1 parent 3769fd7
commit 19bacb5
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 18 deletions.
diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h
@@ -482,6 +482,8 @@ void grib_file_pool_clean(void);
 grib_file* grib_file_pool_get_files(void);
 grib_file* grib_file_open(const char* filename, const char* mode, int* err);
 void grib_file_pool_delete_file(grib_file* file);
+grib_file* grib_file_pool_create_clone(grib_context* c, short clone_id, grib_file* file);
+void grib_file_pool_delete_clone(grib_file* clone);
 void grib_file_close(const char* filename, int force, int* err);
 void grib_file_close_all(int* err);
 grib_file* grib_get_file(const char* filename, int* err);

diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h
@@ -918,6 +918,8 @@ struct grib_file
     char* buffer;
     long refcount;
     grib_file* next;
+    grib_file* pool_file;
+    long pool_file_refcount;
     short id;
 };
 

diff --git a/src/grib_filepool.cc b/src/grib_filepool.cc
@@ -254,6 +254,51 @@ grib_file* grib_file_open(const char* filename, const char* mode, int* err)
     return file;
 }
 
+grib_file* grib_file_pool_create_clone(grib_context* c, short clone_id, grib_file* pool_file)
+{
+    if(pool_file)
+    {
+        grib_file* newfile          = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
+        newfile->id                 = clone_id;
+        newfile->name               = strdup(pool_file->name);
+        newfile->handle             = pool_file->handle;
+        newfile->pool_file          = pool_file;
+        newfile->pool_file_refcount = 0;
+
+        GRIB_MUTEX_INIT_ONCE(&once, &init);
+        GRIB_MUTEX_LOCK(&mutex1);
+
+        ++pool_file->pool_file_refcount;
+
+        GRIB_MUTEX_UNLOCK(&mutex1);
+
+        return newfile;
+    }
+    else
+        return 0;
+}
+
+void grib_file_pool_delete_clone(grib_file* cloned_file)
+{
+    grib_file* pool_file = cloned_file->pool_file;
+    if(pool_file)
+    {
+        GRIB_MUTEX_INIT_ONCE(&once, &init);
+        GRIB_MUTEX_LOCK(&mutex1);
+        if(pool_file->pool_file_refcount > 0)
+        {
+            --pool_file->pool_file_refcount;
+
+            if (pool_file->pool_file_refcount == 0)
+                grib_file_pool_delete_file(pool_file);
+        }
+
+        GRIB_MUTEX_UNLOCK(&mutex1);
+    }
+
+    grib_file_delete(cloned_file);
+}
+
 void grib_file_pool_delete_file(grib_file* file)
 {
     grib_file* prev = NULL;
@@ -263,6 +308,7 @@ void grib_file_pool_delete_file(grib_file* file)
     if (file == file_pool.first) {
         file_pool.first   = file->next;
         file_pool.current = file->next;
+        file_pool.size--;
     }
     else {
         prev              = file_pool.first;
@@ -275,10 +321,13 @@ void grib_file_pool_delete_file(grib_file* file)
         DEBUG_ASSERT(prev);
         if (prev) {
             prev->next = file->next;
+            file_pool.size--;
         }
     }
 
     if (file->handle) {
+        fclose(file->handle);
+        file->handle = NULL;
         file_pool.number_of_opened_files--;
     }
     grib_file_delete(file);
@@ -399,12 +448,14 @@ static grib_file* grib_file_new(grib_context* c, const char* name, int* err)
     next_id++;
     GRIB_MUTEX_UNLOCK(&mutex1);
 
-    file->mode     = 0;
-    file->handle   = 0;
-    file->refcount = 0;
-    file->context  = c;
-    file->next     = 0;
-    file->buffer   = 0;
+    file->mode               = 0;
+    file->handle             = 0;
+    file->refcount           = 0;
+    file->context            = c;
+    file->next               = 0;
+    file->pool_file          = 0;
+    file->pool_file_refcount = 0;
+    file->buffer             = 0;
     return file;
 }
 

diff --git a/src/grib_index.cc b/src/grib_index.cc
@@ -767,7 +767,7 @@ void grib_index_delete(grib_index* index)
     while (file) {
         grib_file* f = file;
         file         = file->next;
-        grib_file_delete(f);
+        grib_file_pool_delete_clone(f);
     }
     grib_context_free(index->context, index);
 }
@@ -1076,7 +1076,6 @@ static int codes_index_add_file_internal(grib_index* index, const char* filename
     char buf[1024] = {0,};
     int err = 0;
     grib_file* indfile;
-    grib_file* newfile;
 
     grib_index_key* index_key = NULL;
     grib_handle* h            = NULL;
@@ -1097,11 +1096,7 @@ static int codes_index_add_file_internal(grib_index* index, const char* filename
 
     if (!index->files) {
         grib_filesid++;
-        newfile         = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
-        newfile->id     = grib_filesid;
-        newfile->name   = strdup(file->name);
-        newfile->handle = file->handle;
-        index->files    = newfile;
+        index->files = grib_file_pool_create_clone(c, grib_filesid, file);
     }
     else {
         indfile = index->files;
@@ -1114,11 +1109,7 @@ static int codes_index_add_file_internal(grib_index* index, const char* filename
         while (indfile->next)
             indfile = indfile->next;
         grib_filesid++;
-        newfile         = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
-        newfile->id     = grib_filesid;
-        newfile->name   = strdup(file->name);
-        newfile->handle = file->handle;
-        indfile->next   = newfile;
+        indfile->next = grib_file_pool_create_clone(c, grib_filesid, file);
     }
 
     fseeko(file->handle, 0, SEEK_SET);