cache management updates

sparqlunicorn · Feb 7, 2025 · 53f1f17 · 53f1f17
1 parent 3a77799
commit 53f1f17
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 19 deletions.
diff --git a/sparql_unicorn.py b/sparql_unicorn.py
@@ -39,7 +39,7 @@
 import sys
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "dependencies")))
 from .util.ui.uiutils import UIUtils
-from .util.layerutils import LayerUtils
+from .util.conf.cacheutils import CacheUtils
 from .util.export.layer.layerexporter import LayerExporter
 from .util.conf.configutils import ConfigUtils
 from .tasks.query.util.triplestorereposynctask import TripleStoreRepositorySyncTask
@@ -268,13 +268,15 @@ def manageTripleStoreConfFiles(self):
         else:
             dir = os.path.join(__location__, "tmp/classtree")
             for f in os.listdir(dir):
-                os.remove(os.path.join(dir, f))
+                if CacheUtils.is_file_older_than_x_days(os.path.join(dir,f),CacheUtils.CLASSTREECACHE_EXPIRY):
+                    os.remove(os.path.join(dir, f))
         if not os.path.exists(os.path.join(__location__, "tmp/geoconcepts")):
             os.mkdir(os.path.join(__location__, "tmp/geoconcepts"))
         else:
             dir = os.path.join(__location__, "tmp/geoconcepts")
             for f in os.listdir(dir):
-                os.remove(os.path.join(dir, f))
+                if CacheUtils.is_file_older_than_x_days(os.path.join(dir,f),CacheUtils.GEOCONCEPTS_EXPIRY):
+                    os.remove(os.path.join(dir, f))
         if not os.path.exists(os.path.join(__location__, "tmp/graphcache")):
             os.mkdir(os.path.join(__location__, "tmp/graphcache"))
         if os.path.isfile(os.path.join(__location__, 'conf/triplestoreconf_personal.json')):

diff --git a/tasks/processing/loadtriplestoretask.py b/tasks/processing/loadtriplestoretask.py
@@ -1,15 +1,13 @@
 from ...util.sparqlutils import SPARQLUtils
+from ...util.conf.cacheutils import CacheUtils
 from rdflib import Graph
 from qgis.core import Qgis,QgsTask, QgsMessageLog
 import os
-import time
 
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
 MESSAGE_CATEGORY = 'LoadTripleStoreTask'
 
-CACHEDAYS=30
-
 ## Loads a graph from an RDF file either by providing an internet address or a file path.
 class LoadTripleStoreTask(QgsTask):
 
@@ -20,10 +18,6 @@ def __init__(self, description, curtriplestoreconf,endpointIndex,dlg):
         self.curtriplestoreconf = curtriplestoreconf
         self.dlg=dlg
 
-    @staticmethod
-    def is_file_older_than_x_days(path, days=1):
-        modtime=os.path.getmtime(path)
-        return ((time.time() - modtime) / 3600 > 24 * days)
 
     def run(self):
         #QgsMessageLog.logMessage('Started task: Serializing loaded graph to '.format(self.description()) + str(self.curtriplestoreconf["resource"]["url"]),
@@ -36,12 +30,12 @@ def run(self):
             path = os.path.join(__location__, "../../tmp/graphcache/" + str(
                 str(self.curtriplestoreconf["resource"]["url"]).replace("/", "_").replace("['", "")
                 .replace("']","").replace("\\", "_").replace(":", "_")) + ".ttl")
-            if os.path.isfile(path) and not LoadTripleStoreTask.is_file_older_than_x_days(path,CACHEDAYS):
-                QgsMessageLog.logMessage('FILE IS PRESENT AND NOT OLDER THAN '+str(CACHEDAYS),MESSAGE_CATEGORY, Qgis.Info)
+            if os.path.isfile(path) and not CacheUtils.is_file_older_than_x_days(path,CacheUtils.GRAPHCACHE_EXPIRY):
+                #QgsMessageLog.logMessage('FILE IS PRESENT AND NOT OLDER THAN '+str(CacheUtils.GRAPHCACHE_EXPIRY),MESSAGE_CATEGORY, Qgis.Info)
                 self.graph.parse(path)
                 self.curtriplestoreconf["instance"]=self.graph
             else:
-                QgsMessageLog.logMessage('FILE IS NOT PRESENT OR OLDER THAN '+str(CACHEDAYS),MESSAGE_CATEGORY, Qgis.Info)
+                #QgsMessageLog.logMessage('FILE IS NOT PRESENT OR OLDER THAN '+str(CacheUtils.GRAPHCACHE_EXPIRY),MESSAGE_CATEGORY, Qgis.Info)
                 #QgsMessageLog.logMessage('Started task: Loading graph from URI '.format(self.description()) +self.curtriplestoreconf["resource"]["url"],
                 #                         MESSAGE_CATEGORY, Qgis.Info)
                 SPARQLUtils.loadGraph(self.curtriplestoreconf["resource"]["url"],self.graph)

diff --git a/tasks/query/enrichment/whattoenrichquerytask.py b/tasks/query/enrichment/whattoenrichquerytask.py
@@ -38,19 +38,19 @@ def __init__(self, description, triplestoreurl, query, searchTerm, prefixes, sea
         item.setText("Loading...")
         self.searchResult.setItem(0,0,item)
         self.searchResult.setMouseTracking(True)
+        if isinstance(self.prefixes, Iterable):
+            self.query="".join(self.prefixes) + self.query
+        else:
+            self.query=str(self.prefixes).replace("None", "") + self.query
+        self.query=SPARQLUtils.queryPreProcessing(self.query, triplestoreconf,None,False,triplestoreurl["type"]=="file")
 
     def run(self):
         QgsMessageLog.logMessage('Started task "{}"'.format(self.description()), MESSAGE_CATEGORY, Qgis.Info)
         QgsMessageLog.logMessage('Started task "{}"'.format(self.searchTerm), MESSAGE_CATEGORY, Qgis.Info)
         self.query=SPARQLUtils.queryPreProcessing(self.query,self.triplestoreconf)
         if self.searchTerm == "":
             return False
-        if isinstance(self.prefixes, Iterable):
-            results = SPARQLUtils.executeQuery(self.triplestoreurl, "".join(self.prefixes) + self.query,
-                                               self.triplestoreconf)
-        else:
-            results = SPARQLUtils.executeQuery(self.triplestoreurl, str(self.prefixes).replace("None", "") + self.query,
-                                               self.triplestoreconf)
+        results = SPARQLUtils.executeQuery(self.triplestoreurl, self.query, self.triplestoreconf)
         if results == False:
             return False
         if len(results["results"]["bindings"]) == 0:

diff --git a/util/conf/cacheutils.py b/util/conf/cacheutils.py
@@ -1,12 +1,24 @@
 from qgis.core import Qgis,QgsTask, QgsMessageLog
 import os
+import time
 
 MESSAGE_CATEGORY = 'CacheUtils'
 
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
 class CacheUtils:
 
+    CLASSTREECACHE_EXPIRY=14
+
+    GEOCONCEPTS_EXPIRY=14
+
+    GRAPHCACHE_EXPIRY=30
+
+    @staticmethod
+    def is_file_older_than_x_days(path, days=1):
+        modtime=os.path.getmtime(path)
+        return ((time.time() - modtime) / 3600 > 24 * days)
+
     @staticmethod
     def graphCacheSize():
         graphcachesize=0