Merge pull request #855 from OpenKnowledgeMaps/dev

refactor: metadata clean up indexes

Author: chreman

server/classes/headstart/preprocessing/Snapshot.php

@@ -28,9 +28,10 @@ public function __construct($ini_array, $query, $id, $service, $service_name, $v
         $snap_php = $ini_array["snapshot"]["snapshot_php"];
         $storage = $ini_array["snapshot"]["storage_path"];
         $nodemodules = $ini_array["snapshot"]["nodemodules_path"];
+        $chrome_executable_path = $ini_array["snapshot"]["chrome_executable_path"];
 
         $url = "{$host}{$snap_php}?{$url_postfix}";
-        $this->cmd = "{$node_path} {$getsvg} \"{$url}\" {$storage}{$post_data['file']}.png {$nodemodules}";
+        $this->cmd = "{$node_path} {$getsvg} \"{$url}\" {$storage}{$post_data['file']}.png {$nodemodules} {$chrome_executable_path}";
     }
 
     public function takeSnapshot() {

server/preprocessing/conf/config.ini

@@ -32,6 +32,8 @@ storage_path = "/path/to/storage/"
 snapshot_php = "server/services/snapshot/headstart_snapshot.php"
 # snapshot_local_protocol fallback for non-server environments
 snapshot_local_protocol = "http://"
+# chrome executable path (optional, only needed if puppeteer cannot find chrome automatically)
+chrome_executable_path = "/path/to/chrome"
 
 
 [connection]

server/preprocessing/other-scripts/base.R

@@ -248,6 +248,7 @@ etl <- function(res, repo, non_public) {
   subject_cleaned = gsub("(wikidata)?\\.org/entity/[qQ]([\\d]+)?", "", subject_cleaned) # remove wikidata classification
   subject_cleaned = gsub("</keyword><keyword>", "", subject_cleaned) # remove </keyword><keyword>
   subject_cleaned = gsub("\\[No keyword\\]", "", subject_cleaned)
+  subject_cleaned = gsub("\\[[^]]*\\]", "", subject_cleaned) # remove any text inside square brackets
   subject_cleaned = gsub("\\[[^\\[]+\\][^\\;]+(;|$)?", "", subject_cleaned) # remove classification
   subject_cleaned = gsub("[0-9]{2,} [A-Z]+[^;]*(;|$)?", "", subject_cleaned) #remove classification
   subject_cleaned = gsub(" -- ", "; ", subject_cleaned) #replace inconsistent keyword separation

server/services/getChartSVG.js

@@ -7,7 +7,11 @@ function timeout(ms) {
 
 (async() => {
     try {
-        const browser = await puppeteer.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox'] });
+        const browser = await puppeteer.launch({
+            headless: true,
+            args: ['--no-sandbox', '--disable-setuid-sandbox'],
+            executablePath: process.argv[5]
+        });
         await timeout(1000)
         const page = await browser.newPage();
         await timeout(1000)

server/services/search.php

@@ -221,6 +221,9 @@ function search(
     } else {
       $vis_type = "overview";
     }
+    if (isset($post_params["vis_type"]) && $post_params["vis_type"] == "geomap") {
+      $vis_type = "geomap";
+    }
     $snapshot = new \headstart\preprocessing\Snapshot($ini_array, $query, $unique_id, $service, $repo2snapshot[$service], $vis_type);
     $snapshot->takeSnapshot();
   }

server/services/snapshot/data-config_aquanavi.js

@@ -0,0 +1,33 @@
+var data_config = {
+    tag: "visualization",
+    mode: "search_repos",
+
+    service: "base",
+
+    title: "",
+    base_unit: "citations",
+    use_area_uri: true,
+    show_multiples: false,
+    show_dropdown: false,
+    preview_type: "pdf",
+    sort_options: ["relevance", "title", "authors", "year"],
+    is_force_areas: true,
+    language: "eng_pubmed",
+    area_force_alpha: 0.015,
+    show_list: true,
+    content_based: true,
+    url_prefix: "https://www.base-search.net/Record/",
+    
+    show_context: true,
+    create_title_from_context: true,
+    context_most_relevant_tooltip: true,
+    
+    doi_outlink: true,
+    filter_menu_dropdown: true,
+    sort_menu_dropdown: true,
+    filter_options: ["all", "open_access"],
+    //show number of open access documents in context
+    show_context_oa_number: false,
+
+    visualization_type: "geomap", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
+};

server/services/snapshot/data-config_base.js

@@ -28,4 +28,6 @@ var data_config = {
     filter_options: ["all", "open_access"],
     //show number of open access documents in context
     show_context_oa_number: false,
+
+    visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
 };

server/services/snapshot/data-config_openaire.js

@@ -64,5 +64,5 @@ var data_config = {
         cited_by_tweeters_count: 'tweets',
         'readers.mendeley': 'readers'
     },
-    
+    visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
 };

server/services/snapshot/data-config_orcid.js

@@ -28,4 +28,5 @@ var data_config = {
   filter_options: ["all", "open_access"],
   //show number of open access documents in context
   show_context_oa_number: false,
+  visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
 };

server/services/snapshot/data-config_pubmed.js

@@ -34,4 +34,5 @@ var data_config = {
     filter_menu_dropdown: true,
     sort_menu_dropdown: true,
     filter_options: ["all", "open_access"],
+    visualization_type: "overview", // Used in the reducer named "chartType" to determine the visualization type: overview, timeline, geomap
 };