Sigma.js demo using node-rapids (#392)

Creation of `node/modules/demo/api-demo` and all supporting packages.

Author: thomcom

lerna.json

@@ -15,6 +15,7 @@
     "modules/demo/ssr/*",
     "modules/demo/tfjs/*",
     "modules/demo/client-server",
+    "modules/demo/api-server",
     "modules/demo/viz-app",
     "modules/demo/sql/*"
   ]

modules/demo/.vscode/launch.json

@@ -13,14 +13,27 @@
         }
     ],
     "configurations": [
+        {
+            "name": "Attach Test",
+            "port": 9229,
+            "request": "attach",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node"
+        },
         {
             "type": "node",
             "request": "launch",
             "name": "Debug Demo (TS only)",
             "program": "${workspaceFolder}/${input:DEMO_NAME}",
             "stopOnEntry": false,
-            "args": ["${input:DEMO_ARGS}"],
-            "runtimeArgs": ["--experimental-vm-modules"],
+            "args": [
+                "${input:DEMO_ARGS}"
+            ],
+            "runtimeArgs": [
+                "--experimental-vm-modules"
+            ],
             "cwd": "${workspaceFolder}/${input:DEMO_NAME}",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
@@ -44,7 +57,9 @@
         {
             "name": "Debug Demo (launch gdb)",
             // hide the individual configurations from the debug dropdown list
-            "presentation": { "hidden": true },
+            "presentation": {
+                "hidden": true
+            },
             "type": "cppdbg",
             "request": "launch",
             "stopAtEntry": false,
@@ -62,9 +77,18 @@
             ],
             "program": "${input:NODE_BINARY}",
             "environment": [
-                { "name": "NODE_DEBUG", "value": "1" },
-                { "name": "NODE_NO_WARNINGS", "value": "1" },
-                { "name": "NODE_ENV", "value": "production" },
+                {
+                    "name": "NODE_DEBUG",
+                    "value": "1"
+                },
+                {
+                    "name": "NODE_NO_WARNINGS",
+                    "value": "1"
+                },
+                {
+                    "name": "NODE_ENV",
+                    "value": "production"
+                },
                 // { "name": "READABLE_STREAM", "value": "disable" },
             ],
             "args": [
@@ -80,7 +104,9 @@
             "type": "node",
             "request": "attach",
             // hide the individual configurations from the debug dropdown list
-            "presentation": { "hidden": true },
+            "presentation": {
+                "hidden": true
+            },
             "port": 9229,
             "timeout": 60000,
             "cwd": "${workspaceFolder}",
@@ -113,7 +139,7 @@
             "command": "shellCommand.execute",
             "args": {
                 "description": "Select a demo to debug",
-                "command": "echo client-server viz-app luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
+                "command": "echo client-server viz-app api-server luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
             }
         },
     ]

modules/demo/api-server/.gitignore

@@ -0,0 +1,61 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules
+jspm_packages
+
+# Optional npm cache directory
+.npm
+
+# Optional REPL history
+.node_repl_history
+
+# 0x
+profile-*
+
+# mac files
+.DS_Store
+
+# vim swap files
+*.swp
+
+# webstorm
+.idea
+
+# vscode
+.vscode
+*code-workspace
+
+# clinic
+profile*
+*clinic*
+*flamegraph*
+
+# Node-rapids .cache symlink
+.cache

modules/demo/api-server/README.md

@@ -0,0 +1,89 @@
+# Fastify HTTP Server Demo
+
+This project is a Fastify-based node http server that allows
+commands and data to be sent to and from the NVIDIA GPUs installed
+on the host machine.
+
+Essentially, the node-rapids system is provided as a backend to any
+HTTP client. At this time only limited functionality is available to
+load JSON files in the `graphology` graph dataset format, plus API
+requests to request Dataframes and their Columns via `apache-arrow`.
+
+Two endpoints, `graphology/nodes` and `graphology/edges` specifically
+return pre-formatted arrays that can be used directly with the
+[sigma.js](https://github.com/jacomyal/sigma.js) renderer. An
+[extra-large-graphs](https://github.com/jacomyal/sigma.js/pull/1252) example PR is in the works
+that utilizes this GPU-accelerated data for rendering larger datasets
+than available via only CPU.
+
+## Main Dependencies
+- @rapidsai/cudf
+- fastify
+- fastify-arrow
+- apache-arrow
+
+An example project that demonstrates this API has a PR being reviewed at [sigma.js](https://github.com/jacomyal/sigma.js),
+but this project does not depend on sigma.js.j
+
+## Installation
+
+To install dependencies, run the following from the root directory for `node-rapids`
+
+```bash
+yarn
+```
+
+To run the demo
+```bash
+# Select the api-server demo from the list of demos
+yarn demo
+# OR specifically with
+cd modules/demo/api-server
+yarn start
+```
+
+## Dataset
+
+Run the graph generator at https://github.com/thomcom/sigma.js/blob/add-gpu-graph-to-example/examples/extra-large-graphs/generate-graph.js
+to create a very large graph using the object
+
+```js
+const state = {
+  order: 1000000,
+  size: 2000000,
+  clusters: 3,
+  edgesRenderer: 'edges-fast'
+};
+```
+
+You don't need to edit the file in order to create a graph of the above size. Simply call the .js via node:
+
+```bash
+node graph-generator.js
+```
+
+Which will create a file `./large-graph.json`. Copy `./large-graph.json` into `api-server/` and then set your
+API request to the location of the file relative to `routes/graphology/index.js`:
+
+```
+curl http://localhost:3000/graphology/read_large_demo?filename=../../large-graph.json
+```
+
+Which will use parallel JSON parsing to load the graph onto the GPU.
+
+## Routes
+
+```txt
+/
+/graphology
+/graphology/read_json
+/graphology/read_large_demo
+/graphology/list_tables
+/graphology/get_table/:table
+/graphology/get_column/:table/:column
+/graphology/nodes/bounds
+/graphology/nodes
+/graphology/edges
+/graphology/release
+```
+

modules/demo/api-server/app.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const path     = require('path')
+const AutoLoad = require('@fastify/autoload')
+
+module.exports = async function(fastify, opts) {
+  // Place here your custom code!
+
+  // Do not touch the following lines
+
+  // This loads all plugins defined in plugins
+  // those should be support plugins that are reused
+  // through your application
+  fastify.register(AutoLoad,
+                   {dir: path.join(__dirname, 'plugins'), options: Object.assign({}, opts)})
+
+  // This loads all plugins defined in routes
+  // define your routes in one of these
+  fastify.register(
+    AutoLoad,
+    {dir: path.join(__dirname, 'routes'), options: Object.assign({}, opts), ignorePattern: /.*.ts/})
+}

modules/demo/api-server/index.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const Path = require('path');
+
+// Change cwd to the example dir so relative file paths are resolved
+process.chdir(__dirname);
+
+const fastify = require.resolve('fastify-cli/cli.js');
+
+const {spawnSync} = require('child_process');
+
+const env = {
+  NEXT_TELEMETRY_DISABLED: 1,  // disable https://fastifyjs.org/telemetry
+  ...process.env,
+};
+
+spawnSync(process.execPath,
+          [fastify, 'start', '-l', 'info', '-P', '-w', 'app.js'],
+          {env, cwd: __dirname, stdio: 'inherit'});

modules/demo/api-server/package.json

@@ -0,0 +1,33 @@
+{
+  "private": true,
+  "name": "@rapidsai/demo-api-server",
+  "main": "index.js",
+  "version": "0.0.1",
+  "license": "Apache-2.0",
+  "author": "NVIDIA, Inc. (https://nvidia.com/)",
+  "maintainers": [
+    "Thomson Comer <[email protected]>"
+  ],
+  "bin": "index.js",
+  "description": "A fastify-based web server that provides browser-access to GPU resources.",
+  "scripts": {
+    "test": "tap \"test/**/*.test.js\"",
+    "start": "fastify start -l info -P -p 3010 -w app.js",
+    "dev": "fastify start -w -l info -P -p 3010 app.js"
+  },
+  "keywords": ["rapids.ai", "node", "NVIDIA", "gpu", "Dataframe", "pandas"],
+  "dependencies": {
+    "@fastify/autoload": "^4.0.0",
+    "@fastify/sensible": "^4.0.0",
+    "@types/node": "17.0.33",
+    "fastify": "^3.0.0",
+    "fastify-arrow": "1.0.0",
+    "fastify-cli": "^3.0.1",
+    "fastify-plugin": "^3.0.0",
+    "@fastify/cors": "latest"
+  },
+  "devDependencies": {
+    "path": "0.12.7",
+    "tap": "^16.1.0"
+  }
+}

modules/demo/api-server/plugins/README.md

@@ -0,0 +1,16 @@
+# Plugins Folder
+
+Plugins define behavior that is common to all the routes in your
+application. Authentication, caching, templates, and all the other cross
+cutting concerns should be handled by plugins placed in this folder.
+
+Files in this folder are typically defined through the
+[`fastify-plugin`](https://github.com/fastify/fastify-plugin) module,
+making them non-encapsulated. They can define decorators and set hooks
+that will then be used in the rest of your application.
+
+Check out:
+
+* [The hitchhiker's guide to plugins](https://www.fastify.io/docs/latest/Guides/Plugins-Guide/)
+* [Fastify decorators](https://www.fastify.io/docs/latest/Reference/Decorators/).
+* [Fastify lifecycle](https://www.fastify.io/docs/latest/Reference/Lifecycle/).

modules/demo/api-server/plugins/sensible.js

@@ -0,0 +1,14 @@
+'use strict'
+
+const fp = require('fastify-plugin')
+
+/**
+ * This plugins adds some utilities to handle http errors
+ *
+ * @see https://github.com/fastify/fastify-sensible
+ */
+module.exports = fp(async function (fastify, opts) {
+  fastify.register(require('@fastify/sensible'), {
+    errorHandler: false
+  })
+})

modules/demo/api-server/plugins/support.js

@@ -0,0 +1,12 @@
+'use strict'
+
+const fp = require('fastify-plugin')
+
+// the use of fastify-plugin is required to be able
+// to export the decorators to the outer scope
+
+module.exports = fp(async function (fastify, opts) {
+  fastify.decorate('someSupport', function () {
+    return 'hugs'
+  })
+})