From 29262d17bfeb718bbfe250fcde484ecfcebf69a8 Mon Sep 17 00:00:00 2001 From: Jit Gan Date: Mon, 2 Mar 2020 23:57:36 -0500 Subject: [PATCH] Implement file upload on Express with postgres integration --- .gitignore | 2 + README.md | 4 +- package-lock.json | 131 ++++++++++++++++++++++++ package.json | 4 +- src/database/migrations/file_migrate.js | 17 +++ src/routes/files.js | 42 ++++++++ src/server.js | 1 + 7 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 src/database/migrations/file_migrate.js create mode 100644 src/routes/files.js diff --git a/.gitignore b/.gitignore index bf127e9..066012b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .env node_modules/ + +tmp/ \ No newline at end of file diff --git a/README.md b/README.md index 648248f..fd4cdf5 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,11 @@ Copy the .env.sample file to a .env file and fill in the missing values - The .env file should not be checked into the repository, it contains production secrets! - When starting up, the server will read the database credentials/other info from the .env file. -Run `npm run knex seed:run` to setup the initial database state +Run `npm run knex seed:run` and `npm run knex migrate:latest` to setup the initial database state Run `npm run start` to start the server + + ### Setup Postgres These are some helpful tutorials: - https://blog.logrocket.com/setting-up-a-restful-api-with-node-js-and-postgresql-d96d6fc892d8/ diff --git a/package-lock.json b/package-lock.json index eac6ff7..0d7d835 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,11 @@ "negotiator": "0.6.2" } }, + "append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" + }, "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", @@ -157,11 +162,25 @@ } } }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, "buffer-writer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.x" + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -228,6 +247,41 @@ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "content-disposition": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", @@ -256,6 +310,11 @@ "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=" }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -321,6 +380,15 @@ "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=" }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.x", + "streamsearch": "0.1.2" + } + }, "dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", @@ -1048,6 +1116,21 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "multer": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz", + "integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==", + "requires": { + "append-field": "^1.0.0", + "busboy": "^0.2.11", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.1", + "object-assign": "^4.1.1", + "on-finished": "^2.3.0", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + } + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -1071,6 +1154,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-copy": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", @@ -1291,6 +1379,11 @@ "xtend": "^4.0.0" } }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -1321,6 +1414,24 @@ "unpipe": "1.0.0" } }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + } + } + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -1621,6 +1732,16 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "tarn": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tarn/-/tarn-2.0.0.tgz", @@ -1688,6 +1809,11 @@ "mime-types": "~2.1.24" } }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", @@ -1763,6 +1889,11 @@ "os-homedir": "^1.0.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/package.json b/package.json index 8795680..8c2f789 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "src/server.js", "scripts": { "knex": "knex --knexfile ./src/database/knexfile.js", - "start": "node src/server.js" + "start": "node src/server.js", + "setup": "npm run knex seed:run & npm run knex migrate:latest" }, "author": "Vedant Roy", "license": "ISC", @@ -14,6 +15,7 @@ "express": "^4.17.1", "find-config": "^1.0.0", "knex": "^0.20.10", + "multer": "^1.4.2", "pg": "^7.18.2" }, "devDependencies": {} diff --git a/src/database/migrations/file_migrate.js b/src/database/migrations/file_migrate.js new file mode 100644 index 0000000..3d003e1 --- /dev/null +++ b/src/database/migrations/file_migrate.js @@ -0,0 +1,17 @@ +// NOTE TO DEVS: +// You can also run "npm run knex migrate:down file_migrate.js" to drop the table +// if you run into any issues. + +// Adds a files table that has a relation with the user table +exports.up = async (knex) => { + return await knex.schema.createTable('files', (table) => { + table.string('filename'); + table.binary('file'); + table.string('username'); + }); +} + +// Drops the files table +exports.down = async (knex) => { + return await knex.schema.dropTable('files'); +} \ No newline at end of file diff --git a/src/routes/files.js b/src/routes/files.js new file mode 100644 index 0000000..d7549dd --- /dev/null +++ b/src/routes/files.js @@ -0,0 +1,42 @@ +const fs = require('fs'); +const knex = require('../database/database'); +const multer = require('multer'); +const router = require('express').Router(); + +const dir = "\\tmp"; + +let storage = multer.diskStorage({ + // file uploaded is stored in local directory + destination: (req, file, cb) => { + const uploadDir = "\\tmp/uploads/" + // create a new local directory if it did not already exist + fs.exists(uploadDir, exist => { + if (!exist) { + return fs.mkdir(uploadDir, {recursive: true }, err => cb(err, uploadDir)); + } + return cb(null, uploadDir); + }); + }, + filename: (req, file, cb) => { + cb(null, Date.now() + '-' + file.originalname); + } +}); + +let upload = multer({ storage: storage }); + +// [base] = localhost:[port]/files +// POST [base]/upload/[a username] attached with a file that has fieldname:'myfile' +// to upload an arbitrary file to the database +router.post('/upload/:user', upload.single('myfile'), async (req, res, next) => { + try { + let username = req.params.user; + let path = req.file.destination + req.file.filename + await knex.raw('INSERT INTO "files" VALUES (?, pg_read_binary_file(?), ?)', + [req.file.filename, path, username]); + res.send("Succesfully uploads file"); + } catch (error) { + console.log(error); + } +}); + +module.exports = router; \ No newline at end of file diff --git a/src/server.js b/src/server.js index 1b748ab..3b0a8d5 100644 --- a/src/server.js +++ b/src/server.js @@ -2,6 +2,7 @@ const express = require('express') const app = express() app.use('/users', require('./routes/users')) +app.use('/files', require('./routes/files')) const port = 8080 app.listen(port, () => console.log(`Server listening on port ${port}.`)) \ No newline at end of file