From 7f68df08a208f2d6384edf5443a5eea4335ce7ba Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Thu, 14 Sep 2023 12:01:06 +0300 Subject: [PATCH] ssl cert reload - run in endpoint bg (bz 2237903) Signed-off-by: Amit Prinz Setter (cherry picked from commit 9484f467213213085c5f08043d5f56d2adacd611) Signed-off-by: Danny Zaken --- src/endpoint/endpoint.js | 11 ++- src/rpc/rpc_http_server.js | 15 +++- src/server/bg_services/server_monitor.js | 13 --- src/server/web_server.js | 8 +- src/util/ssl_utils.js | 101 +++++++++++++---------- 5 files changed, 82 insertions(+), 66 deletions(-) diff --git a/src/endpoint/endpoint.js b/src/endpoint/endpoint.js index e762ce52f5..c4fda9d460 100755 --- a/src/endpoint/endpoint.js +++ b/src/endpoint/endpoint.js @@ -80,6 +80,7 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma /** * @param {EndpointOptions} options */ +/* eslint-disable max-statements */ async function main(options = {}) { try { // the primary just forks and returns, workers will continue to serve @@ -144,11 +145,17 @@ async function main(options = {}) { const endpoint_request_handler = create_endpoint_handler(init_request_sdk, virtual_hosts); const endpoint_request_handler_sts = create_endpoint_handler(init_request_sdk, virtual_hosts, true); - const ssl_cert = await ssl_utils.get_ssl_certificate('S3'); - const ssl_options = { ...ssl_cert, honorCipherOrder: true }; + const ssl_cert_info = await ssl_utils.get_ssl_cert_info('S3'); + const ssl_options = { ...ssl_cert_info.cert, honorCipherOrder: true }; const http_server = http.createServer(endpoint_request_handler); const https_server = https.createServer(ssl_options, endpoint_request_handler); const https_server_sts = https.createServer(ssl_options, endpoint_request_handler_sts); + ssl_cert_info.on('update', updated_ssl_cert_info => { + dbg.log0("Setting updated S3 ssl certs for endpoint."); + const updated_ssl_options = { ...updated_ssl_cert_info.cert, honorCipherOrder: true }; + https_server.setSecureContext(updated_ssl_options); + https_server_sts.setSecureContext(updated_ssl_options); + }); if (http_port > 0) { dbg.log0('Starting S3 HTTP', http_port); diff --git a/src/rpc/rpc_http_server.js b/src/rpc/rpc_http_server.js index cd9f53de16..4a09ca7263 100644 --- a/src/rpc/rpc_http_server.js +++ b/src/rpc/rpc_http_server.js @@ -48,10 +48,17 @@ class RpcHttpServer extends events.EventEmitter { const logging = options.logging; dbg.log0('HTTP SERVER:', 'port', port, 'secure', secure, 'logging', logging); - const ssl_cert = await ssl_utils.get_ssl_certificate('MGMT'); - const server = secure ? - https.createServer({ ...ssl_cert, honorCipherOrder: true }) : - http.createServer(); + let server; + if (secure) { + const ssl_cert_info = await ssl_utils.get_ssl_cert_info('MGMT'); + server = https.createServer({ ...ssl_cert_info.cert, honorCipherOrder: true }); + ssl_cert_info.on('update', updated_cert_info => { + dbg.log0("Setting updated MGMT ssl certs for rpc server."); + server.setSecureContext({ ...updated_cert_info.cert, honorCipherOrder: true }); + }); + } else { + server = http.createServer(); + } this.install_on_server(server, options.default_handler); return P.fromCallback(callback => server.listen(port, callback)) .then(() => server); diff --git a/src/server/bg_services/server_monitor.js b/src/server/bg_services/server_monitor.js index 528a1f4345..d3c2d68cb8 100644 --- a/src/server/bg_services/server_monitor.js +++ b/src/server/bg_services/server_monitor.js @@ -9,7 +9,6 @@ const os_utils = require('../../util/os_utils'); const Dispatcher = require('../notifications/dispatcher'); const server_rpc = require('../server_rpc'); const system_store = require('../system_services/system_store').get_instance(); -const ssl_utils = require('../../util/ssl_utils'); const db_client = require('../../util/db_client'); @@ -53,7 +52,6 @@ async function run_monitors() { _check_dns_and_phonehome(); await _check_internal_ips(); - await _verify_ssl_certs(); await _check_db_disk_usage(); await _check_address_changes(CONTAINER_PLATFORM); } @@ -80,17 +78,6 @@ function _check_internal_ips() { }); } -async function _verify_ssl_certs() { - dbg.log2('_verify_ssl_certs'); - const updated = await ssl_utils.update_certs_from_disk(); - if (updated) { - dbg.log0('_verify_ssl_certs: SSL certificates changed, restarting relevant services'); - await os_utils.restart_services([ - 'webserver' - ]); - } -} - async function _check_db_disk_usage() { dbg.log2('_check_db_disk_usage'); const { fsUsedSize, fsTotalSize } = await db_client.instance().get_db_stats(); diff --git a/src/server/web_server.js b/src/server/web_server.js index 054e64c6cc..8c26b66236 100755 --- a/src/server/web_server.js +++ b/src/server/web_server.js @@ -90,8 +90,12 @@ async function main() { server_rpc.rpc.register_ws_transport(http_server); await P.ninvoke(http_server, 'listen', http_port); - const ssl_cert = await ssl_utils.get_ssl_certificate('MGMT'); - const https_server = https.createServer({ ...ssl_cert, honorCipherOrder: true }, app); + const ssl_cert_info = await ssl_utils.get_ssl_cert_info('MGMT'); + const https_server = https.createServer({ ...ssl_cert_info.cert, honorCipherOrder: true }, app); + ssl_cert_info.on('update', updated_cert_info => { + dbg.log0("Setting updated MGMT ssl certs for web server."); + https_server.setSecureContext({...updated_cert_info.cert, honorCipherOrder: true }); + }); server_rpc.rpc.register_ws_transport(https_server); await P.ninvoke(https_server, 'listen', https_port); diff --git a/src/util/ssl_utils.js b/src/util/ssl_utils.js index 890d8e5232..9a585f16af 100644 --- a/src/util/ssl_utils.js +++ b/src/util/ssl_utils.js @@ -8,18 +8,41 @@ const https = require('https'); const Semaphore = require('../util/semaphore'); const dbg = require('./debug_module')(__filename); const nb_native = require('./nb_native'); +const { EventEmitter } = require('events'); -const init_cert_info = dir => ({ - dir, - cert: null, - is_loaded: false, - is_generated: false, - sem: new Semaphore(1) -}); +class CertInfo extends EventEmitter { + + constructor(dir) { + super(); + this.dir = dir; + this.cert = null; + this.is_loaded = false; + this.is_generated = false; + this.sem = new Semaphore(1); + } + + async file_notification(event, filename) { + try { + const cert_on_disk = await _read_ssl_certificate(this.dir); + if (this.cert.key === cert_on_disk.key) { + return; + } + + this.cert = cert_on_disk; + this.is_generated = false; + + this.emit('update', this); + } catch (err) { + if (err.code !== 'ENOENT') { + dbg.warn(`SSL certificate failed to update from dir ${this.dir}:`, err.message); + } + } + } +} const certs = { - MGMT: init_cert_info('/etc/mgmt-secret'), - S3: init_cert_info('/etc/s3-secret'), + MGMT: new CertInfo('/etc/mgmt-secret'), + S3: new CertInfo('/etc/s3-secret'), }; function generate_ssl_certificate() { @@ -36,19 +59,19 @@ function verify_ssl_certificate(certificate) { } // Get SSL certificate (load once then serve from cache) -function get_ssl_certificate(service) { +function get_ssl_cert_info(service) { const cert_info = certs[service]; if (!cert_info) { throw new Error(`Invalid service name, got: ${service}`); } if (cert_info.is_loaded) { - return cert_info.cert; + return cert_info; } return cert_info.sem.surround(async () => { if (cert_info.is_loaded) { - return cert_info.cert; + return cert_info; } try { @@ -68,40 +91,19 @@ function get_ssl_certificate(service) { } cert_info.is_loaded = true; - return cert_info.cert; - }); -} - -// For each cert that was loaded into memory we check if the cert was changed on disk. -// If so we update it. If any of the certs was updated we return true else we return false. -async function update_certs_from_disk() { - const promiseList = Object.values(certs).map(cert_info => - cert_info.sem.surround(async () => { - if (!cert_info.is_loaded) { - return false; - } - try { - const cert_on_disk = await _read_ssl_certificate(cert_info.dir); - if (cert_info.cert.key === cert_on_disk.key) { - return false; - } - - cert_info.cert = cert_on_disk; - cert_info.is_generated = false; - return true; - - } catch (err) { - if (err.code !== 'ENOENT') { - dbg.warn(`SSL certificate failed to update from dir ${cert_info.dir}:`, err.message); - } - return false; + try { + fs.watch(cert_info.dir, {}, cert_info.file_notification.bind(cert_info)); + } catch (err) { + if (err.code === 'ENOENT') { + dbg.warn("Certificate folder ", cert_info.dir, " does not exist. New certificate won't be loaded."); + } else { + dbg.error("Failed to watch certificate dir ", cert_info.dir, ". err = ", err); } - }) - ); + } - const updatedList = await Promise.all(promiseList); - return updatedList.some(Boolean); + return cert_info; + }); } // Read SSL certificate form disk @@ -125,6 +127,15 @@ function is_using_generated_certs() { ); } +function get_cert_dir(service) { + const cert_info = certs[service]; + if (!cert_info) { + throw new Error(`Invalid service name, got: ${service}`); + } + + return cert_info.dir; +} + // create a default certificate and start an https server to test it in the browser function run_https_test_server() { const server = https.createServer(generate_ssl_certificate()); @@ -144,8 +155,8 @@ function run_https_test_server() { exports.generate_ssl_certificate = generate_ssl_certificate; exports.verify_ssl_certificate = verify_ssl_certificate; -exports.get_ssl_certificate = get_ssl_certificate; +exports.get_ssl_cert_info = get_ssl_cert_info; exports.is_using_generated_certs = is_using_generated_certs; -exports.update_certs_from_disk = update_certs_from_disk; +exports.get_cert_dir = get_cert_dir; if (require.main === module) run_https_test_server();