From ccc8a03266385ae9049d7e8257076608cd6a5f92 Mon Sep 17 00:00:00 2001 From: Prajwal Kulkarni Date: Sun, 28 Jan 2024 22:52:12 +0530 Subject: [PATCH 1/4] Add throttling to download network call and move all URLs to separate file --- src/URLs.ts | 3 +++ src/autoload.ts | 9 +++++---- src/utilities.ts | 51 ++++++++++++++++++++++++++++-------------------- 3 files changed, 38 insertions(+), 25 deletions(-) create mode 100644 src/URLs.ts diff --git a/src/URLs.ts b/src/URLs.ts new file mode 100644 index 000000000..32aedea19 --- /dev/null +++ b/src/URLs.ts @@ -0,0 +1,3 @@ +export const RCSB_FILES_URL = "https://files.rcsb.org/view/"; +export const NCBI_PUBCHEM_URL = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/"; +export const RCSB_MMTF_URL = "https://mmtf.rcsb.org/v1.0/full/"; \ No newline at end of file diff --git a/src/autoload.ts b/src/autoload.ts index dc6398bf0..0e7bb6ddf 100644 --- a/src/autoload.ts +++ b/src/autoload.ts @@ -4,6 +4,7 @@ import { GLViewer, createViewer } from "./GLViewer"; import { SurfaceType } from "./ProteinSurface4"; import { get, specStringToObject } from "./utilities"; import { CC } from "./colors"; +import { NCBI_PUBCHEM_URL, RCSB_FILES_URL } from "URLs"; export var autoinit = false; export var processing_autoinit = false; @@ -40,12 +41,12 @@ export function autoload(viewer?: any, callback?: (arg0: any) => void) { type = null; if (viewerdiv.dataset.pdb) { - datauri.push("https://files.rcsb.org/view/" + viewerdiv.dataset.pdb + ".pdb"); + datauri.push(RCSB_FILES_URL + viewerdiv.dataset.pdb + ".pdb"); datatypes.push("pdb"); } else if (viewerdiv.dataset.cid) { //this doesn't actually work since pubchem does have CORS enabled datatypes.push("sdf"); - datauri.push("https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/" + viewerdiv.dataset.cid + + datauri.push(NCBI_PUBCHEM_URL + viewerdiv.dataset.cid + "/SDF?record_type=3d"); } else if (viewerdiv.dataset.href || viewerdiv.dataset.url) { @@ -67,7 +68,7 @@ export function autoload(viewer?: any, callback?: (arg0: any) => void) { var divdata = viewerdiv.dataset; for (i in divdata) { if ((i.substring(0, 3) === "pdb" && (i !== "pdb"))) { - datauri.push("https://files.rcsb.org/view/" + divdata[i] + ".pdb"); + datauri.push(RCSB_FILES_URL + divdata[i] + ".pdb"); datatypes.push('pdb'); } else if (i.substring(0, 4) === "href" && (i !== "href")) { @@ -75,7 +76,7 @@ export function autoload(viewer?: any, callback?: (arg0: any) => void) { datauri.push(uri); datatypes.push(uri.substring(uri.lastIndexOf('.') + 1)); } else if (i.substring(0, 3) === "cid" && (i !== "cid")) { - datauri.push("https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/" + divdata[i] + "/SDF?record_type=3d"); + datauri.push(NCBI_PUBCHEM_URL + divdata[i] + "/SDF?record_type=3d"); datatypes.push('sdf'); } } diff --git a/src/utilities.ts b/src/utilities.ts index dd9ac67a5..7188b6e5b 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -1,5 +1,6 @@ //a collection of miscellaneous utility functions +import { NCBI_PUBCHEM_URL, RCSB_FILES_URL, RCSB_MMTF_URL } from "URLs"; import { builtinGradients, Gradient } from "./Gradient"; import { VolumeData } from "./VolumeData"; import { builtinColorSchemes, CC, elementColors, htmlColors, Color } from "./colors"; @@ -344,20 +345,22 @@ function checkStatus(response) { return response; } +let isRequestProcessing = false; /** * Fetch data from URL * * @param uri URL * @param callback Function to call with data */ -export function get(uri, callback?) { - var promise = fetch(uri).then(checkStatus).then((response) => response.text()); +export function get(uri:string, callback?:(value: unknown) => unknown) { + const promise = fetch(uri).then(checkStatus).then((response) => response.text()).finally(()=>isRequestProcessing = false); if (callback) return promise.then(callback); - else - return promise; + return promise; } + +type RequestMethod = "GET"|"POST"|"PUT"|"DELETE"|"HEAD"|"OPTIONS"|"PATCH"; /** * Download binary data (e.g. a gzipped file) into an array buffer and provide * arraybuffer to callback. @@ -367,19 +370,17 @@ export function get(uri, callback?) { * @param {string} [postdata] - data for POST request * @return {Promise} */ -export function getbin(uri, callback?, request?, postdata?) { - var promise; - if (request == "POST") { - promise = fetch(uri, { method: 'POST', body: postdata }) - .then((response) => checkStatus(response)) - .then((response) => response.arrayBuffer()); - } else { - promise = fetch(uri).then((response) => checkStatus(response)) - .then((response) => response.arrayBuffer()); - } +export function getbin(uri:string, callback?:(value: unknown) => unknown, request?: RequestMethod, postdata?) { + const promise = fetch(uri, { method: request || "GET", body: postdata }) + .then((response) => checkStatus(response)) + .then((response) => response.arrayBuffer()) + .catch(err=>console.log(err)) + .finally(()=>isRequestProcessing = false); + + if (callback) return promise.then(callback); - else return promise; + return promise; }; @@ -409,6 +410,11 @@ export function download(query, viewer, options, callback?) { var promise = null; var m = viewer.addModel(); + if(isRequestProcessing){ + alert("Request already processing, please wait"); + return; + } + if (query.indexOf(':') < 0) { //no type specifier, guess if (query.length == 4) { @@ -420,13 +426,14 @@ export function download(query, viewer, options, callback?) { } } if (query.substring(0, 5) === 'mmtf:') { - pdbUri = options && options.pdbUri ? options.pdbUri : "https://mmtf.rcsb.org/v1.0/full/"; + pdbUri = options && options.pdbUri ? options.pdbUri : RCSB_MMTF_URL; query = query.substring(5).toUpperCase(); uri = pdbUri + query; if (options && typeof options.noComputeSecondaryStructure === 'undefined') { //when fetch directly from pdb, trust structure annotations options.noComputeSecondaryStructure = true; } + isRequestProcessing = true; promise = new Promise(function (resolve) { getbin(uri) .then(function (ret) { @@ -454,11 +461,11 @@ export function download(query, viewer, options, callback?) { return; } if (type == 'mmtf') { - mmtfUri = options && options.mmtfUri ? options.mmtfUri : 'https://mmtf.rcsb.org/v1.0/full/'; + mmtfUri = options && options.mmtfUri ? options.mmtfUri : RCSB_MMTF_URL; uri = mmtfUri + query.toUpperCase(); } else { - pdbUri = options && options.pdbUri ? options.pdbUri : "https://files.rcsb.org/view/"; + pdbUri = options && options.pdbUri ? options.pdbUri : RCSB_FILES_URL; uri = pdbUri + query + "." + type; } @@ -468,7 +475,7 @@ export function download(query, viewer, options, callback?) { if (!query.match(/^[0-9]+$/)) { alert("Wrong Compound ID"); return; } - uri = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/cid/" + query + + uri = NCBI_PUBCHEM_URL + query + "/SDF?record_type=3d"; } else if (query.substring(0, 4) == 'url:') { uri = query.substring(4); @@ -480,6 +487,7 @@ export function download(query, viewer, options, callback?) { viewer.zoomTo(); viewer.render(); }; + isRequestProcessing = true; promise = new Promise(function (resolve) { if (type == 'mmtf') { //binary data getbin(uri) @@ -488,7 +496,7 @@ export function download(query, viewer, options, callback?) { resolve(m); }).catch(function () { //if mmtf server is being annoying, fallback to text - pdbUri = options && options.pdbUri ? options.pdbUri : "https://files.rcsb.org/view/"; + pdbUri = options && options.pdbUri ? options.pdbUri : RCSB_FILES_URL; uri = pdbUri + query + ".pdb"; type = "pdb"; console.log("falling back to pdb format"); @@ -520,7 +528,8 @@ export function download(query, viewer, options, callback?) { }); return m; } - else return promise; + + return promise; }; From f149c1f85bf96ee29ffb85ed68d4b3ee025f0aee Mon Sep 17 00:00:00 2001 From: Prajwal Kulkarni Date: Sun, 28 Jan 2024 23:17:56 +0530 Subject: [PATCH 2/4] Update import path --- src/autoload.ts | 2 +- src/utilities.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/autoload.ts b/src/autoload.ts index 0e7bb6ddf..1efcc9cc4 100644 --- a/src/autoload.ts +++ b/src/autoload.ts @@ -4,7 +4,7 @@ import { GLViewer, createViewer } from "./GLViewer"; import { SurfaceType } from "./ProteinSurface4"; import { get, specStringToObject } from "./utilities"; import { CC } from "./colors"; -import { NCBI_PUBCHEM_URL, RCSB_FILES_URL } from "URLs"; +import { NCBI_PUBCHEM_URL, RCSB_FILES_URL } from "./URLs"; export var autoinit = false; export var processing_autoinit = false; diff --git a/src/utilities.ts b/src/utilities.ts index 7188b6e5b..1b6cc1f03 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -1,6 +1,6 @@ //a collection of miscellaneous utility functions -import { NCBI_PUBCHEM_URL, RCSB_FILES_URL, RCSB_MMTF_URL } from "URLs"; +import { NCBI_PUBCHEM_URL, RCSB_FILES_URL, RCSB_MMTF_URL } from "./URLs"; import { builtinGradients, Gradient } from "./Gradient"; import { VolumeData } from "./VolumeData"; import { builtinColorSchemes, CC, elementColors, htmlColors, Color } from "./colors"; @@ -352,7 +352,7 @@ let isRequestProcessing = false; * @param uri URL * @param callback Function to call with data */ -export function get(uri:string, callback?:(value: unknown) => unknown) { +export function get(uri:string, callback?) { const promise = fetch(uri).then(checkStatus).then((response) => response.text()).finally(()=>isRequestProcessing = false); if (callback) return promise.then(callback); @@ -371,7 +371,7 @@ type RequestMethod = "GET"|"POST"|"PUT"|"DELETE"|"HEAD"|"OPTIONS"|"PATCH"; * @return {Promise} */ -export function getbin(uri:string, callback?:(value: unknown) => unknown, request?: RequestMethod, postdata?) { +export function getbin(uri:string, callback?, request?: RequestMethod, postdata?) { const promise = fetch(uri, { method: request || "GET", body: postdata }) .then((response) => checkStatus(response)) .then((response) => response.arrayBuffer()) From fbd3ce78b41fc1c766aacab07e9207f8144d19eb Mon Sep 17 00:00:00 2001 From: Prajwal Kulkarni Date: Tue, 6 Feb 2024 21:24:34 +0530 Subject: [PATCH 3/4] Minor changes to utilities.ts --- src/utilities.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/utilities.ts b/src/utilities.ts index 1b6cc1f03..f68034bc4 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -375,7 +375,6 @@ export function getbin(uri:string, callback?, request?: RequestMethod, postdata? const promise = fetch(uri, { method: request || "GET", body: postdata }) .then((response) => checkStatus(response)) .then((response) => response.arrayBuffer()) - .catch(err=>console.log(err)) .finally(()=>isRequestProcessing = false); @@ -473,7 +472,7 @@ export function download(query, viewer, options, callback?) { type = "sdf"; query = query.substring(4); if (!query.match(/^[0-9]+$/)) { - alert("Wrong Compound ID"); return; + return alert("Wrong Compound ID"); } uri = NCBI_PUBCHEM_URL + query + "/SDF?record_type=3d"; @@ -489,7 +488,7 @@ export function download(query, viewer, options, callback?) { }; isRequestProcessing = true; promise = new Promise(function (resolve) { - if (type == 'mmtf') { //binary data + if (type === 'mmtf') { //binary data getbin(uri) .then(function (ret) { handler(ret); From 3dbd73d8cec9cceabd74891b970cb5a2cbb11328 Mon Sep 17 00:00:00 2001 From: Prajwal Kulkarni Date: Wed, 28 Feb 2024 23:03:21 +0530 Subject: [PATCH 4/4] Remove throttling --- src/utilities.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/utilities.ts b/src/utilities.ts index f68034bc4..930c62982 100644 --- a/src/utilities.ts +++ b/src/utilities.ts @@ -345,7 +345,6 @@ function checkStatus(response) { return response; } -let isRequestProcessing = false; /** * Fetch data from URL * @@ -353,7 +352,7 @@ let isRequestProcessing = false; * @param callback Function to call with data */ export function get(uri:string, callback?) { - const promise = fetch(uri).then(checkStatus).then((response) => response.text()).finally(()=>isRequestProcessing = false); + const promise = fetch(uri).then(checkStatus).then((response) => response.text()); if (callback) return promise.then(callback); return promise; @@ -374,9 +373,7 @@ type RequestMethod = "GET"|"POST"|"PUT"|"DELETE"|"HEAD"|"OPTIONS"|"PATCH"; export function getbin(uri:string, callback?, request?: RequestMethod, postdata?) { const promise = fetch(uri, { method: request || "GET", body: postdata }) .then((response) => checkStatus(response)) - .then((response) => response.arrayBuffer()) - .finally(()=>isRequestProcessing = false); - + .then((response) => response.arrayBuffer()); if (callback) return promise.then(callback); return promise; @@ -409,11 +406,6 @@ export function download(query, viewer, options, callback?) { var promise = null; var m = viewer.addModel(); - if(isRequestProcessing){ - alert("Request already processing, please wait"); - return; - } - if (query.indexOf(':') < 0) { //no type specifier, guess if (query.length == 4) { @@ -432,7 +424,7 @@ export function download(query, viewer, options, callback?) { //when fetch directly from pdb, trust structure annotations options.noComputeSecondaryStructure = true; } - isRequestProcessing = true; + promise = new Promise(function (resolve) { getbin(uri) .then(function (ret) { @@ -486,7 +478,7 @@ export function download(query, viewer, options, callback?) { viewer.zoomTo(); viewer.render(); }; - isRequestProcessing = true; + promise = new Promise(function (resolve) { if (type === 'mmtf') { //binary data getbin(uri)