From 0f5d7f27a456edfeee7516a6ed6dd8d73b9bfc1f Mon Sep 17 00:00:00 2001 From: rejectliu Date: Sat, 29 Nov 2025 15:19:20 +0800 Subject: [PATCH 1/2] This PR adds support for string interpolation of `$var:`, `$res:`, and `$encrypted:` references within string values. Previously, these references were only replaced when they appeared as the entire value. Now they can be embedded within strings, enabling use cases like `"Bearer $var:api_token"` or `"https://api.example.com?key=$var:api_key"`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When using generated UI forms or HTTP request nodes, users often need to compose strings that include variable values. For example: ```javascript // Input in generated UI: var1: "$var:f/Narration/jina_api_key" api_key: "Bearer $var:f/Narration/jina_api_key" ``` Previously, the behavior was: - ✅ `var1` → Replaced with actual value: `"jina_4797cd5bb8f54314"` - ❌ `api_key` → **NOT replaced**, remained as literal string: `"Bearer $var:f/Narration/jina_api_key"` This limitation forced users to handle string concatenation in their scripts, which is problematic for generic HTTP request nodes where the code doesn't know which parameters need concatenation. This PR implements string interpolation for variable/resource/encrypted references: ```rust Value::String(y) if y.starts_with("$var:") => { // Only replaces if entire string is "$var:path" } ``` ```rust Value::String(y) if y.starts_with("$var:") && !y.contains(' ') => { // Exact match: entire string is a variable reference // Returns the variable value (could be any JSON type) } Value::String(y) if (*RE_RES_VAR).is_match(&y) => { // String interpolation: scan and replace all occurrences // Always returns a string with substitutions made } ``` The regex pattern `\$var:([^\s"'\},\)\]&;]+)` extracts variable paths, stopping at common delimiters: - Whitespace, quotes (`"`, `'`) - JSON/array terminators (`}`, `,`, `)`, `]`) - URL/query separators (`&`, `;`) This ensures proper extraction in contexts like: - JSON values: `{"Authorization": "$var:token"}` - URL parameters: `?key=$var:api_key&other=value` - Arrays: `["$var:item1", "$var:item2"]` - Function calls: `func($var:arg)` 1. **Added regex patterns** for extracting variable/resource/encrypted references: ```rust static ref RE_VAR_PATTERN: Regex = Regex::new(r#"\$var:([^\s"'\},\)\]&;]+)"#).unwrap(); static ref RE_RES_PATTERN: Regex = Regex::new(r#"\$res:([^\s"'\},\)\]&;]+)"#).unwrap(); static ref RE_ENCRYPTED_PATTERN: Regex = Regex::new(r#"\$encrypted:([^\s"'\},\)\]&;]+)"#).unwrap(); ``` 2. **Modified `transform_json_value()`** to handle both exact matches and string interpolation: - Exact match (e.g., `"$var:path"`) → Returns variable value as-is (preserves type) - String interpolation (e.g., `"Bearer $var:token"`) → Scans string, replaces all matches, returns string Applied the same changes to the API layer's `transform_json_value()` function to ensure consistency across all variable resolution paths. ```javascript // Input: "$var:f/config/timeout" // Variable value: 30 (number) // Output: 30 (preserves type) ``` ```javascript // Input: "Bearer $var:f/auth/api_token" // Variable value: "sk-abc123" // Output: "Bearer sk-abc123" (string) ``` ```javascript // Input: "$var:greeting $var:name" // Variables: greeting="Hello", name="World" // Output: "Hello World" ``` ```javascript // Input: "https://api.example.com?key=$var:api_key&user=$res:f/config/user_id" // Output: "https://api.example.com?key=abc123&user=42" ``` - If a referenced variable/resource doesn't exist, the job fails with a clear error message: ``` Variable f/auth/api_token not found in string interpolation for `Authorization`: ... ``` - Path validation is maintained (e.g., resources must have at least 2 path segments) Tested with various scenarios: - ✅ Single variable in string - ✅ Multiple variables in same string - ✅ Mixed `$var:` and `$res:` references - ✅ JSON contexts with quotes, brackets, braces - ✅ URL query parameters with `&` separators - ✅ Array/function contexts - ✅ Whitespace and special character boundaries - ✅ Non-existent variable error handling ✅ **Fully backward compatible** - Exact match behavior unchanged: `"$var:path"` still returns the variable value with original type - Existing scripts and workflows continue to work without modification - Only adds new functionality for strings containing embedded references ❌ **No frontend changes required** All variable substitution happens on the backend. Frontend components that display or edit `$var:` references remain unchanged. Fixes the issue where users couldn't use variable references within composed strings, particularly problematic for: - Generic HTTP request nodes - Authorization headers - API URL construction - Configuration templates --- - `backend/windmill-worker/src/common.rs` - Worker-side variable transformation - `backend/windmill-api/src/resources.rs` - API-side variable transformation --- backend/windmill-api/src/resources.rs | 91 ++++++++++++++++++++++++- backend/windmill-worker/src/common.rs | 96 +++++++++++++++++++++++++-- 2 files changed, 181 insertions(+), 6 deletions(-) diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 503c60e1f8b38..4cc22eaf79e12 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -43,6 +43,13 @@ use windmill_common::{ worker::{CLOUD_HOSTED, TMP_DIR}, workspaces::get_ducklake_instance_pg_catalog_password, }; +use regex::Regex; + +lazy_static::lazy_static! { + static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|res)\:"#).unwrap(); + static ref RE_VAR_PATTERN: Regex = Regex::new(r#"\$var:([^\s"'\},\)\]&;]+)"#).unwrap(); + static ref RE_RES_PATTERN: Regex = Regex::new(r#"\$res:([^\s"'\},\)\]&;]+)"#).unwrap(); +} pub fn workspaced_service() -> Router { Router::new() @@ -534,7 +541,8 @@ pub async fn transform_json_value<'c>( token: &str, ) -> Result { match v { - Value::String(y) if y.starts_with("$var:") => { + Value::String(y) if y.starts_with("$var:") && !y.contains(' ') => { + // Exact match: entire string is a variable reference let path = y.strip_prefix("$var:").unwrap(); let userdb_authed = UserDbWithOptAuthed { authed: authed, user_db: user_db.clone(), db: db.clone() }; @@ -558,7 +566,8 @@ pub async fn transform_json_value<'c>( .await?; Ok(Value::String(v)) } - Value::String(y) if y.starts_with("$res:") => { + Value::String(y) if y.starts_with("$res:") && !y.contains(' ') => { + // Exact match: entire string is a resource reference let path = y.strip_prefix("$res:").unwrap(); if path.split("/").count() < 2 { return Err(Error::internal_err(format!( @@ -582,6 +591,84 @@ pub async fn transform_json_value<'c>( Ok(Value::Null) } } + Value::String(y) if (*RE_RES_VAR).is_match(&y) => { + // String interpolation: contains variable/resource references + let mut result = y.clone(); + let userdb_authed = + UserDbWithOptAuthed { authed: authed, user_db: user_db.clone(), db: db.clone() }; + + // Replace $var: references + for cap in (*RE_VAR_PATTERN).captures_iter(&y) { + let full_match = cap.get(0).unwrap().as_str(); + let path = cap.get(1).unwrap().as_str(); + + match crate::variables::get_value_internal( + &userdb_authed, + db, + workspace, + path, + &user_db + .clone() + .map(|_| authed.into()) + .unwrap_or(AuditAuthor { + email: "backend".to_string(), + username: "backend".to_string(), + username_override: None, + token_prefix: None, + }), + false, + ) + .await + { + Ok(value) => { + result = result.replace(full_match, &value); + } + Err(e) => { + return Err(Error::NotFound(format!( + "Variable {path} not found in string interpolation: {e:#}" + ))); + } + } + } + + // Replace $res: references + for cap in (*RE_RES_PATTERN).captures_iter(&y) { + let full_match = cap.get(0).unwrap().as_str(); + let path = cap.get(1).unwrap().as_str(); + + if path.split("/").count() < 2 { + return Err(Error::internal_err(format!( + "String contains invalid resource path: {path}" + ))); + } + + let mut tx: Transaction<'_, Postgres> = + authed_transaction_or_default(authed, user_db.clone(), db).await?; + let v = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path, + &workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + if let Some(Some(v)) = v { + let value_str = if v.is_string() { + v.as_str().unwrap().to_string() + } else { + v.to_string() + }; + result = result.replace(full_match, &value_str); + } else { + return Err(Error::NotFound(format!( + "Resource {path} not found in string interpolation" + ))); + } + } + + Ok(Value::String(result)) + } Value::String(y) if y.starts_with("$") && job_id.is_some() => { let mut tx = authed_transaction_or_default(authed, user_db.clone(), db).await?; let job_id = job_id.unwrap(); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 4856e030f7fbc..e87411dd44dff 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -129,6 +129,9 @@ pub async fn write_file_binary(dir: &str, path: &str, content: &[u8]) -> error:: lazy_static::lazy_static! { static ref RE_RES_VAR: Regex = Regex::new(r#"\$(?:var|res|encrypted)\:"#).unwrap(); + static ref RE_VAR_PATTERN: Regex = Regex::new(r#"\$var:([^\s"'\},\)\]&;]+)"#).unwrap(); + static ref RE_RES_PATTERN: Regex = Regex::new(r#"\$res:([^\s"'\},\)\]&;]+)"#).unwrap(); + static ref RE_ENCRYPTED_PATTERN: Regex = Regex::new(r#"\$encrypted:([^\s"'\},\)\]&;]+)"#).unwrap(); } pub async fn transform_json<'a>( @@ -225,7 +228,8 @@ pub async fn transform_json_value( conn: &Connection, ) -> error::Result { match v { - Value::String(y) if y.starts_with("$var:") => { + Value::String(y) if y.starts_with("$var:") && !y.contains(' ') => { + // Exact match: entire string is a variable reference let path = y.strip_prefix("$var:").unwrap(); client .get_variable_value(path) @@ -235,7 +239,8 @@ pub async fn transform_json_value( Error::NotFound(format!("Variable {path} not found for `{name}`: {e:#}")) }) } - Value::String(y) if y.starts_with("$res:") => { + Value::String(y) if y.starts_with("$res:") && !y.contains(' ') => { + // Exact match: entire string is a resource reference let path = y.strip_prefix("$res:").unwrap(); if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") { @@ -253,7 +258,8 @@ pub async fn transform_json_value( Error::NotFound(format!("Resource {path} not found for `{name}`: {e:#}")) }) } - Value::String(y) if y.starts_with("$encrypted:") => { + Value::String(y) if y.starts_with("$encrypted:") && !y.contains(' ') => { + // Exact match: entire string is an encrypted reference match conn { Connection::Sql(db) => { let encrypted = y.strip_prefix("$encrypted:").unwrap(); @@ -277,8 +283,90 @@ pub async fn transform_json_value( Err(Error::NotFound("Http connection not supported".to_string())) } } + } + Value::String(y) if (*RE_RES_VAR).is_match(&y) => { + // String interpolation: contains variable/resource/encrypted references + let mut result = y.clone(); + + // Replace $var: references + for cap in (*RE_VAR_PATTERN).captures_iter(&y) { + let full_match = cap.get(0).unwrap().as_str(); + let path = cap.get(1).unwrap().as_str(); + + match client.get_variable_value(path).await { + Ok(value) => { + result = result.replace(full_match, &value); + } + Err(e) => { + return Err(Error::NotFound(format!( + "Variable {path} not found in string interpolation for `{name}`: {e:#}" + ))); + } + } + } + + // Replace $res: references + for cap in (*RE_RES_PATTERN).captures_iter(&y) { + let full_match = cap.get(0).unwrap().as_str(); + let path = cap.get(1).unwrap().as_str(); + + if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") { + return Err(Error::internal_err(format!( + "Argument `{name}` contains invalid resource path: {path}", + ))); + } + + match client + .get_resource_value_interpolated::( + path, + Some(job.id.to_string()), + ) + .await + { + Ok(value) => { + let value_str = if value.is_string() { + value.as_str().unwrap().to_string() + } else { + value.to_string() + }; + result = result.replace(full_match, &value_str); + } + Err(e) => { + return Err(Error::NotFound(format!( + "Resource {path} not found in string interpolation for `{name}`: {e:#}" + ))); + } + } + } + + // Replace $encrypted: references + if let Connection::Sql(db) = conn { + for cap in (*RE_ENCRYPTED_PATTERN).captures_iter(&y) { + let full_match = cap.get(0).unwrap().as_str(); + let encrypted = cap.get(1).unwrap().as_str(); + + let root_job_id = get_root_job_id(&job); + let mc = build_crypt_with_key_suffix( + db, + &job.workspace_id, + &root_job_id.to_string(), + ) + .await?; + + match decrypt(&mc, encrypted.to_string()) { + Ok(decrypted) => { + result = result.replace(full_match, &decrypted); + } + Err(e) => { + return Err(Error::internal_err(format!( + "Failed to decrypt '$encrypted:' value in string interpolation: {e}" + ))); + } + } + } + } - // let path = y.strip_prefix("$res:").unwrap(); + Ok(json!(result)) } Value::String(y) if y.starts_with("$") => { let variables = get_reserved_variables(job, &client.token, conn, None).await?; From 3388be84c864bcdb24a3fc3963f52d163a746f7e Mon Sep 17 00:00:00 2001 From: rejectliu Date: Sat, 29 Nov 2025 15:28:51 +0800 Subject: [PATCH 2/2] only support simple --- backend/windmill-api/src/resources.rs | 16 +++++++++++++++- backend/windmill-worker/src/common.rs | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 4cc22eaf79e12..5d5996c4e8ac8 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -656,8 +656,22 @@ pub async fn transform_json_value<'c>( if let Some(Some(v)) = v { let value_str = if v.is_string() { v.as_str().unwrap().to_string() - } else { + } else if v.is_number() { + v.to_string() + } else if v.is_boolean() { v.to_string() + } else if v.is_null() { + String::new() + } else { + // Object or Array - not suitable for string interpolation + return Err(Error::BadRequest(format!( + "Cannot interpolate resource '$res:{path}' into string. \ + The resource contains an object or array with multiple fields. \ + \nTo fix this:\n\ + 1. Use the resource as a separate parameter (e.g., set the whole parameter to '$res:{path}'), OR\n\ + 2. Create a simpler variable/resource with just the single value you need, OR\n\ + 3. Extract the field in your script code after receiving the full resource object." + ))); }; result = result.replace(full_match, &value_str); } else { diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index e87411dd44dff..6a97825dd2bbf 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -326,8 +326,22 @@ pub async fn transform_json_value( Ok(value) => { let value_str = if value.is_string() { value.as_str().unwrap().to_string() - } else { + } else if value.is_number() { + value.to_string() + } else if value.is_boolean() { value.to_string() + } else if value.is_null() { + String::new() + } else { + // Object or Array - not suitable for string interpolation + return Err(Error::BadRequest(format!( + "Cannot interpolate resource '$res:{path}' into string for argument `{name}`. \ + The resource contains an object or array with multiple fields. \ + \nTo fix this:\n\ + 1. Use the resource as a separate parameter (e.g., set the whole parameter to '$res:{path}'), OR\n\ + 2. Create a simpler variable/resource with just the single value you need, OR\n\ + 3. Extract the field in your script code after receiving the full resource object." + ))); }; result = result.replace(full_match, &value_str); }