From 93ec666abe1e0c45c22ce723734df7d161e88283 Mon Sep 17 00:00:00 2001
From: Hiroaki Nakamura <hnakamur@gmail.com>
Date: Tue, 30 Jul 2019 18:46:38 +0900
Subject: [PATCH 1/3] Add ngx_http_lua_ffi_shdict_store_when for set_when and
safe_set_when
---
README.markdown | 40 +++++
doc/HttpLuaModule.wiki | 32 ++++
src/ngx_http_lua_shdict.c | 302 ++++++++++++++++++++++++++++++++++++++
t/162-shdict-set-when.t | 139 ++++++++++++++++++
4 files changed, 513 insertions(+)
create mode 100644 t/162-shdict-set-when.t
diff --git a/README.markdown b/README.markdown
index 7902550d5e..922536739a 100644
--- a/README.markdown
+++ b/README.markdown
@@ -3261,6 +3261,8 @@ Nginx API for Lua
* [ngx.shared.DICT.get_stale](#ngxshareddictget_stale)
* [ngx.shared.DICT.set](#ngxshareddictset)
* [ngx.shared.DICT.safe_set](#ngxshareddictsafe_set)
+* [ngx.shared.DICT.set_when](#ngxshareddictset_when)
+* [ngx.shared.DICT.safe_set_when](#ngxshareddictsafe_set_when)
* [ngx.shared.DICT.add](#ngxshareddictadd)
* [ngx.shared.DICT.safe_add](#ngxshareddictsafe_add)
* [ngx.shared.DICT.replace](#ngxshareddictreplace)
@@ -6400,6 +6402,8 @@ The resulting object `dict` has the following methods:
* [get_stale](#ngxshareddictget_stale)
* [set](#ngxshareddictset)
* [safe_set](#ngxshareddictsafe_set)
+* [set_when](#ngxshareddictset_when)
+* [safe_set_when](#ngxshareddictsafe_set_when)
* [add](#ngxshareddictadd)
* [safe_add](#ngxshareddictsafe_add)
* [replace](#ngxshareddictreplace)
@@ -6590,6 +6594,42 @@ See also [ngx.shared.DICT](#ngxshareddict).
[Back to TOC](#nginx-api-for-lua)
+ngx.shared.DICT.set_when
+------------------------
+**syntax:** *success, err, forcible = ngx.shared.DICT:set(key, old_value, value, exptime?, flags?)*
+
+**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua**
+
+Just like the [set](#ngxshareddictset) method, but only stores the key-value pair into the dictionary [ngx.shared.DICT](#ngxshareddict) if the value for key is the same as `old_value`.
+
+If the `key` argument does *not* exist in the dictionary (or expired already), the `success` return value will be `true`.
+
+If the value for the key is *not* the same as `old_value`, the `success` return value will be `false` and the `err` value will be `"already modified"`.
+
+This feature was first introduced in the `v0.10.16rc1` release.
+
+See also [ngx.shared.DICT](#ngxshareddict).
+
+[Back to TOC](#nginx-api-for-lua)
+
+ngx.shared.DICT.safe_set_when
+-----------------------------
+**syntax:** *ok, err = ngx.shared.DICT:safe_set_when(key, old_value, value, exptime?, flags?)*
+
+**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua**
+
+Similar to the [set_when](#ngxshareddictset_when) method, but never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return `nil` and the string "no memory".
+
+If the `key` argument does *not* exist in the dictionary (or expired already), the `success` return value will be `true`.
+
+If the value for the key is *not* the same as `old_value`, the `success` return value will be `false` and the `err` value will be `"already modified"`.
+
+This feature was first introduced in the `v0.10.16rc1` release.
+
+See also [ngx.shared.DICT](#ngxshareddict).
+
+[Back to TOC](#nginx-api-for-lua)
+
ngx.shared.DICT.add
-------------------
diff --git a/doc/HttpLuaModule.wiki b/doc/HttpLuaModule.wiki
index 1993b05a5d..1617f599d4 100644
--- a/doc/HttpLuaModule.wiki
+++ b/doc/HttpLuaModule.wiki
@@ -5393,6 +5393,8 @@ The resulting object <code>dict</code> has the following methods:
* [[#ngx.shared.DICT.get_stale|get_stale]]
* [[#ngx.shared.DICT.set|set]]
* [[#ngx.shared.DICT.safe_set|safe_set]]
+* [[#ngx.shared.DICT.set_when|set_when]]
+* [[#ngx.shared.DICT.safe_set_when|safe_set_when]]
* [[#ngx.shared.DICT.add|add]]
* [[#ngx.shared.DICT.safe_add|safe_add]]
* [[#ngx.shared.DICT.replace|replace]]
@@ -5563,6 +5565,36 @@ This feature was first introduced in the <code>v0.7.18</code> release.
See also [[#ngx.shared.DICT|ngx.shared.DICT]].
+== ngx.shared.DICT.set_when ==
+'''syntax:''' ''success, err, forcible = ngx.shared.DICT:set(key, old_value, value, exptime?, flags?)''
+
+'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*''
+
+Just like the [[#ngx.shared.DICT.set|set]] method, but only stores the key-value pair into the dictionary [[#ngx.shared.DICT|ngx.shared.DICT]] if the value for key is the same as <code>old_value</code>.
+
+If the <code>key</code> argument does ''not'' exist in the dictionary (or expired already), the <code>success</code> return value will be <code>true</code>.
+
+If the value for the key is ''not'' the same as <code>old_value</code>, the <code>success</code> return value will be <code>false</code> and the <code>err</code> value will be <code>"already modified"</code>.
+
+This feature was first introduced in the <code>v0.10.16rc1</code> release.
+
+See also [[#ngx.shared.DICT|ngx.shared.DICT]].
+
+== ngx.shared.DICT.safe_set_when ==
+'''syntax:''' ''ok, err = ngx.shared.DICT:safe_set_when(key, old_value, value, exptime?, flags?)''
+
+'''context:''' ''init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*''
+
+Similar to the [[#ngx.shared.DICT.set_when|set_when]] method, but never overrides the (least recently used) unexpired items in the store when running out of storage in the shared memory zone. In this case, it will immediately return <code>nil</code> and the string "no memory".
+
+If the <code>key</code> argument does ''not'' exist in the dictionary (or expired already), the <code>success</code> return value will be <code>true</code>.
+
+If the value for the key is ''not'' the same as <code>old_value</code>, the <code>success</code> return value will be <code>false</code> and the <code>err</code> value will be <code>"already modified"</code>.
+
+This feature was first introduced in the <code>v0.10.16rc1</code> release.
+
+See also [[#ngx.shared.DICT|ngx.shared.DICT]].
+
== ngx.shared.DICT.add ==
'''syntax:''' ''success, err, forcible = ngx.shared.DICT:add(key, value, exptime?, flags?)''
diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c
index 2fd2a94c13..e6fe6ed583 100644
--- a/src/ngx_http_lua_shdict.c
+++ b/src/ngx_http_lua_shdict.c
@@ -1569,6 +1569,308 @@ ngx_http_lua_ffi_shdict_store(ngx_shm_zone_t *zone, int op, u_char *key,
}
+int
+ngx_http_lua_ffi_shdict_store_when(ngx_shm_zone_t *zone, int op, u_char *key,
+ size_t key_len, int old_value_type, u_char *old_str_value_buf,
+ size_t old_str_value_len, double old_num_value, int value_type,
+ u_char *str_value_buf, size_t str_value_len, double num_value,
+ long exptime, int user_flags, char **errmsg, int *forcible)
+{
+ int i, n;
+ u_char old_c, c, *p;
+ uint32_t hash;
+ ngx_int_t rc;
+ ngx_time_t *tp;
+ ngx_queue_t *queue, *q;
+ ngx_rbtree_node_t *node;
+ ngx_http_lua_shdict_ctx_t *ctx;
+ ngx_http_lua_shdict_node_t *sd;
+
+ dd("exptime: %ld", exptime);
+
+ ctx = zone->data;
+
+ *forcible = 0;
+
+ hash = ngx_crc32_short(key, key_len);
+
+ switch (old_value_type) {
+
+ case SHDICT_TSTRING:
+ /* do nothing */
+ break;
+
+ case SHDICT_TNUMBER:
+ dd("num value: %lf", old_num_value);
+ old_str_value_buf = (u_char *) &old_num_value;
+ old_str_value_len = sizeof(double);
+ break;
+
+ case SHDICT_TBOOLEAN:
+ old_c = old_num_value ? 1 : 0;
+ old_str_value_buf = &old_c;
+ old_str_value_len = sizeof(u_char);
+ break;
+
+ case LUA_TNIL:
+ old_str_value_buf = NULL;
+ old_str_value_len = 0;
+ break;
+
+ default:
+ *errmsg = "unsupported old_value type";
+ return NGX_ERROR;
+ }
+
+ switch (value_type) {
+
+ case SHDICT_TSTRING:
+ /* do nothing */
+ break;
+
+ case SHDICT_TNUMBER:
+ dd("num value: %lf", num_value);
+ str_value_buf = (u_char *) &num_value;
+ str_value_len = sizeof(double);
+ break;
+
+ case SHDICT_TBOOLEAN:
+ c = num_value ? 1 : 0;
+ str_value_buf = &c;
+ str_value_len = sizeof(u_char);
+ break;
+
+ case LUA_TNIL:
+ if (op & (NGX_HTTP_LUA_SHDICT_ADD|NGX_HTTP_LUA_SHDICT_REPLACE)) {
+ *errmsg = "attempt to add or replace nil values";
+ return NGX_ERROR;
+ }
+
+ str_value_buf = NULL;
+ str_value_len = 0;
+ break;
+
+ default:
+ *errmsg = "unsupported value type";
+ return NGX_ERROR;
+ }
+
+ ngx_shmtx_lock(&ctx->shpool->mutex);
+
+#if 1
+ ngx_http_lua_shdict_expire(ctx, 1);
+#endif
+
+ rc = ngx_http_lua_shdict_lookup(zone, hash, key, key_len, &sd);
+
+ dd("lookup returns %d", (int) rc);
+
+ if (op & NGX_HTTP_LUA_SHDICT_REPLACE) {
+
+ if (rc == NGX_DECLINED || rc == NGX_DONE) {
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+ *errmsg = "not found";
+ return NGX_DECLINED;
+ }
+
+ /* rc == NGX_OK */
+
+ goto replace;
+ }
+
+ if (op & NGX_HTTP_LUA_SHDICT_ADD) {
+
+ if (rc == NGX_OK) {
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+ *errmsg = "exists";
+ return NGX_DECLINED;
+ }
+
+ if (rc == NGX_DONE) {
+ /* exists but expired */
+
+ dd("go to replace");
+ goto replace;
+ }
+
+ /* rc == NGX_DECLINED */
+
+ dd("go to insert");
+ goto insert;
+ }
+
+ if (rc == NGX_OK || rc == NGX_DONE) {
+
+ if (sd->value_type != old_value_type
+ || (size_t) sd->value_len != old_str_value_len
+ || memcmp(sd->data + sd->key_len, old_str_value_buf, sd->value_len) != 0)
+ {
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+ *errmsg = "already modified";
+ return NGX_ERROR;
+ }
+
+ if (value_type == LUA_TNIL) {
+ goto remove;
+ }
+
+replace:
+
+ if (str_value_buf
+ && str_value_len == (size_t) sd->value_len
+ && sd->value_type != SHDICT_TLIST)
+ {
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
+ "lua shared dict set: found old entry and value "
+ "size matched, reusing it");
+
+ ngx_queue_remove(&sd->queue);
+ ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);
+
+ sd->key_len = (u_short) key_len;
+
+ if (exptime > 0) {
+ tp = ngx_timeofday();
+ sd->expires = (uint64_t) tp->sec * 1000 + tp->msec
+ + (uint64_t) exptime;
+
+ } else {
+ sd->expires = 0;
+ }
+
+ sd->user_flags = user_flags;
+
+ sd->value_len = (uint32_t) str_value_len;
+
+ dd("setting value type to %d", value_type);
+
+ sd->value_type = (uint8_t) value_type;
+
+ p = ngx_copy(sd->data, key, key_len);
+ ngx_memcpy(p, str_value_buf, str_value_len);
+
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+
+ return NGX_OK;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
+ "lua shared dict set: found old entry but value size "
+ "NOT matched, removing it first");
+
+remove:
+
+ if (sd->value_type == SHDICT_TLIST) {
+ queue = ngx_http_lua_shdict_get_list_head(sd, key_len);
+
+ for (q = ngx_queue_head(queue);
+ q != ngx_queue_sentinel(queue);
+ q = ngx_queue_next(q))
+ {
+ p = (u_char *) ngx_queue_data(q,
+ ngx_http_lua_shdict_list_node_t,
+ queue);
+
+ ngx_slab_free_locked(ctx->shpool, p);
+ }
+ }
+
+ ngx_queue_remove(&sd->queue);
+
+ node = (ngx_rbtree_node_t *)
+ ((u_char *) sd - offsetof(ngx_rbtree_node_t, color));
+
+ ngx_rbtree_delete(&ctx->sh->rbtree, node);
+
+ ngx_slab_free_locked(ctx->shpool, node);
+
+ }
+
+insert:
+
+ /* rc == NGX_DECLINED or value size unmatch */
+
+ if (str_value_buf == NULL) {
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+ return NGX_OK;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
+ "lua shared dict set: creating a new entry");
+
+ n = offsetof(ngx_rbtree_node_t, color)
+ + offsetof(ngx_http_lua_shdict_node_t, data)
+ + key_len
+ + str_value_len;
+
+ node = ngx_slab_alloc_locked(ctx->shpool, n);
+
+ if (node == NULL) {
+
+ if (op & NGX_HTTP_LUA_SHDICT_SAFE_STORE) {
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+
+ *errmsg = "no memory";
+ return NGX_ERROR;
+ }
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
+ "lua shared dict set: overriding non-expired items "
+ "due to memory shortage for entry \"%*s\"", key_len,
+ key);
+
+ for (i = 0; i < 30; i++) {
+ if (ngx_http_lua_shdict_expire(ctx, 0) == 0) {
+ break;
+ }
+
+ *forcible = 1;
+
+ node = ngx_slab_alloc_locked(ctx->shpool, n);
+ if (node != NULL) {
+ goto allocated;
+ }
+ }
+
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+
+ *errmsg = "no memory";
+ return NGX_ERROR;
+ }
+
+allocated:
+
+ sd = (ngx_http_lua_shdict_node_t *) &node->color;
+
+ node->key = hash;
+ sd->key_len = (u_short) key_len;
+
+ if (exptime > 0) {
+ tp = ngx_timeofday();
+ sd->expires = (uint64_t) tp->sec * 1000 + tp->msec
+ + (uint64_t) exptime;
+
+ } else {
+ sd->expires = 0;
+ }
+
+ sd->user_flags = user_flags;
+ sd->value_len = (uint32_t) str_value_len;
+ dd("setting value type to %d", value_type);
+ sd->value_type = (uint8_t) value_type;
+
+ p = ngx_copy(sd->data, key, key_len);
+ ngx_memcpy(p, str_value_buf, str_value_len);
+
+ ngx_rbtree_insert(&ctx->sh->rbtree, node);
+ ngx_queue_insert_head(&ctx->sh->lru_queue, &sd->queue);
+ ngx_shmtx_unlock(&ctx->shpool->mutex);
+
+ return NGX_OK;
+}
+
+
int
ngx_http_lua_ffi_shdict_get(ngx_shm_zone_t *zone, u_char *key,
size_t key_len, int *value_type, u_char **str_value_buf,
diff --git a/t/162-shdict-set-when.t b/t/162-shdict-set-when.t
new file mode 100644
index 0000000000..b9701baed9
--- /dev/null
+++ b/t/162-shdict-set-when.t
@@ -0,0 +1,139 @@
+# vim:set ft= ts=4 sw=4 et fdm=marker:
+use Test::Nginx::Socket::Lua;
+
+#worker_connections(1014);
+#master_process_enabled(1);
+#log_level('warn');
+
+#repeat_each(2);
+
+plan tests => repeat_each() * (blocks() * 3);
+
+#no_diff();
+no_long_string();
+#master_on();
+#workers(2);
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set_when success
+--- http_config
+ lua_shared_dict dogs 1m;
+--- config
+ location = /test {
+ content_by_lua '
+ local dogs = ngx.shared.dogs
+ dogs:set("foo", 32)
+ dogs:set_when("foo", 32, 33)
+ local val = dogs:get("foo")
+ ngx.say(val, " ", type(val))
+ ';
+ }
+--- request
+GET /test
+--- response_body
+33 number
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: set_when fail
+--- http_config
+ lua_shared_dict dogs 1m;
+--- config
+ location = /test {
+ content_by_lua '
+ local dogs = ngx.shared.dogs
+ dogs:set("foo", 32)
+ local ok, err, forcible = dogs:set_when("foo", 32, 33)
+ ngx.say(ok, " ", err, " ", forcible)
+ local ok, err, forcible = dogs:set_when("foo", 32, 34)
+ ngx.say(ok, " ", err, " ", forcible)
+ local val = dogs:get("foo")
+ ngx.say(val, " ", type(val))
+ ';
+ }
+--- request
+GET /test
+--- response_body
+true nil false
+false already modified false
+33 number
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: set_when success for expired value
+--- http_config
+ lua_shared_dict dogs 1m;
+--- config
+ location = /test {
+ content_by_lua '
+ local dogs = ngx.shared.dogs
+ dogs:set("foo", 32, 0.01)
+ ngx.sleep(0.02)
+ local ok, err, forcible = dogs:set_when("foo", 32, 33)
+ ngx.say(ok, " ", err, " ", forcible)
+ local val = dogs:get("foo")
+ ngx.say(val, " ", type(val))
+ ';
+ }
+--- request
+GET /test
+--- response_body
+true nil false
+33 number
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: set_when success for unmatched expired value
+--- http_config
+ lua_shared_dict dogs 1m;
+--- config
+ location = /test {
+ content_by_lua '
+ local dogs = ngx.shared.dogs
+ dogs:set("foo", 32, 0.01)
+ ngx.sleep(0.02)
+ local ok, err, forcible = dogs:set_when("foo", 31, 33)
+ ngx.say(ok, " ", err, " ", forcible)
+ local val = dogs:get("foo")
+ ngx.say(val, " ", type(val))
+ ';
+ }
+--- request
+GET /test
+--- response_body
+true nil false
+33 number
+--- no_error_log
+[error]
+
+
+
+=== TEST 5: set_when success when old_value did not exist
+--- http_config
+ lua_shared_dict dogs 1m;
+--- config
+ location = /test {
+ content_by_lua '
+ local dogs = ngx.shared.dogs
+ local ok, err, forcible = dogs:set_when("foo", 32, 33)
+ ngx.say(ok, " ", err, " ", forcible)
+ local val = dogs:get("foo")
+ ngx.say(val, " ", type(val))
+ ';
+ }
+--- request
+GET /test
+--- response_body
+true nil false
+33 number
+--- no_error_log
+[error]
From 86b9f1e1f2e5dd30329a6bdf3aec09f4bf3ad85a Mon Sep 17 00:00:00 2001
From: Hiroaki Nakamura <hnakamur@gmail.com>
Date: Wed, 7 Aug 2019 15:48:49 +0000
Subject: [PATCH 2/3] Fold the line exceeding 80 columns
---
src/ngx_http_lua_shdict.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c
index e6fe6ed583..7f5b75a037 100644
--- a/src/ngx_http_lua_shdict.c
+++ b/src/ngx_http_lua_shdict.c
@@ -1703,7 +1703,8 @@ ngx_http_lua_ffi_shdict_store_when(ngx_shm_zone_t *zone, int op, u_char *key,
if (sd->value_type != old_value_type
|| (size_t) sd->value_len != old_str_value_len
- || memcmp(sd->data + sd->key_len, old_str_value_buf, sd->value_len) != 0)
+ || memcmp(sd->data + sd->key_len, old_str_value_buf,
+ sd->value_len) != 0)
{
ngx_shmtx_unlock(&ctx->shpool->mutex);
*errmsg = "already modified";
From 7c81f9560fc509ffe807b7e02e148c8b91cd1456 Mon Sep 17 00:00:00 2001
From: Hiroaki Nakamura <hnakamur@gmail.com>
Date: Thu, 8 Aug 2019 14:07:04 +0000
Subject: [PATCH 3/3] Add 2 to shdict metatable count for set_when and
safe_set_when
---
t/062-count.t | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/t/062-count.t b/t/062-count.t
index b481763abf..c3153a75d3 100644
--- a/t/062-count.t
+++ b/t/062-count.t
@@ -283,7 +283,7 @@ n = 5
--- request
GET /test
--- response_body
-n = 22
+n = 24
--- no_error_log
[error]