bug: panic: vnode 161 should not be accessed by this table

[fuyufjh]

Describe the bug

No response

Error message/log

thread 'rw-streaming' panicked at src/common/src/hash/table_distribution.rs:278:5:
vnode 161 should not be accessed by this table
stack backtrace:
   0:     0x564faf430b49 - std::backtrace_rs::backtrace::libunwind::trace::hb2aae96f7a63ad27
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5
   1:     0x564faf430b49 - std::backtrace_rs::backtrace::trace_unsynchronized::h21f79cd140dff600
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x564faf430b49 - std::sys_common::backtrace::_print_fmt::hc8ebdc1b64ae52ee
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:68:5
   3:     0x564faf430b49 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h7ba491a968dfae1d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x564fa2c3e03b - core::fmt::rt::Argument::fmt::he322040f856802e1
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/fmt/rt.rs:165:63
   5:     0x564fa2c3e03b - core::fmt::write::hb4f7da2a314ef133
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/fmt/mod.rs:1168:21
   6:     0x564faf4297cf - std::io::Write::write_fmt::h9e79d11af9b4686b
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/io/mod.rs:1835:15
   7:     0x564faf43091e - std::sys_common::backtrace::_print::h9276d72631ea2ebd
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x564faf43091e - std::sys_common::backtrace::print::h301edb80f46c58f8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x564faf431c89 - std::panicking::default_hook::{{closure}}::h08c68fc043d12d43
  10:     0x564faf431a35 - std::panicking::default_hook::hdf65558cafad5039
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:298:9
  11:     0x564fac889abb - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::hdc9eda93008da126
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2077:9
  12:     0x564fac889abb - risingwave_rt::panic_hook::set_panic_hook::{{closure}}::h40b35e75a058830c
                               at /risingwave/src/utils/runtime/src/panic_hook.rs:25:9
  13:     0x564fac889abb - std::panicking::update_hook::{{closure}}::h68c4945031a81df0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:233:47
  14:     0x564faf4322cb - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h69f90a497fd344b0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2077:9
  15:     0x564faf4322cb - std::panicking::rust_panic_with_hook::h9576f09a7a058c94
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:799:13
  16:     0x564faf432074 - std::panicking::begin_panic_handler::{{closure}}::hd8f56243f630c203
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:664:13
  17:     0x564faf431009 - std::sys_common::backtrace::__rust_end_short_backtrace::h5bed6f770a09acf8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:171:18
  18:     0x564faf431db7 - rust_begin_unwind
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:652:5
  19:     0x564fa2c3a593 - core::panicking::panic_fmt::h088f9f84f0369284
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panicking.rs:72:14
  20:     0x564fae4928ce - risingwave_common::hash::table_distribution::check_vnode_is_set::h9a2a1997ee2edfd7
                               at /risingwave/src/common/src/hash/table_distribution.rs:278:5
  21:     0x564fae4928ce - risingwave_common::hash::table_distribution::compute_vnode::ha4d64fd06e5f7134
                               at /risingwave/src/common/src/hash/table_distribution.rs:207:5
  22:     0x564fad205b01 - risingwave_common::hash::table_distribution::TableDistribution::try_compute_vnode_by_pk_prefix::{{closure}}::hc06e2b21e417f30a
                               at /risingwave/src/common/src/hash/table_distribution.rs:185:26
  23:     0x564fad205b01 - core::bool::<impl bool>::then::h451568f55131b42a
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/bool.rs:60:24
  24:     0x564fad205b01 - risingwave_common::hash::table_distribution::TableDistribution::try_compute_vnode_by_pk_prefix::h2f236c01155816ee
                               at /risingwave/src/common/src/hash/table_distribution.rs:185:18
  25:     0x564fad205b01 - risingwave_stream::common::table::state_table::StateTableInner<S,SD,_,_>::compute_prefix_vnode::he0776be36ab0a1e8
                               at /risingwave/src/stream/src/common/table/state_table.rs:731:9
  26:     0x564fad205b01 - risingwave_stream::common::table::state_table::StateTableInner<S,SD,_,_>::iter_with_prefix_inner::{{closure}}::h4b3575c56b3f2588
                               at /risingwave/src/stream/src/common/table/state_table.rs:1503:21
  27:     0x564fad205b01 - risingwave_stream::common::table::state_table::StateTableInner<S,SD,_,_>::iter_with_prefix::{{closure}}::haaa7a990a728e504
                               at /risingwave/src/stream/src/common/table/state_table.rs:1477:14
  28:     0x564fad3d4780 - risingwave_stream::executor::join::hash_join::JoinHashMap<K,S>::fetch_cached_state::{{closure}}::hd36aa65a556c4f53
                               at /risingwave/src/stream/src/executor/join/hash_join.rs:480:18
  29:     0x564fad3a5205 - risingwave_stream::executor::join::hash_join::JoinHashMap<K,S>::take_state::{{closure}}::h96b2c49feb66a9c0
                               at /risingwave/src/stream/src/executor/join/hash_join.rs:334:42
  30:     0x564fad3a29f5 - risingwave_stream::executor::hash_join::HashJoinExecutor<K,S,_>::hash_eq_match::{{closure}}::h1d7c539aefbd2cc8
                               at /risingwave/src/stream/src/executor/hash_join.rs:734:32
  31:     0x564fad4651bb - risingwave_stream::executor::hash_join::HashJoinExecutor<K,S,_>::eq_join_oneside::{{closure}}::h32e530e942b7beb7
                               at /risingwave/src/stream/src/executor/hash_join.rs:831:62
  32:     0x564facfbbb5e - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hc78cb1068720123e
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  33:     0x564facfbbb5e - risingwave_stream::executor::hash_join::HashJoinExecutor<K,S,_>::into_stream::{{closure}}::h852778de2073ed67
                               at /risingwave/src/stream/src/executor/hash_join.rs:456:5
  34:     0x564facfbbb5e - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h56f065eb76a10181
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  35:     0x564fad68ac16 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h877210e014f83086
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  36:     0x564fad68ac16 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h681c70e5a7afe59b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  37:     0x564fad68ac16 - futures_util::stream::stream::StreamExt::poll_next_unpin::h15756315efa1e456
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  38:     0x564fad68ac16 - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::hfa1c0da8fabde435
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  39:     0x564fad68ac16 - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::h45e18e5fe94c5920
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15
  40:     0x564fae591d2f - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::hebad1b901283d44d
                               at /risingwave/src/stream/src/executor/wrapper/trace.rs:116:10
  41:     0x564fae591d2f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h9709172f222f492a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  42:     0x564fae591d2f - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::h72a3911c3bfd96ae
                               at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1
  43:     0x564fae591d2f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h1d1e58437526b7c9
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  44:     0x564fae591d2f - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::he6672af7c2ca9b86
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  45:     0x564fae591d2f - futures_util::stream::stream::StreamExt::poll_next_unpin::h42180eab5f9aa475
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  46:     0x564fae58ca9c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h5520e11f52fe7ce2
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  47:     0x564fae58ca9c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h4e4c61dc90595786
                               at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44
  48:     0x564fae58ca9c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hca9041c664a7e3ec
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  49:     0x564fae58ca9c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf1693f253d16c06b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  50:     0x564fae58ca9c - <S as futures_core::stream::TryStream>::try_poll_next::h134f27024c83896f
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9
  51:     0x564fae58ca9c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::hb80b4c01fd8d9fe7
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9
  52:     0x564fae0e6a88 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::hfe16a339cdfaf25d
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9
  53:     0x564fae0e6a88 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h75a3f99b866c3950
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
  54:     0x564fae0e6a88 - tokio::task::task_local::LocalKey<T>::scope_inner::h1c9b72d05d695686
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
  55:     0x564fae0e6a88 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h2c9e1bc24af93afb
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
  56:     0x564fae591809 - risingwave_common::util::epoch::task_local::scope::{{closure}}::hc84ce08551f641b1
                               at /risingwave/src/common/src/util/epoch.rs:249:47
  57:     0x564fae591809 - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::hb1f1241002054386
                               at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59
  58:     0x564fae591809 - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5500c77672825964
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  59:     0x564fae591809 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h76f7525e2d9513c0
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  60:     0x564fae591809 - futures_util::stream::stream::StreamExt::poll_next_unpin::h3b929460a23cf7c8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  61:     0x564fadb6f89c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::hadad9091fb2824aa
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  62:     0x564fadb6f89c - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::hab3ce426a75b0910
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
  63:     0x564fae524faf - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::hba31a4be5ac83fb4
                               at /risingwave/src/stream/src/executor/wrapper/trace.rs:53:69
  64:     0x564fae524faf - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h62e59e52d0e077b8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  65:     0x564fae6655c1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h77fc8b9b61e6d7d4
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  66:     0x564fad7901d8 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h877210e014f83086
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  67:     0x564fad7901d8 - risingwave_stream::executor::project::Inner::execute::{{closure}}::hc98812bc6a5a8645
                               at /risingwave/src/stream/src/executor/project.rs:147:5
  68:     0x564fad7901d8 - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4cedcccc1ceedf01
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  69:     0x564fad68ac16 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h877210e014f83086
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  70:     0x564fad68ac16 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h681c70e5a7afe59b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  71:     0x564fad68ac16 - futures_util::stream::stream::StreamExt::poll_next_unpin::h15756315efa1e456
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  72:     0x564fad68ac16 - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::hfa1c0da8fabde435
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  73:     0x564fad68ac16 - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::h45e18e5fe94c5920
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15
  74:     0x564fae591d2f - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::hebad1b901283d44d
                               at /risingwave/src/stream/src/executor/wrapper/trace.rs:116:10
  75:     0x564fae591d2f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h9709172f222f492a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  76:     0x564fae591d2f - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::h72a3911c3bfd96ae
                               at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1
  77:     0x564fae591d2f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h1d1e58437526b7c9
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  78:     0x564fae591d2f - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::he6672af7c2ca9b86
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  79:     0x564fae591d2f - futures_util::stream::stream::StreamExt::poll_next_unpin::h42180eab5f9aa475
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  80:     0x564fae58ca9c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h5520e11f52fe7ce2
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  81:     0x564fae58ca9c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h4e4c61dc90595786
                               at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44
  82:     0x564fae58ca9c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hca9041c664a7e3ec
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  83:     0x564fae58ca9c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf1693f253d16c06b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  84:     0x564fae58ca9c - <S as futures_core::stream::TryStream>::try_poll_next::h134f27024c83896f
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9
  85:     0x564fae58ca9c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::hb80b4c01fd8d9fe7
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9
  86:     0x564fae0e6a88 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::hfe16a339cdfaf25d
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9
  87:     0x564fae0e6a88 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h75a3f99b866c3950
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
  88:     0x564fae0e6a88 - tokio::task::task_local::LocalKey<T>::scope_inner::h1c9b72d05d695686
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
  89:     0x564fae0e6a88 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h2c9e1bc24af93afb
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
  90:     0x564fae591809 - risingwave_common::util::epoch::task_local::scope::{{closure}}::hc84ce08551f641b1
                               at /risingwave/src/common/src/util/epoch.rs:249:47
  91:     0x564fae591809 - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::hb1f1241002054386
                               at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59
  92:     0x564fae591809 - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5500c77672825964
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  93:     0x564fae591809 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h76f7525e2d9513c0
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  94:     0x564fae591809 - futures_util::stream::stream::StreamExt::poll_next_unpin::h3b929460a23cf7c8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  95:     0x564fadb6f89c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::hadad9091fb2824aa
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  96:     0x564fadb6f89c - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::hab3ce426a75b0910
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
  97:     0x564fae524faf - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::hba31a4be5ac83fb4
                               at /risingwave/src/stream/src/executor/wrapper/trace.rs:53:69
  98:     0x564fae524faf - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h62e59e52d0e077b8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
  99:     0x564fae6655c1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h77fc8b9b61e6d7d4
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
 100:     0x564fae6613fa - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h877210e014f83086
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
 101:     0x564fae6613fa - <risingwave_stream::executor::dispatch::DispatchExecutor as risingwave_stream::executor::StreamConsumer>::execute::{{closure}}::ha14ea788273c23aa
                               at /risingwave/src/stream/src/executor/dispatch.rs:392:9
 102:     0x564fae6613fa - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4967b28bad99f93b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33
 103:     0x564fae6613fa - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h60e950d8716d5bd4
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
 104:     0x564fadb530f5 - <&mut S as futures_core::stream::Stream>::poll_next::hbec0f93f557f81c8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:104:9
 105:     0x564fadb530f5 - <tokio_stream::stream_ext::next::Next<St> as core::future::future::Future>::poll::h7854cb4274638a82
                               at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/next.rs:42:29
 106:     0x564fadb530f5 - <tokio_stream::stream_ext::try_next::TryNext<St> as core::future::future::Future>::poll::h0d81f28e33744a8f
                               at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/try_next.rs:43:9
 107:     0x564fadb530f5 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h05315484fbd35513
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
 108:     0x564fad6b62f0 - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::hd8ca51a80ba1bf4e
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15
 109:     0x564fae50f040 - risingwave_stream::executor::actor::Actor<C>::run_consumer::{{closure}}::hc3aa4293a065e518
                               at /risingwave/src/stream/src/executor/actor.rs:233:18
 110:     0x564fae50f040 - <tokio::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll::h7bdd305c3ba058ab
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/maybe_done.rs:62:56
 111:     0x564faea0f5ce - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::{{closure}}::h6d7bf9e921588a60
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/macros/join.rs:126:24
 112:     0x564faea0f5ce - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::he86aba77d63a0da9
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/poll_fn.rs:58:9
 113:     0x564fae0e7083 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::ha29edd3e4658f3d1
                               at /risingwave/src/stream/src/executor/actor.rs:183:17
 114:     0x564fae0e7083 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::he7c18ac053d043c4
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
 115:     0x564fae0e7083 - tokio::task::task_local::LocalKey<T>::scope_inner::ha003aa03a957afde
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
 116:     0x564fae0e7083 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h1b89a8676a2ab7b5
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
 117:     0x564fae0e7083 - risingwave_expr::expr_context::expr_context_scope::{{closure}}::h35058136f8cf2c85
                               at /risingwave/src/expr/core/src/expr_context.rs:35:65
 118:     0x564fae0e7083 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::haebb35a4b5c08d92
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
 119:     0x564fae0e7083 - tokio::task::task_local::LocalKey<T>::scope_inner::h04b0b9ac6cb8d6a0
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
 120:     0x564fae0e7083 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h386e203994fffbb5
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
 121:     0x564fadd125b4 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::h6658df0a8e9f4cc4
                               at /risingwave/src/stream/src/executor/actor.rs:191:10
 122:     0x564fadd125b4 - <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll::h84cbcd3df661c31c
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/map.rs:55:37
 123:     0x564fae0e9218 - <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll::hc6794a45635af4fd
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13
 124:     0x564fae0e9218 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h126983cabf4156be
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
 125:     0x564fae0e9218 - tokio::task::task_local::LocalKey<T>::scope_inner::h93b6ec2049fce605
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
 126:     0x564fae0e9218 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::he26fcdc833bfafc8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
 127:     0x564fae50bec9 - await_tree::root::TreeRoot::instrument::{{closure}}::h51146d18502e7182
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/root.rs:43:34
 128:     0x564fae50bec9 - <futures_util::future::either::Either<A,B> as core::future::future::Future>::poll::h9d1c616b473e65c2
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/either.rs:109:32
 129:     0x564fae50bec9 - core::ops::function::FnOnce::call_once::h2b5fc596489e8c2e
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/ops/function.rs:250:5
 130:     0x564fae50bec9 - tokio_metrics::task::instrument_poll::h736daf4a148d6fdb
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2530:15
 131:     0x564fae50bec9 - <tokio_metrics::task::Instrumented<T> as core::future::future::Future>::poll::h53b59574a6f7e4f5
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2430:9
 132:     0x564fae0e93cb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::ha425603f9d1ed85a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31
 133:     0x564fae0e93cb - tokio::task::task_local::LocalKey<T>::scope_inner::hdd8745bd7066ac13
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19
 134:     0x564fae0e93cb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hffa27a2cc74947b5
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19
 135:     0x564fadae9e2a - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h9ef3e9f409d8784a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
 136:     0x564fadae9e2a - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h475eb47822a7ed50
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
 137:     0x564fadae9e2a - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h515332b5f880508d
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
 138:     0x564fadae9e2a - tokio::runtime::task::core::Core<T,S>::poll::h92c44397ff9abfa9
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30
 139:     0x564fae89cc1e - tokio::runtime::task::harness::poll_future::{{closure}}::hd9c10635ea920de6
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
 140:     0x564fae89cc1e - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h9490558460e885f3
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
 141:     0x564fae89cc1e - std::panicking::try::do_call::h5757132b82d2e057
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
 142:     0x564fae89cc1e - std::panicking::try::h1df45c3c5d92f2ef
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
 143:     0x564fae89cc1e - std::panic::catch_unwind::hbf38cac8e5d76d0d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
 144:     0x564fae89cc1e - tokio::runtime::task::harness::poll_future::hca6407c80e836336
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
 145:     0x564fae89cc1e - tokio::runtime::task::harness::Harness<T,S>::poll_inner::h1be20a649ebd331b
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
 146:     0x564fae89cc1e - tokio::runtime::task::harness::Harness<T,S>::poll::h7c7e037a9a0b033f
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
 147:     0x564faf4ba10a - tokio::runtime::task::raw::RawTask::poll::h626249af0491f4e3
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
 148:     0x564faf4ba10a - tokio::runtime::task::LocalNotified<S>::run::hb8addeee3a3c54c0
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9
 149:     0x564faf4ba10a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}::h1c841ab6748b3bcf
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:18
 150:     0x564faf4ba10a - tokio::runtime::coop::with_budget::h0bbe3908baf78f4a
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5
 151:     0x564faf4ba10a - tokio::runtime::coop::budget::ha88aed2bb84c970d
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5
 152:     0x564faf4ba10a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h3832fe87202eebc2
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9
 153:     0x564faf4b6135 - tokio::runtime::scheduler::multi_thread::worker::Context::run::h27a10854a9d63bfa
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:535:24
 154:     0x564faf4b6135 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}::hf07c3ad4fe0cf498
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21
 155:     0x564faf4b6135 - tokio::runtime::context::scoped::Scoped<T>::set::h75da236f9d897da1
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9
 156:     0x564faf4f9608 - tokio::runtime::context::set_scheduler::{{closure}}::h59b22c1a323eb811
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26
 157:     0x564faf4f9608 - std::thread::local::LocalKey<T>::try_with::hc20591cd664efd9d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:283:12
 158:     0x564faf4f9608 - std::thread::local::LocalKey<T>::with::hd228c34d1db99eda
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:260:9
 159:     0x564faf4f9608 - tokio::runtime::context::set_scheduler::he6413f92dcfbb709
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:17
 160:     0x564faf4f9608 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::h90145e52818cfa51
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9
 161:     0x564faf4f9608 - tokio::runtime::context::runtime::enter_runtime::h6e91b28ba394eb3e
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
 162:     0x564faf4b9f0c - tokio::runtime::scheduler::multi_thread::worker::run::he443bfcf26770424
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5
 163:     0x564faf4f08c1 - tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}::h673489d56cab43a8
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45
 164:     0x564faf4f08c1 - <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll::hf52c40ec5bbb3240
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21
 165:     0x564faf4e0c5f - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h4f8a96a8215e6853
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
 166:     0x564faf4ef126 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h231abc877973020e
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
 167:     0x564faf4ef126 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h7d95a0441d1292fe
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
 168:     0x564faf4ef126 - tokio::runtime::task::core::Core<T,S>::poll::hdd5e41a0f32ed5a0
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30
 169:     0x564faf4ccd22 - tokio::runtime::task::harness::poll_future::{{closure}}::h7cb22b0f9a0bc4a3
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
 170:     0x564faf4ccd22 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hfed28fbab9b0a03e
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
 171:     0x564faf4ccd22 - std::panicking::try::do_call::hb65cc09bf2bd4d7b
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
 172:     0x564faf4ccd22 - std::panicking::try::h7db0e518a9b8a97d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
 173:     0x564faf4ccd22 - std::panic::catch_unwind::hbdd2d16e43372a1b
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
 174:     0x564faf4ccd22 - tokio::runtime::task::harness::poll_future::hd81255e7b4d02094
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
 175:     0x564faf4ccd22 - tokio::runtime::task::harness::Harness<T,S>::poll_inner::hc2f4bd3db966b047
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
 176:     0x564faf4ccd22 - tokio::runtime::task::harness::Harness<T,S>::poll::h4b451fd3e734c254
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
 177:     0x564faf4c0b42 - tokio::runtime::task::raw::RawTask::poll::h626249af0491f4e3
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
 178:     0x564faf4c0b42 - tokio::runtime::task::UnownedTask<S>::run::h98beb39b303a29c5
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9
 179:     0x564faf4c0b42 - tokio::runtime::blocking::pool::Task::run::hac573af663f13975
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9
 180:     0x564faf4c0b42 - tokio::runtime::blocking::pool::Inner::run::had6d86a4109477f7
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17
 181:     0x564faf4c9eb0 - tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::hf78130f6419d7891
                               at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13
 182:     0x564faf4c9eb0 - std::sys_common::backtrace::__rust_begin_short_backtrace::hbf1dc341a32fb6c8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:155:18
 183:     0x564faf4d641a - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hebce08b4b59837b0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:542:17
 184:     0x564faf4d641a - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hd490dda97349681d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
 185:     0x564faf4d641a - std::panicking::try::do_call::h577f074763d50b1d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
 186:     0x564faf4d641a - std::panicking::try::h83c2307555e8761b
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
 187:     0x564faf4d641a - std::panic::catch_unwind::h102860fca5fc3899
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
 188:     0x564faf4d641a - std::thread::Builder::spawn_unchecked_::{{closure}}::hd27a3f18384072df
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:541:30
 189:     0x564faf4d641a - core::ops::function::FnOnce::call_once{{vtable.shim}}::h0a939909032743c8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/ops/function.rs:250:5
 190:     0x564faf43854b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hd9d8a12a4cec8653
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
 191:     0x564faf43854b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc87e6ebe32d18c90
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
 192:     0x564faf43854b - std::sys::pal::unix::thread::Thread::new::thread_start::h68df5163d04f9130
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys/pal/unix/thread.rs:108:17


*** await tree context of current task ***

Actor 13408: `` [35.133s]
  Epoch 7634122441162752 [4.224s]
    Project 346000000015 [4.224s]
      HashJoin 346000000014 [4.224s]  <== current
 193:     0x7efda3864a94 - start_thread
                               at ./nptl/pthread_create.c:447:8
 194:     0x7efda38f1a34 - __GI___clone
                               at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:100
 195:                0x0 - <unknown>
[Detached 2]
  LocalInput (actor 13435) [4.224s]
[Detached 3]
  LocalInput (actor 13434) [4.224s]
[Detached 6]
  LocalInput (actor 13430) [4.224s]
[Detached 12]
  LocalInput (actor 13437) [4.224s]
[Detached 14]
  GroupTopN 346000000013 [4.224s]
    Project 346000000012 [4.224s]
      Filter 346000000011 [4.224s]
        ProjectSet 346000000010 [4.224s]
          Project 34600000000F [4.224s]
            HashAgg 34600000000E [4.224s]
              Merge 34600000000D [4.224s]
                LocalInput (actor 13426) [4.224s]
                LocalInput (actor 13424) [4.224s]
                LocalInput (actor 13422) [4.224s]
                LocalInput (actor 13425) [4.224s]
                LocalInput (actor 13423) [4.224s]
                LocalInput (actor 13428) [4.224s]
                LocalInput (actor 13429) [4.224s]
                LocalInput (actor 13427) [4.224s]
[Detached 26]
  LocalInput (actor 13433) [4.224s]
[Detached 33]
  LocalInput (actor 13436) [4.224s]
[Detached 34]
  LocalInput (actor 13432) [4.224s]

To Reproduce

No response

Expected behavior

No response

How did you deploy RisingWave?

Self-deployed. Single node

The version of RisingWave

2.0.3

Additional context

Didn't do any scaling operations.

[fuyufjh]

Found some clues.

Here is part of the problematic MV

WITH xxx AS (
    SELECT *,
           COALESCE(
               (SELECT (value->>'v')::INT
                FROM jsonb_array_elements(actions_info) AS elements(value)
                WHERE value->>'action' IN ('action-type-a', 'action-type-b')
                LIMIT 1),
               v2
           ) AS foo
    FROM t
) ...

The subquery will be unnested into

└─StreamProject
  └─StreamHashJoin { type: LeftOuter, predicate: t.actions_info IS NOT DISTINCT FROM t.actions_info }
    ├─StreamExchange { dist: HashShard(t.actions_info) }
    │ └─StreamTableScan { table: t, columns: [...] }
    └─StreamGroupTopN { order: [t.actions_info ASC], limit: 1, offset: 0, group_key: [t.actions_info] }
      └─StreamProject { exprs: [t.actions_info, JsonbAccessStr(JsonbArrayElements($0), 'v':Varchar)::Int32 as $expr1, projected_row_id] }
        └─StreamFilter { predicate: In(JsonbAccessStr(JsonbArrayElements($0), 'action':Varchar), 'action-type-a':Varchar, 'action-type-b': Varchar) }
          └─StreamProjectSet { select_list: [$0, JsonbArrayElements($0)] }
            └─StreamProject { exprs: [t.actions_info] }
              └─StreamHashAgg { group_key: [t.actions_info], aggs: [count] }
                └─StreamExchange { dist: HashShard(t.actions_info) }
                  └─StreamTableScan { table: t, columns: [actions_info, _row_id] }

The join condition is on a JSONB column t.actions_info.

https://github.com/risingwavelabs/jsonbb/blob/bc93534cc38a11ca01da5fbe32eb53d09350e54b/src/value_ref.rs#L543-L549