Skip to content

HIVE-29496: Subsequent transactional materialized view rebuild fails when source tables are iceberg tables #6434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kasakrisz merged 5 commits into from
Apr 30, 2026
Merged

HIVE-29496: Subsequent transactional materialized view rebuild fails when source tables are iceberg tables #6434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8e48682
HIVE-29496: Subsequent transactional materialized view rebuild fails …
kasakrisz Apr 15, 2026
1bdab35
revert lock acquisition move
kasakrisz Apr 28, 2026
3634e3c
trigger txn id generation
kasakrisz Apr 28, 2026
d8788ce
open txn by txnManager instead of getValidTxnList()
kasakrisz Apr 28, 2026
6cc759f
check whether txn is open
kasakrisz Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc9.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
-- Test subsequent rebuild of transactional MV when source table is Iceberg.

-- SORT_QUERY_RESULTS
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.support.concurrency=true;

create table ice_basetable (a int, b string) stored by iceberg;
insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie');

create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as

@InvisibleProgrammer InvisibleProgrammer Apr 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thinking out loud: shouldn't we add a test case when the MV itself stored by iceberg?

@kasakrisz kasakrisz Apr 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there are already tests like
https://github.com/apache/hive/blob/master/iceberg/iceberg-handler/src/test/queries/positive/mv_iceberg_orc2.q

select b, a from ice_basetable;

create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
select b, a from ice_basetable;

select * from mv_acid;
select * from mv_insert_only;

insert into ice_basetable values (5, 'amia');
alter materialized view mv_acid rebuild;
alter materialized view mv_insert_only rebuild;

insert into ice_basetable values (4, 'mania');
alter materialized view mv_acid rebuild;
alter materialized view mv_insert_only rebuild;

select * from mv_acid;
select * from mv_insert_only;
156 changes: 156 additions & 0 deletions iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_orc9.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
PREHOOK: query: create table ice_basetable (a int, b string) stored by iceberg
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@ice_basetable
POSTHOOK: query: create table ice_basetable (a int, b string) stored by iceberg
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_basetable
PREHOOK: query: insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_basetable
POSTHOOK: query: insert into ice_basetable values (1, 'alfred'),(2, 'bob'),(2, 'bonnie'),(3, 'calvin'),(3, 'charlie')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_basetable
PREHOOK: query: create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as
select b, a from ice_basetable
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: database:default
PREHOOK: Output: default@mv_acid
POSTHOOK: query: create materialized view mv_acid STORED AS ORC TBLPROPERTIES ('transactional'='true') as
select b, a from ice_basetable
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: database:default
POSTHOOK: Output: default@mv_acid
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
select b, a from ice_basetable
PREHOOK: type: CREATE_MATERIALIZED_VIEW
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: database:default
PREHOOK: Output: default@mv_insert_only
POSTHOOK: query: create materialized view mv_insert_only STORED AS ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') as
select b, a from ice_basetable
POSTHOOK: type: CREATE_MATERIALIZED_VIEW
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: database:default
POSTHOOK: Output: default@mv_insert_only
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: select * from mv_acid
PREHOOK: type: QUERY
PREHOOK: Input: default@mv_acid
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from mv_acid
POSTHOOK: type: QUERY
POSTHOOK: Input: default@mv_acid
POSTHOOK: Output: hdfs://### HDFS PATH ###
alfred 1
bob 2
bonnie 2
calvin 3
charlie 3
PREHOOK: query: select * from mv_insert_only
PREHOOK: type: QUERY
PREHOOK: Input: default@mv_insert_only
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from mv_insert_only
POSTHOOK: type: QUERY
POSTHOOK: Input: default@mv_insert_only
POSTHOOK: Output: hdfs://### HDFS PATH ###
alfred 1
bob 2
bonnie 2
calvin 3
charlie 3
PREHOOK: query: insert into ice_basetable values (5, 'amia')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_basetable
POSTHOOK: query: insert into ice_basetable values (5, 'amia')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_basetable
PREHOOK: query: alter materialized view mv_acid rebuild
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: default@mv_acid
POSTHOOK: query: alter materialized view mv_acid rebuild
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: default@mv_acid
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: alter materialized view mv_insert_only rebuild
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: default@mv_insert_only
POSTHOOK: query: alter materialized view mv_insert_only rebuild
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: default@mv_insert_only
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: insert into ice_basetable values (4, 'mania')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@ice_basetable
POSTHOOK: query: insert into ice_basetable values (4, 'mania')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_basetable
PREHOOK: query: alter materialized view mv_acid rebuild
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: default@mv_acid
POSTHOOK: query: alter materialized view mv_acid rebuild
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: default@mv_acid
POSTHOOK: Lineage: mv_acid.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_acid.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: alter materialized view mv_insert_only rebuild
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
PREHOOK: Input: default@ice_basetable
PREHOOK: Output: default@mv_insert_only
POSTHOOK: query: alter materialized view mv_insert_only rebuild
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REBUILD
POSTHOOK: Input: default@ice_basetable
POSTHOOK: Output: default@mv_insert_only
POSTHOOK: Lineage: mv_insert_only.a SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:a, type:int, comment:null), ]
POSTHOOK: Lineage: mv_insert_only.b SIMPLE [(ice_basetable)ice_basetable.FieldSchema(name:b, type:string, comment:null), ]
PREHOOK: query: select * from mv_acid
PREHOOK: type: QUERY
PREHOOK: Input: default@mv_acid
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from mv_acid
POSTHOOK: type: QUERY
POSTHOOK: Input: default@mv_acid
POSTHOOK: Output: hdfs://### HDFS PATH ###
alfred 1
amia 5
bob 2
bonnie 2
calvin 3
charlie 3
mania 4
PREHOOK: query: select * from mv_insert_only
PREHOOK: type: QUERY
PREHOOK: Input: default@mv_insert_only
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: select * from mv_insert_only
POSTHOOK: type: QUERY
POSTHOOK: Input: default@mv_insert_only
POSTHOOK: Output: hdfs://### HDFS PATH ###
alfred 1
amia 5
bob 2
bonnie 2
calvin 3
charlie 3
mania 4
4 changes: 4 additions & 0 deletions ...oop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.LockMaterializationRebuildRequest;
import org.apache.hadoop.hive.metastore.api.LockState;
import org.apache.hadoop.hive.metastore.api.TxnType;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
Expand Down Expand Up @@ -213,6 +214,9 @@ private ASTNode getRewrittenAST(TableName tableName) throws SemanticException {
HiveTxnManager txnManager = getTxnMgr();
LockState state;
try {
if (!txnManager.isTxnOpen()) {
txnManager.openTxn(ctx, conf.getUser(), TxnType.MATER_VIEW_REBUILD);
}
state = txnManager.acquireMaterializationRebuildLock(new LockMaterializationRebuildRequest(tableName.getCat(),
tableName.getDb(), tableName.getTable(), txnManager.getCurrentTxnId())).getState();
} catch (LockException e) {
Expand Down
Loading