set extra_properties at create time via properties() override, alter on incremental runs

Author: a-monteiro

macros/dune_dbt_overrides/properties.sql

@@ -0,0 +1,81 @@
+{#
+  Override of the dbt-trino adapter's properties() macro to inject dune.public
+  into extra_properties at CREATE TABLE time.
+
+  Configure per model via config():
+    , dune_public = true   -- make table publicly visible on Dune
+    , dune_public = false  -- (default) keep table private
+
+  Or set for an entire folder in dbt_project.yml:
+    models:
+      my_project:
+        public_models:
+          +dune_public: true
+
+  Only runs in prod. On incremental (non-full-refresh) runs, set_table_visibility
+  handles it via ALTER TABLE instead.
+#}
+{% macro properties(temporary=False) %}
+  {%- set _properties = config.get('properties') -%}
+  {%- set table_format = config.get('table_format') -%}
+  {%- set file_format = config.get('file_format') -%}
+
+  {%- if file_format -%}
+    {%- if _properties -%}
+      {%- if _properties.format -%}
+        {% set msg %}
+          You can specify either 'file_format' or 'properties.format' configurations, but not both.
+        {% endset %}
+        {% do exceptions.raise_compiler_error(msg) %}
+      {%- else -%}
+        {%- do _properties.update({'format': "'" ~ file_format ~ "'"}) -%}
+      {%- endif -%}
+    {%- else -%}
+      {%- set _properties = {'format': "'" ~ file_format ~ "'"} -%}
+    {%- endif -%}
+  {%- endif -%}
+
+  {%- if table_format -%}
+    {%- if _properties -%}
+      {%- if _properties.type -%}
+        {% set msg %}
+          You can specify either 'table_format' or 'properties.type' configurations, but not both.
+        {% endset %}
+        {% do exceptions.raise_compiler_error(msg) %}
+      {%- else -%}
+        {%- do _properties.update({'type': "'" ~ table_format ~ "'"}) -%}
+      {%- endif -%}
+    {%- else -%}
+      {%- set _properties = {'type': "'" ~ table_format ~ "'"} -%}
+    {%- endif -%}
+  {%- endif -%}
+
+  {%- if temporary -%}
+    {%- if _properties -%}
+      {%- if _properties.location -%}
+          {%- do _properties.update({'location': _properties.location[:-1] ~ "__dbt_tmp'"}) -%}
+      {%- endif -%}
+    {%- endif -%}
+  {%- endif -%}
+
+  {#-- Inject dune.public into extra_properties at CREATE time (prod only) --#}
+  {%- set dune_public = config.get('dune_public') -%}
+  {%- if dune_public is not none and target.name == 'prod' -%}
+    {%- set visibility_value = 'true' if dune_public else 'false' -%}
+    {%- set extra_props_sql = "map_from_entries(ARRAY[ROW('dune.public', '" ~ visibility_value ~ "')])" -%}
+    {%- if _properties is none -%}
+      {%- set _properties = {'extra_properties': extra_props_sql} -%}
+    {%- else -%}
+      {%- do _properties.update({'extra_properties': extra_props_sql}) -%}
+    {%- endif -%}
+  {%- endif -%}
+
+  {%- if _properties is not none -%}
+      WITH (
+          {%- for key, value in _properties.items() -%}
+            {{ key }} = {{ value }}
+            {%- if not loop.last -%}{{ ',\n  ' }}{%- endif -%}
+          {%- endfor -%}
+      )
+  {%- endif -%}
+{%- endmacro -%}

macros/dune_dbt_overrides/set_table_visibility.sql

@@ -8,7 +8,11 @@
 {%- endmacro -%}
 
 {#
-  set_table_visibility: post-hook macro that sets the dune.public extra property on tables/incrementals.
+  set_table_visibility: post-hook that sets dune.public on incremental (non-full-refresh) runs.
+
+  For table materializations and incremental full-refreshes, extra_properties is set
+  at CREATE TABLE time via the overridden properties() macro. This hook handles the
+  incremental case where no CREATE is issued (INSERT/MERGE only).
 
   Configure per model via config():
     , dune_public = true   -- make table publicly visible on Dune
@@ -20,10 +24,12 @@
         public_models:
           +dune_public: true
 
-  Only runs in prod. Views are not supported yet.
+  Only runs in prod. Views are not supported.
 #}
 {% macro set_table_visibility(this, materialization) %}
-{%- if target.name == 'prod' and materialization in ('table', 'incremental') -%}
+{%- if target.name == 'prod'
+    and materialization == 'incremental'
+    and not flags.FULL_REFRESH -%}
   {%- set dune_public = config.get('dune_public', false) -%}
   {%- set properties = {'dune.public': 'true' if dune_public else 'false'} -%}
   ALTER TABLE {{ this }}

models/templates/dbt_template_append_incremental_model.sql

@@ -9,7 +9,7 @@
     alias = 'dbt_template_append_incremental_model'
     , materialized = 'incremental'
     , incremental_strategy = 'append'
-    -- , dune_public = true  -- uncomment to make this table publicly visible on Dune
+    , dune_public = false
     , properties = {
         "partitioned_by": "ARRAY['block_date']"
     }

models/templates/dbt_template_delete_insert_incremental_model.sql

@@ -4,7 +4,7 @@
     , incremental_strategy = 'delete+insert'
     , unique_key = ['block_number', 'block_date']
     , incremental_predicates = ["block_date >= now() - interval '1' day"]
-    -- , dune_public = true  -- uncomment to make this table publicly visible on Dune
+    , dune_public = false
     , properties = {
         "partitioned_by": "ARRAY['block_date']"
     }

models/templates/dbt_template_merge_incremental_model.sql

@@ -4,7 +4,7 @@
     , incremental_strategy = 'merge'
     , unique_key = ['block_number', 'block_date']
     , incremental_predicates = ["DBT_INTERNAL_DEST.block_date >= now() - interval '1' day"]
-    -- , dune_public = true  -- uncomment to make this table publicly visible on Dune
+    , dune_public = false
     , properties = {
         "partitioned_by": "ARRAY['block_date']"
     }

models/templates/dbt_template_table_model.sql

@@ -1,7 +1,7 @@
 {{ config(
     alias = 'dbt_template_table_model'
     , materialized = 'table'
-    -- , dune_public = true  -- uncomment to make this table publicly visible on Dune
+    , dune_public = false
     , properties = {
         "partitioned_by": "ARRAY['block_date']"
     }