Skip to content

Commit

Permalink
Implement drop_glue_database macro and method (#758)
Browse files Browse the repository at this point in the history
  • Loading branch information
svdimchenko authored Feb 10, 2025
1 parent 999dd7f commit 014d5f9
Show file tree
Hide file tree
Showing 11 changed files with 46 additions and 13 deletions.
2 changes: 0 additions & 2 deletions dbt-adapters/.changes/1.14.0.md
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
## dbt-adapters 1.14.0 - February 07, 2025


1 change: 0 additions & 1 deletion dbt-adapters/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -281,4 +281,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie)
### Security

* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140))

2 changes: 1 addition & 1 deletion dbt-adapters/src/dbt/adapters/sql/connections.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def _execute_query_with_retry(

fire_event(
AdapterEventDebug(
message=f"Got a retryable error {type(e)}. {retry_limit-attempt} retries left. Retrying in 1 second.\nError:\n{e}"
message=f"Got a retryable error {type(e)}. {retry_limit - attempt} retries left. Retrying in 1 second.\nError:\n{e}"
)
)
time.sleep(1)
Expand Down
2 changes: 0 additions & 2 deletions dbt-athena/.changes/1.9.1.md
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
## dbt-athena 1.9.1 - February 07, 2025


6 changes: 6 additions & 0 deletions dbt-athena/.changes/unreleased/Features-20250129-121455.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Implement drop_glue_database macro and method
time: 2025-01-29T12:14:55.46675+01:00
custom:
Author: svdimchenko
Issue: "408"
1 change: 0 additions & 1 deletion dbt-athena/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie)
For information on prior major and minor releases, see their changelogs:

- [1.8](https://github.com/dbt-labs/dbt-athena/blob/main/CHANGELOG.md)

23 changes: 22 additions & 1 deletion dbt-athena/src/dbt/adapters/athena/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from functools import lru_cache
from textwrap import dedent
from threading import Lock
from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, Type
from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, Type
from urllib.parse import urlparse
from uuid import uuid4

Expand Down Expand Up @@ -69,6 +69,10 @@
from dbt.adapters.contracts.relation import RelationConfig
from dbt.adapters.sql import SQLAdapter


if TYPE_CHECKING:
from mypy_boto3_glue.client import GlueClient

boto3_client_lock = Lock()


Expand Down Expand Up @@ -1242,6 +1246,23 @@ def delete_from_glue_catalog(self, relation: AthenaRelation) -> None:
LOGGER.error(e)
raise e

@available
def drop_glue_database(self, database_name: str, catalog_name: str = "awsdatacatalog") -> None:
conn = self.connections.get_thread_connection()
creds = conn.credentials
client = conn.handle
catalog = self._get_data_catalog(catalog_name)
catalog_id = get_catalog_id(catalog)

with boto3_client_lock:
glue_client: GlueClient = client.session.client(
"glue",
region_name=client.region_name,
config=get_boto3_config(num_retries=creds.effective_num_retries),
)
glue_client.delete_database(Name=database_name, CatalogId=catalog_id)
LOGGER.debug(f"Glue database successfully deleted: {catalog_name}.{database_name}")

@available.parse_none
def valid_snapshot_target(self, relation: BaseRelation) -> None:
"""Log an error to help developers migrate to the new snapshot logic"""
Expand Down
5 changes: 5 additions & 0 deletions dbt-athena/src/dbt/include/athena/macros/adapters/schema.sql
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@
drop schema if exists {{ relation.without_identifier().render_hive() }} cascade
{% endcall %}
{% endmacro %}


{% macro drop_glue_database(database_name, catalog_name='awsdatacatalog') -%}
{{ adapter.drop_glue_database(database_name, catalog_name) }}
{% endmacro %}
12 changes: 12 additions & 0 deletions dbt-athena/tests/unit/test_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from dbt_common.exceptions import ConnectionError, DbtRuntimeError
from moto import mock_aws
from moto.core import DEFAULT_ACCOUNT_ID
from mypy_boto3_glue.client import GlueClient

from dbt.adapters.athena import AthenaAdapter
from dbt.adapters.athena import Plugin as AthenaPlugin
Expand Down Expand Up @@ -1361,6 +1362,17 @@ def test_format_unsupported_type(self):
with pytest.raises(ValueError):
self.adapter.format_value_for_partition("test", "unsupported_type")

@mock_aws
def test_drop_glue_database(self):
glue_client: GlueClient = boto3.client("glue", region_name=AWS_REGION)
test_input = {"Name": "test"}
glue_client.create_database(DatabaseInput=test_input)
database_list = glue_client.get_databases()["DatabaseList"]
assert [test_input["Name"]] == [db["Name"] for db in database_list]
self.adapter.acquire_connection("dummy")
self.adapter.drop_glue_database(database_name=test_input["Name"])
assert glue_client.get_databases()["DatabaseList"] == []


class TestAthenaFilterCatalog:
def test__catalog_filter_table(self):
Expand Down
2 changes: 0 additions & 2 deletions dbt-tests-adapter/.changes/1.11.0.md
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
## dbt-tests-adapter 1.11.0 - February 07, 2025


3 changes: 0 additions & 3 deletions dbt-tests-adapter/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie)
- For details on how to document a change, see the [contributing guide](/CONTRIBUTING.md#changelog-entry)

## dbt-tests-adapter 1.11.0 - February 07, 2025



0 comments on commit 014d5f9

Please sign in to comment.