diff --git a/UPDATING.md b/UPDATING.md index 4a67b267383d0..663df68cd9b69 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -38,6 +38,8 @@ assists people when migrating to a new version. - [29121](https://github.com/apache/superset/pull/29121) Removed the `css`, `position_json`, and `json_metadata` from the payload of the dashboard list endpoint (`GET api/v1/dashboard`) for performance reasons. - [29163](https://github.com/apache/superset/pull/29163) Removed the `SHARE_QUERIES_VIA_KV_STORE` and `KV_STORE` feature flags and changed the way Superset shares SQL Lab queries to use permalinks. The legacy `/kv` API was removed but we still support legacy links in 5.0. In 6.0, only permalinks will be supported. - [25166](https://github.com/apache/superset/pull/25166) Changed the default configuration of `UPLOAD_FOLDER` from `/app/static/uploads/` to `/static/uploads/`. It also removed the unused `IMG_UPLOAD_FOLDER` and `IMG_UPLOAD_URL` configuration options. +- [30284](https://github.com/apache/superset/pull/30284) Deprecated GLOBAL_ASYNC_QUERIES_REDIS_CONFIG in favor of the new GLOBAL_ASYNC_QUERIES_CACHE_BACKEND configuration. To leverage Redis Sentinel, set CACHE_TYPE to RedisSentinelCache, or use RedisCache for standalone Redis + ### Potential Downtime diff --git a/superset-websocket/README.md b/superset-websocket/README.md index bfaf940b1111a..79cbe8139725b 100644 --- a/superset-websocket/README.md +++ b/superset-websocket/README.md @@ -91,7 +91,7 @@ Note also that `localhost` and `127.0.0.1` are not considered the same host. For The following config values must contain the same values in both the Flask app config and `config.json`: ```text -GLOBAL_ASYNC_QUERIES_REDIS_CONFIG +GLOBAL_ASYNC_QUERIES_CACHE_BACKEND GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX GLOBAL_ASYNC_QUERIES_JWT_COOKIE_NAME GLOBAL_ASYNC_QUERIES_JWT_SECRET diff --git a/superset/async_events/async_query_manager.py b/superset/async_events/async_query_manager.py index 0e59b2c3ddfd5..d955cc8827876 100644 --- a/superset/async_events/async_query_manager.py +++ b/superset/async_events/async_query_manager.py @@ -18,10 +18,9 @@ import logging import uuid -from typing import Any, Literal, Optional, Union +from typing import Any, Literal, Optional import jwt -import redis from flask import Flask, Request, request, Response, session from flask_caching.backends.base import BaseCache @@ -43,6 +42,10 @@ class AsyncQueryTokenException(Exception): # noqa: N818 pass +class UnsupportedCacheBackendError(Exception): # noqa: N818 + pass + + class AsyncQueryJobException(Exception): # noqa: N818 pass @@ -77,7 +80,7 @@ def increment_id(entry_id: str) -> str: def get_cache_backend( config: dict[str, Any], -) -> Union[RedisCacheBackend, RedisSentinelCacheBackend, redis.Redis]: # type: ignore +) -> RedisCacheBackend | RedisSentinelCacheBackend: cache_config = config.get("GLOBAL_ASYNC_QUERIES_CACHE_BACKEND", {}) cache_type = cache_config.get("CACHE_TYPE") @@ -87,11 +90,8 @@ def get_cache_backend( if cache_type == "RedisSentinelCache": return RedisSentinelCacheBackend.from_config(cache_config) - # TODO: Deprecate hardcoded plain Redis code and expand cache backend options. - # Maintain backward compatibility with 'GLOBAL_ASYNC_QUERIES_REDIS_CONFIG' until it is deprecated. # noqa: E501 - return redis.Redis( - **config["GLOBAL_ASYNC_QUERIES_REDIS_CONFIG"], decode_responses=True - ) + # TODO: Expand cache backend options. + raise UnsupportedCacheBackendError("Unsupported cache backend configuration") class AsyncQueryManager: diff --git a/superset/config.py b/superset/config.py index 0a9d150ce622b..ae944f3ca11ef 100644 --- a/superset/config.py +++ b/superset/config.py @@ -1719,13 +1719,6 @@ def EMAIL_HEADER_MUTATOR( # pylint: disable=invalid-name,unused-argument # noq GLOBAL_ASYNC_QUERY_MANAGER_CLASS = ( "superset.async_events.async_query_manager.AsyncQueryManager" ) -GLOBAL_ASYNC_QUERIES_REDIS_CONFIG = { - "port": 6379, - "host": "127.0.0.1", - "password": "", - "db": 0, - "ssl": False, -} GLOBAL_ASYNC_QUERIES_REDIS_STREAM_PREFIX = "async-events-" GLOBAL_ASYNC_QUERIES_REDIS_STREAM_LIMIT = 1000 GLOBAL_ASYNC_QUERIES_REDIS_STREAM_LIMIT_FIREHOSE = 1000000 @@ -1746,7 +1739,6 @@ def EMAIL_HEADER_MUTATOR( # pylint: disable=invalid-name,unused-argument # noq # Global async queries cache backend configuration options: # - Set 'CACHE_TYPE' to 'RedisCache' for RedisCacheBackend. # - Set 'CACHE_TYPE' to 'RedisSentinelCache' for RedisSentinelCacheBackend. -# - Set 'CACHE_TYPE' to 'None' to fall back on 'GLOBAL_ASYNC_QUERIES_REDIS_CONFIG'. GLOBAL_ASYNC_QUERIES_CACHE_BACKEND = { "CACHE_TYPE": "RedisCache", "CACHE_REDIS_HOST": "localhost", diff --git a/tests/integration_tests/async_events/api_tests.py b/tests/integration_tests/async_events/api_tests.py index 8a3b9c3bfc495..02852ab6fb85c 100644 --- a/tests/integration_tests/async_events/api_tests.py +++ b/tests/integration_tests/async_events/api_tests.py @@ -18,7 +18,6 @@ from unittest import mock import pytest -import redis from superset.async_events.cache_backend import ( RedisCacheBackend, @@ -129,10 +128,6 @@ def test_events_redis_sentinel_cache_backend(self, mock_uuid4): RedisSentinelCacheBackend, self._test_events_logic ) - @mock.patch("uuid.uuid4", return_value=UUID) - def test_events_redis(self, mock_uuid4): - self.run_test_with_cache_backend(redis.Redis, self._test_events_logic) - def test_events_no_login(self): app._got_first_request = False async_query_manager.init_app(app) diff --git a/tests/integration_tests/tasks/async_queries_tests.py b/tests/integration_tests/tasks/async_queries_tests.py index 1d9ebdf2e8164..728076011bd43 100644 --- a/tests/integration_tests/tasks/async_queries_tests.py +++ b/tests/integration_tests/tasks/async_queries_tests.py @@ -20,7 +20,6 @@ from uuid import uuid4 import pytest -import redis from celery.exceptions import SoftTimeLimitExceeded from parameterized import parameterized @@ -52,7 +51,6 @@ class TestAsyncQueries(SupersetTestCase): [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @mock.patch("superset.tasks.async_queries.set_form_data") @@ -88,7 +86,6 @@ def test_load_chart_data_into_cache( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @mock.patch.object( @@ -125,7 +122,6 @@ def test_load_chart_data_into_cache_error( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @mock.patch.object(ChartDataCommand, "run") @@ -163,7 +159,6 @@ def test_soft_timeout_load_chart_data_into_cache( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") @@ -209,7 +204,6 @@ def test_load_explore_json_into_cache( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @mock.patch.object(async_query_manager, "update_job") @@ -245,7 +239,6 @@ def test_load_explore_json_into_cache_error( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ] ) @mock.patch.object(ChartDataCommand, "run") diff --git a/tests/unit_tests/async_events/async_query_manager_tests.py b/tests/unit_tests/async_events/async_query_manager_tests.py index 005058b00f97c..15e3ce68fce61 100644 --- a/tests/unit_tests/async_events/async_query_manager_tests.py +++ b/tests/unit_tests/async_events/async_query_manager_tests.py @@ -17,7 +17,6 @@ from unittest import mock from unittest.mock import ANY, Mock -import redis from flask import g from jwt import encode from pytest import fixture, mark, raises # noqa: PT013 @@ -84,7 +83,6 @@ def test_parse_channel_id_from_request_bad_jwt(async_query_manager): [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ], ) @mock.patch("superset.is_feature_enabled") @@ -129,7 +127,6 @@ def test_submit_chart_data_job_as_guest_user( [ ("RedisCacheBackend", mock.Mock(spec=RedisCacheBackend)), ("RedisSentinelCacheBackend", mock.Mock(spec=RedisSentinelCacheBackend)), - ("redis.Redis", mock.Mock(spec=redis.Redis)), ], ) @mock.patch("superset.is_feature_enabled")