From 92f8a24777c2d8c8740ac416dc544ed387bcb4ca Mon Sep 17 00:00:00 2001 From: jessicamack Date: Wed, 3 Jul 2024 11:32:40 -0400 Subject: [PATCH 1/8] unpin channels-redis The bug that initially caused the upgrade block has been resolved https://github.com/django/channels_redis/issues/332 --- requirements/requirements.in | 2 +- requirements/requirements.txt | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 3dc2bb6a46a4..fb35d5c32156 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -7,7 +7,7 @@ azure-keyvault boto3 botocore channels -channels-redis==3.4.1 # see UPGRADE BLOCKERs +channels-redis cryptography>=41.0.7 # CVE-2023-49083 Cython<3 # due to https://github.com/yaml/pyyaml/pull/702 daphne diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 59241c34fd36..bb47b0f3a121 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -7,8 +7,6 @@ aiohttp==3.9.5 # twilio aiohttp-retry==2.8.3 # via twilio -aioredis==1.3.1 - # via channels-redis aiosignal==1.3.1 # via aiohttp annotated-types==0.6.0 @@ -30,7 +28,6 @@ asn1==2.7.0 async-timeout==4.0.3 # via # aiohttp - # aioredis # redis attrs==23.2.0 # via @@ -88,7 +85,7 @@ channels==3.0.5 # via # -r /awx_devel/requirements/requirements.in # channels-redis -channels-redis==3.4.1 +channels-redis==4.2.0 # via -r /awx_devel/requirements/requirements.in charset-normalizer==3.3.2 # via requests @@ -202,9 +199,7 @@ grpcio==1.62.2 # -r /awx_devel/requirements/requirements.in # opentelemetry-exporter-otlp-proto-grpc hiredis==2.0.0 - # via - # -r /awx_devel/requirements/requirements.in - # aioredis + # via -r /awx_devel/requirements/requirements.in hyperlink==21.0.0 # via # autobahn @@ -450,7 +445,9 @@ pyzstd==0.15.10 receptorctl==1.4.4 # via -r /awx_devel/requirements/requirements.in redis==5.0.1 - # via -r /awx_devel/requirements/requirements.in + # via + # -r /awx_devel/requirements/requirements.in + # channels-redis referencing==0.33.0 # via # jsonschema From 61804e525abaad8e72fd4318444d104c7b468e00 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Wed, 3 Jul 2024 13:00:52 -0400 Subject: [PATCH 2/8] replace aioredis Exception with a redis Exception Version 4.0.0 of channel-redis migrated the underlying Redis library from aioredis to redis-py. The Exception has been changed to an equivalent --- awx/main/wsrelay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/wsrelay.py b/awx/main/wsrelay.py index dced6c4e837e..0dcb39506929 100644 --- a/awx/main/wsrelay.py +++ b/awx/main/wsrelay.py @@ -8,7 +8,7 @@ import aiohttp from aiohttp import client_exceptions -import aioredis +import redis from channels.layers import get_channel_layer @@ -191,7 +191,7 @@ async def run_producer(self, name, websocket, group): return continue - except aioredis.errors.ConnectionClosedError: + except redis.exceptions.ConnectionError: logger.info(f"Producer {name} lost connection to Redis, shutting down.") return From 5d065a313894787ce6e8503db160d60854aecf50 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Wed, 3 Jul 2024 16:35:40 -0400 Subject: [PATCH 3/8] remove unused license --- licenses/aioredis.txt | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 licenses/aioredis.txt diff --git a/licenses/aioredis.txt b/licenses/aioredis.txt deleted file mode 100644 index 4d8920e897b4..000000000000 --- a/licenses/aioredis.txt +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014-2017 Alexey Popravka -Copyright (c) 2021 Sean Stewart - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 998458e80f2f192f4f9356be11b5dd3318301362 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Mon, 22 Jul 2024 12:17:16 -0400 Subject: [PATCH 4/8] remove UPGRADE BLOCKER in README --- requirements/README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/requirements/README.md b/requirements/README.md index df3d0a3aaa7c..3f200e4cdde8 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -120,15 +120,6 @@ in the top-level Makefile. If modifying this library make sure testing with the offline build is performed to confirm it is functionally working. -### channels-redis - -Due to an upstream bug (linked below), we see `RuntimeError: Event loop is closed` errors with newer versions of `channels-redis`. -Upstream is aware of the bug and it is likely to be fixed in the next release according to the issue linked below. -For now, we pin to the old version, 3.4.1 - -* https://github.com/django/channels_redis/issues/332 -* https://github.com/ansible/awx/issues/13313 - ### hiredis The hiredis 2.1.0 release doesn't provide source distribution on PyPI which prevents users to build that python package from the From eff25e962a094114868c546e379929adb02ee276 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Wed, 24 Jul 2024 12:19:13 -0400 Subject: [PATCH 5/8] remove hiredis it was an indirect dependency from aioredis which was removed --- requirements/requirements.in | 1 - requirements/requirements.txt | 2 -- 2 files changed, 3 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index fb35d5c32156..148f8475bc0d 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -29,7 +29,6 @@ djangorestframework-yaml filelock GitPython>=3.1.37 # CVE-2023-41040 grpcio<1.63.0 # 1.63.0+ requires cython>=3 -hiredis==2.0.0 # see UPGRADE BLOCKERs irc jinja2>=3.1.3 # CVE-2024-22195 JSON-log-formatter diff --git a/requirements/requirements.txt b/requirements/requirements.txt index bb47b0f3a121..e26bb17fc963 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -198,8 +198,6 @@ grpcio==1.62.2 # via # -r /awx_devel/requirements/requirements.in # opentelemetry-exporter-otlp-proto-grpc -hiredis==2.0.0 - # via -r /awx_devel/requirements/requirements.in hyperlink==21.0.0 # via # autobahn From 75662bd527d95e87d3a093f0616e22869264cf51 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Wed, 24 Jul 2024 12:34:08 -0400 Subject: [PATCH 6/8] remove unused license --- licenses/hiredis.txt | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 licenses/hiredis.txt diff --git a/licenses/hiredis.txt b/licenses/hiredis.txt deleted file mode 100644 index a5fc9739551f..000000000000 --- a/licenses/hiredis.txt +++ /dev/null @@ -1,29 +0,0 @@ -Copyright (c) 2009-2011, Salvatore Sanfilippo -Copyright (c) 2010-2011, Pieter Noordhuis - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of Redis nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 2445b06978175fa2a132509fbf507ebd957f6586 Mon Sep 17 00:00:00 2001 From: jessicamack Date: Thu, 25 Jul 2024 12:46:00 -0400 Subject: [PATCH 7/8] add back hiredis it's potentially providing a performance boost. install explicitly as a part of redis. upgrade to more recent version --- licenses/hiredis.txt | 29 +++++++++++++++++++++++++++++ requirements/requirements.in | 2 +- requirements/requirements.txt | 4 +++- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 licenses/hiredis.txt diff --git a/licenses/hiredis.txt b/licenses/hiredis.txt new file mode 100644 index 000000000000..a5fc9739551f --- /dev/null +++ b/licenses/hiredis.txt @@ -0,0 +1,29 @@ +Copyright (c) 2009-2011, Salvatore Sanfilippo +Copyright (c) 2010-2011, Pieter Noordhuis + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/requirements/requirements.in b/requirements/requirements.in index 148f8475bc0d..fd07d01f1261 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -59,7 +59,7 @@ receptorctl social-auth-core[openidconnect]==4.4.2 # see UPGRADE BLOCKERs social-auth-app-django==5.4.0 # see UPGRADE BLOCKERs sqlparse>=0.4.4 # Required by django https://github.com/ansible/awx/security/dependabot/96 -redis +redis[hiredis] requests slack-sdk tacacs_plus==1.0 # UPGRADE BLOCKER: auth does not work with later versions diff --git a/requirements/requirements.txt b/requirements/requirements.txt index e26bb17fc963..39b613996c71 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -198,6 +198,8 @@ grpcio==1.62.2 # via # -r /awx_devel/requirements/requirements.in # opentelemetry-exporter-otlp-proto-grpc +hiredis==3.0.0 + # via redis hyperlink==21.0.0 # via # autobahn @@ -442,7 +444,7 @@ pyzstd==0.15.10 # via -r /awx_devel/requirements/requirements.in receptorctl==1.4.4 # via -r /awx_devel/requirements/requirements.in -redis==5.0.1 +redis[hiredis]==5.0.1 # via # -r /awx_devel/requirements/requirements.in # channels-redis From 8a19c6371c7cd04ac77013584fc9d5c48af5e1ee Mon Sep 17 00:00:00 2001 From: jessicamack Date: Tue, 6 Aug 2024 10:41:52 -0400 Subject: [PATCH 8/8] remove UPGRADE BLOCKER for hiredis it was also addressed as a part of this PR --- requirements/README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/requirements/README.md b/requirements/README.md index 3f200e4cdde8..90640dc37236 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -120,15 +120,6 @@ in the top-level Makefile. If modifying this library make sure testing with the offline build is performed to confirm it is functionally working. -### hiredis - -The hiredis 2.1.0 release doesn't provide source distribution on PyPI which prevents users to build that python package from the -sources. -Downgrading to 2.0.0 (which provides source distribution) until the channels-redis issue is fixed or a newer hiredis version is -available on PyPi with source distribution. - -* https://github.com/redis/hiredis-py/issues/138 - ## Library Notes ### pexpect