From d3aa1e4b2bcfff2d97031b761372677a8bfa4a14 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 10:32:55 +0530 Subject: [PATCH 1/6] ci (fix): version output --- .github/workflows/merge-main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml index 4af5aeae..67c269c0 100644 --- a/.github/workflows/merge-main.yml +++ b/.github/workflows/merge-main.yml @@ -28,10 +28,10 @@ jobs: - name: Dry Run to get next release version id: next_version + shell: bash run: | pip install python-semantic-release - export NEXT_TAG_VERSION=$(semantic-release --noop version --print) - echo "new_tag_version=${NEXT_TAG_VERSION}" >> $GITHUB_OUTPUT + echo "new_tag_version=$(semantic-release --noop version --print)" >> $GITHUB_OUTPUT - name: Python Semantic Release id: release From d39fed4d71d5c8acf48d96492d64393c8c93081a Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:15:24 +0530 Subject: [PATCH 2/6] ci (fix): Increased es java mem limit --- .ci/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml index b6900da8..e88ef29d 100644 --- a/.ci/docker-compose.yml +++ b/.ci/docker-compose.yml @@ -35,7 +35,7 @@ services: - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358 - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization - http.cors.allow-credentials=true - - ES_JAVA_OPTS=-Xms750m -Xmx2g + - ES_JAVA_OPTS=-Xms750m -Xmx3g ulimits: memlock: soft: -1 From 0a596e0c3dcedaf406954390a78253be0f1609ec Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:40:07 +0530 Subject: [PATCH 3/6] ci (fix): Modify pr testing - Removed integration tests and required components - Used unittest for unit tests - Disabled audio from disk test --- .ci/docker-compose.yml | 113 +++++++++--------- .github/workflows/pr-tests.yml | 2 +- .../operators/test_audio_vec_embedding.py | 4 +- 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml index e88ef29d..35dc0ba9 100644 --- a/.ci/docker-compose.yml +++ b/.ci/docker-compose.yml @@ -4,59 +4,62 @@ services: context: ./../src dockerfile: Dockerfile.test target: test - env_file: ./../src/test.env - command: bash -c "python -c 'import time'; - while ! curl store:9200 | grep -q 'You Know, for Search' - && ! curl queue:15672 | grep -q ''; - do python -c 'time.sleep(10)'; - done; - nose2 tests" - links: - - store - - queue - depends_on: - store: - condition: service_started - queue: - condition: service_started +# env_file: ./../src/test.env + command: | +# bash -c "python -c 'import time'; +# while ! curl store:9200 | grep -q 'You Know, for Search' +# && ! curl queue:15672 | grep -q ''; +# do python -c 'time.sleep(10)'; +# done; + bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py" | + bash -c "python -m unittest core.operators.test_vid_vec_rep_resnet" | + bash -c "python -m unittest core.operators.test_audio_vec_embedding" +# links: +# - store +# - queue +# depends_on: +# store: +# condition: service_started +# queue: +# condition: service_started - store: - container_name: es - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0 - volumes: - - ./.docker/es/data:/var/lib/elasticsearch/data -# ports: -# - "9300:9300" -# - "9200:9200" - environment: - - xpack.security.enabled=false - - discovery.type=single-node - - http.cors.enabled=true - - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358 - - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization - - http.cors.allow-credentials=true - - ES_JAVA_OPTS=-Xms750m -Xmx3g - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - cap_add: - - IPC_LOCK - - queue: - image: rabbitmq:3.12.12-management - container_name: rabbitmq - hostname: rabbit - volumes: - - ./.docker/rabbitmq/data:/var/lib/rabbitmq - - ./.docker/rabbitmq/logs:/var/log/rabbitmq - environment: - RABBITMQ_ERLANG_COOKIE: "secret-cookie" - RABBITMQ_DEFAULT_USER: "admin" - RABBITMQ_DEFAULT_PASS: "Admin123" -# ports: -# - "5672:5672" -# - "15672:15672" +# store: +# container_name: es +# image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0 +# volumes: +# - ./.docker/es/data:/var/lib/elasticsearch/data +## ports: +## - "9300:9300" +## - "9200:9200" +# environment: +# - xpack.security.enabled=false +# - discovery.type=single-node +# - http.cors.enabled=true +# - http.cors.allow-origin=http://localhost:1358,http://127.0.0.1:1358 +# - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization +# - http.cors.allow-credentials=true +# - ES_JAVA_OPTS=-Xms750m -Xmx2g +# ulimits: +# memlock: +# soft: -1 +# hard: -1 +# nofile: +# soft: 65536 +# hard: 65536 +# cap_add: +# - IPC_LOCK +# +# queue: +# image: rabbitmq:3.12.12-management +# container_name: rabbitmq +# hostname: rabbit +# volumes: +# - ./.docker/rabbitmq/data:/var/lib/rabbitmq +# - ./.docker/rabbitmq/logs:/var/log/rabbitmq +# environment: +# RABBITMQ_ERLANG_COOKIE: "secret-cookie" +# RABBITMQ_DEFAULT_USER: "admin" +# RABBITMQ_DEFAULT_PASS: "Admin123" +## ports: +## - "5672:5672" +## - "15672:15672" diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 8be3368f..4a7c036c 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -26,7 +26,7 @@ jobs: - name: Launch containers to run tests id: vars run: | - docker compose -f ./.ci/docker-compose.yml -p ci up -d store queue sut + docker compose -f ./.ci/docker-compose.yml -p ci up -d sut docker logs -f ci-sut-1 echo "sut_output=$(docker wait ci-sut-1)" >> $GITHUB_OUTPUT diff --git a/src/core/operators/test_audio_vec_embedding.py b/src/core/operators/test_audio_vec_embedding.py index d9b6e28f..c96a7d9e 100644 --- a/src/core/operators/test_audio_vec_embedding.py +++ b/src/core/operators/test_audio_vec_embedding.py @@ -14,7 +14,7 @@ def tearDownClass(cls): # delete config files pass - # @skip + @skip def test_sample_audio_from_disk(self): audio_file_path = AudioFactory.make_from_file_on_disk(r'core/operators/sample_data/audio.wav') audio_emb = audio_vec_embedding.run(audio_file_path) @@ -26,4 +26,4 @@ def test_sample_audio_from_url(self): "https://raw.githubusercontent.com/tattle-made/feluda/main/src/core/operators/sample_data/audio.wav" ) audio_emb = audio_vec_embedding.run(audio_path) - self.assertEqual(2048, len(audio_emb)) \ No newline at end of file + self.assertEqual(2048, len(audio_emb)) From 148fd8d5eba83d023abd8869ec4c7408bf579955 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:47:31 +0530 Subject: [PATCH 4/6] ci: Fix yaml issue --- .ci/docker-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml index 35dc0ba9..4930f251 100644 --- a/.ci/docker-compose.yml +++ b/.ci/docker-compose.yml @@ -4,13 +4,7 @@ services: context: ./../src dockerfile: Dockerfile.test target: test -# env_file: ./../src/test.env command: | -# bash -c "python -c 'import time'; -# while ! curl store:9200 | grep -q 'You Know, for Search' -# && ! curl queue:15672 | grep -q ''; -# do python -c 'time.sleep(10)'; -# done; bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py" | bash -c "python -m unittest core.operators.test_vid_vec_rep_resnet" | bash -c "python -m unittest core.operators.test_audio_vec_embedding" From a320c493fff2ab4fdddd12bb0b68e0c728055d51 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:50:36 +0530 Subject: [PATCH 5/6] ci: Fix yaml issue --- .ci/docker-compose.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml index 4930f251..8ebbb856 100644 --- a/.ci/docker-compose.yml +++ b/.ci/docker-compose.yml @@ -4,10 +4,16 @@ services: context: ./../src dockerfile: Dockerfile.test target: test +# env_file: ./../src/test.env command: | - bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py" | - bash -c "python -m unittest core.operators.test_vid_vec_rep_resnet" | - bash -c "python -m unittest core.operators.test_audio_vec_embedding" +# bash -c "python -c 'import time'; +# while ! curl store:9200 | grep -q 'You Know, for Search' +# && ! curl queue:15672 | grep -q ''; +# do python -c 'time.sleep(10)'; +# done; + bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py"; + python -m unittest core.operators.test_vid_vec_rep_resnet; + python -m unittest core.operators.test_audio_vec_embedding" # links: # - store # - queue From 93f1e51ce074ef7998b9719a6003e24416367a13 Mon Sep 17 00:00:00 2001 From: Aurora <5505558+duggalsu@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:05:09 +0530 Subject: [PATCH 6/6] ci: Fix docker yml --- .ci/docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ci/docker-compose.yml b/.ci/docker-compose.yml index 8ebbb856..5eac6ca6 100644 --- a/.ci/docker-compose.yml +++ b/.ci/docker-compose.yml @@ -5,15 +5,14 @@ services: dockerfile: Dockerfile.test target: test # env_file: ./../src/test.env - command: | + command: bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py"; + python -m unittest core.operators.test_vid_vec_rep_resnet; + python -m unittest core.operators.test_audio_vec_embedding" # bash -c "python -c 'import time'; # while ! curl store:9200 | grep -q 'You Know, for Search' # && ! curl queue:15672 | grep -q ''; # do python -c 'time.sleep(10)'; # done; - bash -c "python -m unittest discover -s ./tests/core/models -p "test_*.py"; - python -m unittest core.operators.test_vid_vec_rep_resnet; - python -m unittest core.operators.test_audio_vec_embedding" # links: # - store # - queue