Getting ready for next release with more updates (#62)

* Getting ready for next release with more updates * Fixed linter errors * Updated codecov
prabhuomkar · Mar 9, 2024 · ded158a · ded158a
1 parent 5531983
commit ded158a
Show file tree

Hide file tree

Showing 14 changed files with 127 additions and 79 deletions.
diff --git a/api/config/config.go b/api/config/config.go
@@ -63,11 +63,11 @@ type (
 		Faces                  bool   `envconfig:"SMRITI_ML_FACES" default:"true"`
 		PlacesProvider         string `envconfig:"SMRITI_ML_PLACES_PROVIDER" default:"openstreetmap"`
 		ClassificationProvider string `envconfig:"SMRITI_ML_CLASSIFICATION_PROVIDER" default:"pytorch"`
-		ClassificationParams   string `envconfig:"SMRITI_ML_CLASSIFICATION_PARAMS" default:"{\"file\":\"classification_v240229.pt\"}"`
+		ClassificationParams   string `envconfig:"SMRITI_ML_CLASSIFICATION_PARAMS" default:"{\"file\":\"classification_v240309.pt\"}"`
 		OCRProvider            string `envconfig:"SMRITI_ML_OCR_PROVIDER" default:"paddlepaddle"`
 		OCRParams              string `envconfig:"SMRITI_ML_OCR_PARAMS" default:"{\"det_model_dir\":\"det_onnx\",\"rec_model_dir\":\"rec_onnx\",\"cls_model_dir\":\"cls_onnx\"}"`
 		SearchProvider         string `envconfig:"SMRITI_ML_SEARCH_PROVIDER" default:"pytorch"`
-		SearchParams           string `envconfig:"SMRITI_ML_SEARCH_PARAMS" default:"{\"tokenizer_dir\":\"search_tokenizer\",\"processor_dir\":\"search_processor\",\"text_file\":\"search_text_v240229.pt\",\"vision_file\":\"search_vision_v240229.pt\"}"` //nolint:lll
+		SearchParams           string `envconfig:"SMRITI_ML_SEARCH_PARAMS" default:"{\"tokenizer_dir\":\"search_tokenizer\",\"processor_dir\":\"search_processor\",\"text_file\":\"search_text_v240309.pt\",\"vision_file\":\"search_vision_v240309.pt\"}"` //nolint:lll
 		FacesProvider          string `envconfig:"SMRITI_ML_FACES_PROVIDER" default:"pytorch"`
 		FacesParams            string `envconfig:"SMRITI_ML_FACES_PARAMS" default:"{\"minutes\":\"1\",\"face_threshold\":\"0.9\",\"model\":\"vggface2\",\"clustering\":\"annoy\"}"`
 		MetadataParams         string `envconfig:"SMRITI_ML_METADATA_PARAMS" default:"{\"thumbnail_size\":\"512\"}"`

diff --git a/api/internal/handlers/handlers.go b/api/internal/handlers/handlers.go
@@ -2,6 +2,7 @@ package handlers
 
 import (
 	"api/config"
+	"api/internal/models"
 	"api/pkg/cache"
 	"api/pkg/services/worker"
 	"fmt"
@@ -68,6 +69,12 @@ func getMediaItemFilters(ctx echo.Context) string {
 	if mediaItemCategory != "" {
 		filterQuery += fmt.Sprintf(" AND mediaitem_category = '%s'", mediaItemCategory)
 	}
+	mediaItemStatus := ctx.QueryParam("status")
+	if mediaItemStatus != "" && (mediaItemStatus == string(models.Unspecified) ||
+		mediaItemStatus == string(models.Ready) || mediaItemStatus == string(models.Processing) ||
+		mediaItemStatus == string(models.Failed)) {
+		filterQuery += fmt.Sprintf(" AND status = '%s'", mediaItemStatus)
+	}
 	return filterQuery
 }
 

diff --git a/codecov.yml b/codecov.yml
@@ -2,7 +2,7 @@ coverage:
   status:
     project:
       default:
-        target: 90%
+        target: 60%
         threshold: 10%
         paths:
           - api

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -21,11 +21,11 @@ services:
       SMRITI_STORAGE_SECRET_KEY: smritipass
     volumes:
       - ./storage:/storage:rw
-    deploy:
-      resources:
-        limits:
-          cpus: '0.2'
-          memory: 256M
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '0.2'
+    #       memory: 256M
   worker:
     container_name: worker
     build: worker
@@ -39,14 +39,15 @@ services:
       SMRITI_API_HOST: api
       PYTHONUNBUFFERED: '1'
       TOKENIZERS_PARALLELISM: false
+      SMRITI_LOG_LEVEL: DEBUG
     volumes:
       - ./storage:/storage:rw
       - ./models:/models:rw
-    deploy:
-      resources:
-        limits:
-          cpus: '1.2'
-          memory: 3G
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '1.2'
+    #       memory: 3G
   # infra services
   database:
     container_name: database
@@ -58,11 +59,11 @@ services:
       POSTGRES_DB: smriti
     ports:
       - '5432:5432'
-    deploy:
-      resources:
-        limits:
-          cpus: '0.2'
-          memory: 256M
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '0.2'
+    #       memory: 256M
   cache:
     container_name: cache
     image: redis:7.2.4-alpine
@@ -71,11 +72,11 @@ services:
       --requirepass smritipass
     ports:
       - '6379:6379'
-    deploy:
-      resources:
-        limits:
-          cpus: '0.2'
-          memory: 256M
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '0.2'
+    #       memory: 256M
   storage:
     container_name: storage
     image: minio/minio:RELEASE.2024-02-04T22-36-13Z
@@ -88,8 +89,8 @@ services:
       MINIO_ROOT_PASSWORD: smritipass
     command: server --address 0.0.0.0:9000 /storage
     restart: always
-    deploy:
-      resources:
-        limits:
-          cpus: '0.2'
-          memory: 256M
+    # deploy:
+    #   resources:
+    #     limits:
+    #       cpus: '0.2'
+    #       memory: 256M
diff --git a/docs/docs/dev-guide/roadmap.md b/docs/docs/dev-guide/roadmap.md
@@ -13,7 +13,7 @@
 - [x] Get Machine Learning Powered Faces & People Collection (v2023.08.31)
 - [x] Planning recurrent upgrades of third party libraries (v2023.08.31)
 - [x] Open up for contributions to build Mobile & Web Application (v2023.08.31)
-- [ ] Enhance Image & Video Support (v2024.02.29)
-- [ ] Improvements to E2E Test Suite (v2024.02.29)
+- [x] Enhance Image & Video Support (v2024.03.09)
+- [x] Improvements to E2E Test Suite (v2024.03.09)
 - [ ] Sync content from Social Media Websites (v2024.04.30)
 - [ ] Generation of Memories (v2024.04.30)
diff --git a/docs/docs/user-guide/features/search.md b/docs/docs/user-guide/features/search.md
@@ -3,7 +3,7 @@ Enable this feature using API Configuration:
 ```bash
 SMRITI_ML_SEARCH: true
 SMRITI_ML_SEARCH_PROVIDER: pytorch
-SMRITI_ML_SEARCH_PARAMS: {"tokenizer_dir":"search_tokenizer","processor_dir":"search_processor","text_file":"search_text_v240229.pt","vision_file":"search_vision_v240229.pt"}
+SMRITI_ML_SEARCH_PARAMS: {"tokenizer_dir":"search_tokenizer","processor_dir":"search_processor","text_file":"search_text_v240309.pt","vision_file":"search_vision_v240309.pt"}
 ```
 
 ## Use Cases

diff --git a/docs/docs/user-guide/features/things.md b/docs/docs/user-guide/features/things.md
@@ -4,7 +4,7 @@ Enable this feature using API Configuration:
 SMRITI_FEATURE_THINGS: true
 SMRITI_ML_CLASSIFICATION: true
 SMRITI_ML_CLASSIFICATION_PROVIDER: pytorch
-SMRITI_ML_CLASSIFICATION_PARAMS: {"file":"classification_v240229.pt"}
+SMRITI_ML_CLASSIFICATION_PARAMS: {"file":"classification_v240309.pt"}
 ```
 
 ## Use Cases

diff --git a/docs/package-lock.json b/docs/package-lock.json
diff --git a/docs/package.json b/docs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "smriti",
-  "version": "24.02.29",
+  "version": "24.03.09",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
@@ -2,7 +2,7 @@ openapi: 3.0.3
 info:
   title: Smriti API
   description: Smarter Home for all your Photos and Videos
-  version: 24.02.29
+  version: 24.03.09
 servers:
   - url: https://localhost:5001
 tags:
@@ -146,6 +146,12 @@ paths:
           schema:
             type: string
             example: live
+        - name: status
+          in: query
+          description: MediaItem Status
+          schema:
+            type: string
+            example: FAILED
       summary: Get all existing mediaItems
       description: Get all existing mediaItems
       operationId: getMediaItems
@@ -1323,7 +1329,7 @@ components:
       properties:
         version:
           type: string
-          example: 24.02.29
+          example: 24.03.09
         gitSha:
           type: string
           example: addf120b430021c36c232c99ef8d926aea2acd6b

diff --git a/scripts/generate_test_data.py b/scripts/generate_test_data.py
@@ -4,10 +4,11 @@
 import zipfile
 import shutil
 import time
+import multiprocessing as mp
 
 
 API_URL ='http://localhost:5001'
-DOWNLOAD_PHOTOS_URL = 'https://www.dropbox.com/sh/q7yqg7uufaflqjg/AABAo-QEwNIAjxyZSJD0ICzDa?dl=1'
+DOWNLOAD_SAMPLES_URL = 'https://www.dropbox.com/scl/fo/yyy82163nqh5ii5aqm8vz/h?rlkey=bjlvrz198fu9zntu6tmvgb2ya&dl=1'
 
 # create user
 res = requests.post(f'{API_URL}/v1/users', auth=('smriti', 'smritiT3st!'), json={'name':'Jeff Dean','username':'jeffdean','password':'jeffT3st!','features':'{"albums":true,'+
@@ -28,11 +29,29 @@
 print('✅ got auth token')
 headers = {'Authorization': f'Bearer {access_token}'}
 
+# upload function
+def upload(file_type, file):
+    files = {'file': open(f'samples/{file_type}/{file}', 'rb')}
+    res = requests.post(f'{API_URL}/v1/mediaItems', files=files, headers=headers)
+    if res.status_code != 201:
+        res = res.json()
+        print(f'❌ error uploading sample mediaitem: {res["message"]}')
+        exit(0)
+    res = res.json()
+    while True:
+        res = requests.get(f'{API_URL}/v1/mediaItems/{res["id"]}', files=files, headers=headers)
+        assert res.status_code == 200
+        res = res.json()
+        if res['status'] == 'READY':
+            break
+        time.sleep(5)
+    return res['id']
+
 # download and upload mediaitems
 mediaitems = []
 print('ℹ️ downloading sample mediaitems, hang on...')
 if not os.path.exists('samples.zip'):
-    response = requests.get(DOWNLOAD_PHOTOS_URL)
+    response = requests.get(DOWNLOAD_SAMPLES_URL)
     if response.status_code == 200:
         local_file_path = 'samples.zip'
         with open(local_file_path, 'wb') as local_file:
@@ -45,22 +64,10 @@
 with zipfile.ZipFile('samples.zip', 'r') as zip_ref:
     zip_ref.extractall('samples')
 print('ℹ️ uploading sample mediaitems, hang on...')
-for file in os.listdir('samples'):
-    files = {'file': open(f'samples/{file}', 'rb')}
-    res = requests.post(f'{API_URL}/v1/mediaItems', files=files, headers=headers)
-    if res.status_code != 201:
-        res = res.json()
-        print(f'❌ error uploading sample mediaitem: {res["message"]}')
-        exit(0)
-    res = res.json()
-    mediaitems.append(res['id'])
-    while True:
-        res = requests.get(f'{API_URL}/v1/mediaItems/{res["id"]}', files=files, headers=headers)
-        assert res.status_code == 200
-        res = res.json()
-        if res['status'] == 'READY':
-            break
-        time.sleep(2)
+for file_type in os.listdir('samples'):
+    files = os.listdir(f'samples/{file_type}')
+    with mp.Pool(processes=mp.cpu_count()-1) as pool:
+        mediaitems = pool.starmap(upload, list(zip([file_type for _ in range(len(files))], files)), chunksize=2)
 print('✅ uploaded sample mediaitems')
 
 # create albums

diff --git a/worker/requirements.txt b/worker/requirements.txt
@@ -10,7 +10,7 @@ opencv-python==4.9.0.80
 prometheus-client==0.19.0
 Pillow==10.2.0
 PyExifTool==0.5.6
-rawpy==0.19.0
+rawpy==0.19.1
 requests==2.31.0
 schedule==1.2.1
 torch==2.2.0