diff --git a/.gitignore b/.gitignore index 135b4176..9b9f9587 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ scripts/export_var*.sh scripts/data.plt # ignore benchmark results scripts/*/ + +# locust auth files +locust_auth.json diff --git a/locust/README.md b/locust/README.md new file mode 100644 index 00000000..934b1268 --- /dev/null +++ b/locust/README.md @@ -0,0 +1,72 @@ +# GeoRepo Load Testing Using Locust + +## Description + +Load test using Locust. + +- Python based class +- Easy to generate scenario test using python +- Nice UI and charts that updates in real time + + +## Authentication Config + +Create a json file under locust directory called `locust_auth.json`. +Below is the sample: + +``` +[ + { + "username": "YOUR_USERNAME", + "api_key": "YOUR_PASSWORD", + "wait_time_start": null, + "wait_time_end": null + } +] +``` + +We can configure `wait_time_start` and `wait_time_end` for each user. If it is null, then the wait_time by default is a constant 1 second. + + +## Usage: Virtual env + +1. Create virtual environment +``` +mkvirtualenv geo_locust +``` + +Or activate existing virtual environment +``` +workon geo_locust +``` + +2. Install locust +``` +pip3 install locust +``` + +3. Run locust master +``` +locust -f locustfiles --class-picker +``` + +Web UI is available on http://localhost:8089/ + + +## Usage: Docker Compose + +TODO: docker compose for running locust + + +## Using Locust Web UI + +TODO: add screenshots. + +To start a new test: +1. Pick one or more the User class +2. (Optional) Configure tags in User class +3. Set number of users +4. Set ramp up +5. Set the host +6. (Advanced Options) Set maximum run time +7. Click Start diff --git a/locust/common/__init__.py b/locust/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/locust/common/api.py b/locust/common/api.py new file mode 100644 index 00000000..d51a206d --- /dev/null +++ b/locust/common/api.py @@ -0,0 +1,242 @@ +# coding=utf-8 +""" +GeoRepo. + +.. note:: API Class for Locust Load Testing +""" + +import time +import random +from json import JSONDecodeError + + +class ApiTaskTag: + """Represent the tag for a task.""" + + pass + + +class ApiPageResult: + """Represent Api response with pagination.""" + + def __init__(self, json_dict): + """Initialize the class.""" + self.page = json_dict['page'] + self.total_page = json_dict['total_page'] + self.page_size = json_dict['page_size'] + self.results = json_dict['results'] + + def random_item(self): + """Get random item from results list.""" + if self.results is None or len(self.results) == 0: + return {} + + return random.choice(self.results) + +class Api: + """Provides api call to GeoRepo.""" + + DEFAULT_MODULE = 'Admin Boundaries' + + DEFAULT_HEADERS = { + 'user-agent': ( + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36' + ) + } + + def __init__(self, client, user): + """Initialize the class.""" + self.client = client + self.user = user + + def _get_headers(self): + """Get headers for API Call.""" + headers = { + 'Authorization': 'Token ' + self.user['api_key'], + 'GeoRepo-User-Key': self.user['username'], + } + headers.update(self.DEFAULT_HEADERS) + return headers + + def _build_api_endpoint_pagination(self, endpoint, page, page_size): + """Build url to api endpoint with pagination request.""" + url = endpoint + if '?' in url: + url += f'&page{page}&page_size={page_size}' + else: + url += f'?page{page}&page_size={page_size}' + + return url + + def wait(self): + """Wait for another API call.""" + time.sleep(self.user['wait_time'](self)) + + def _api_get_list(self, api_endpoint, request_name): + """Call GET API that returns page result.""" + result = None + with ( + self.client.get( + api_endpoint, + catch_response=True, + headers=self._get_headers(), + name=request_name + ) + ) as response: + try: + result = ApiPageResult(response.json()) + except JSONDecodeError: + response.failure( + "Response could not be decoded as JSON" + ) + except KeyError: + response.failure( + "Response did not contain expected key" + ) + return result + + def _api_get_detail(self, api_endpoint, request_name): + """Call GET API that returns dictionary.""" + result = None + with ( + self.client.get( + api_endpoint, + catch_response=True, + headers=self._get_headers(), + name=request_name + ) + ) as response: + try: + result = response.json() + except JSONDecodeError: + response.failure( + "Response could not be decoded as JSON" + ) + except KeyError: + response.failure( + "Response did not contain expected key" + ) + return result + + def _api_get_page( + self, api_endpoint, request_name, page=None, page_size=50): + """Fetch GET API for given page. + + if page is None, then it will randomize a page. + """ + url = self._build_api_endpoint_pagination( + api_endpoint, + page if page else 1, + page_size + ) + result = self._api_get_list(url, request_name) + + if result and page is None and result.total_page > 1: + # check if need to fetch random page + rand_page = random.randint(1, result.total_page) + if rand_page != 1: + self.wait() + url = self._build_api_endpoint_pagination( + api_endpoint, + rand_page, + page_size + ) + result = self._api_get_list(url, request_name) + return result + + def module_list(self): + """Call module list API.""" + result = self._api_get_list( + '/api/v1/search/module/list/', + 'module_list' + ) + + if result is None: + return None + + # parse the admin boundaries uuid + module_uuid = None + for module in result.results: + if ( + module.get('name', '') == self.DEFAULT_MODULE + ): + module_uuid = module.get('uuid', None) + break + return module_uuid + + def dataset_list(self, module_uuid): + """Call dataset list API.""" + if module_uuid is None: + return None + return self._api_get_page( + f'/api/v1/search/module/{module_uuid}/dataset/list/', + 'dataset_list' + ) + + def dataset_detail(self, dataset_uuid): + """Call dataset detail API.""" + self.client.get( + f'/api/v1/search/dataset/{dataset_uuid}/', + headers=self._get_headers(), + name='dataset_detail' + ) + + def view_list(self, dataset_uuid): + """Call view list API.""" + if dataset_uuid is None: + return None + return self._api_get_page( + f'/api/v1/search/dataset/{dataset_uuid}/view/list/', + 'view_list' + ) + + def view_detail(self, view_uuid): + """Call view detail API.""" + if view_uuid is None: + return None + return self._api_get_detail( + f'/api/v1/search/view/{view_uuid}/', + 'view_detail' + ) + + def view_list_by_user(self): + """Call view list by user API.""" + pass + + def view_centroid(self): + """Call view centroid API.""" + pass + + def find_entity_by_level(self, view_uuid, admin_level): + """Call entity list by admin level API.""" + if view_uuid is None: + return None + return self._api_get_page( + f'/api/v1/search/view/{view_uuid}/entity/level/{admin_level}/', + f'entity_by_level_{admin_level}' + ) + + def find_entity_by_ucode(self): + """Call find entity by ucode API.""" + pass + + def find_entity_by_concept_ucode(self): + """Call find entity by concept ucode API.""" + pass + + def find_entity_by_id(self): + """Call find entity by id API.""" + pass + + def find_entity_by_level_and_parent_ucode(self): + """Call find entity by level and parent_ucode API.""" + pass + + def find_entity_by_level_and_parent_cucode(self): + """Call find entity by level and parent cucode API.""" + pass + + def find_entity_list_by_level0(self): + """Call entity list by level 0 API.""" + pass diff --git a/locust/common/auth.py b/locust/common/auth.py new file mode 100644 index 00000000..565a833b --- /dev/null +++ b/locust/common/auth.py @@ -0,0 +1,39 @@ +# coding=utf-8 +""" +GeoRepo. + +.. note:: Auth for Locust Load Testing +""" + +import json +import random +from locust import between, constant + + +class AuthConfig: + """Auth users from config json file.""" + + DEFAULT_WAIT_TIME = 1 # 1 second + + def __init__(self, file_path='/mnt/locust/locust_auth.json'): + """Initialize the class.""" + with open(file_path, 'r') as json_file: + self.users = json.load(json_file) + + def get_user(self): + """Get random user.""" + user = random.choice(self.users) + + wait_time = constant(self.DEFAULT_WAIT_TIME) + if user['wait_time_start'] and user['wait_time_end']: + wait_time = between( + user['wait_time_start'], user['wait_time_end']) + + return { + 'username': user['username'], + 'api_key': user['api_key'], + 'wait_time': wait_time + } + + +auth_config = AuthConfig('locust_auth.json') diff --git a/locust/locustfiles/__init__.py b/locust/locustfiles/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/locust/locustfiles/entity.py b/locust/locustfiles/entity.py new file mode 100644 index 00000000..0d44c05d --- /dev/null +++ b/locust/locustfiles/entity.py @@ -0,0 +1,57 @@ +# coding=utf-8 +""" +GeoRepo-OS. + +.. note:: Class for Testing Search Entity +""" + +import random +from locust import HttpUser, task + +from common.auth import auth_config +from common.api import Api + + +class SearchEntityUserScenario(HttpUser): + """Search entity scenario for testing API.""" + + def on_start(self): + """Set the test.""" + self.api = Api(self.client, auth_config.get_user()) + + def wait_time(self): + """Get wait_time in second.""" + return self.api.user['wait_time'](self) + + @task + def search_entity_workflow(self): + """Workflow to search entity.""" + # get module list + module_uuid = self.api.module_list() + self.api.wait() + + # get dataset + dataset_list = self.api.dataset_list(module_uuid) + if not dataset_list: + return + self.api.wait() + + # get view list + dataset = dataset_list.random_item() + view_list = self.api.view_list(dataset.get('uuid', None)) + if not view_list: + return + self.api.wait() + + # get view detail + view = view_list.random_item() + view_detail = self.api.view_detail(view.get('uuid', None)) + if not view_detail or len(view_detail.get('dataset_levels', [])) == 0: + return + self.api.wait() + + admin_level = random.choice(view_detail.get('dataset_levels', [])) + self.api.find_entity_by_level( + view.get('uuid', None), + admin_level.get('level', 0) + ) diff --git a/locust/locustfiles/vector_tile.py b/locust/locustfiles/vector_tile.py new file mode 100644 index 00000000..bf343a2b --- /dev/null +++ b/locust/locustfiles/vector_tile.py @@ -0,0 +1,6 @@ +# coding=utf-8 +""" +GeoRepo-OS. + +.. note:: Class for Locust Load Testing +"""