diff --git a/CHANGELOG.md b/CHANGELOG.md index 63b55159..8af2f532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,15 +16,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for Object Storage Tenants in S3v4 object storage. The associated environment variable is `F7T_S3_TENANT` and it can be empty or be `null` or `none` when the tenant is not needed. Otherwise the tenant name has to be set. - The task that is returned from a successful `GET /jobs/acct` would returns the attribute `time`, which is `cputime` from slurm. The attribute will remain and `cputime` and `elapsed` will be also returned. Similarly, `time_left` is actually the time of termination of the jobs. `time_left` will remain for compatibility reasons, but `elapsed` attribute will also be returned. - Added `F7T_AUTH_ISSUER` to specify the JWT token issuer to be checked by Kong GW - - Removed `F7T_AUTH_REALM` and `F7T_AUTH_URL` which are no longer needed +- Removed `F7T_AUTH_REALM` and `F7T_AUTH_URL` which are no longer needed -## Changed +### Changed - CI/CD pipeline is now adapted to create helm charts images and push to a repository when TDS or Prod are tagged - Also secrets now can be managed from ExternalSecrets on K8s deployment - Deployment on TDS triggers ArgoCD deployment - Demo and k8s deployments have the Swagger UI API specification at unauthenticated `/docs` endpoint +### Fixed + +- Take into account `pageNumber` and `pageSize` arguments in `GET /compute/jobs` and `GET /compute/acct`. + ## [1.13.1] ### Added diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index d7e13cbf..f914d635 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -2456,14 +2456,9 @@ components: pageSize: name: pageSize in: query - description: Number of entries returned + description: Number of entries returned (default is set only when pageNumber is set) schema: type: integer - enum: - - 10 - - 25 - - 50 - - 100 default: 25 pageNumber: name: pageNumber diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index c6d5fe75..855562dc 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -2622,14 +2622,9 @@ components: pageSize: name: pageSize in: query - description: Number of entries returned + description: Number of entries returned (default is set only when pageNumber is set) schema: type: integer - enum: - - 10 - - 25 - - 50 - - 100 default: 25 pageNumber: name: pageNumber diff --git a/src/compute/compute.py b/src/compute/compute.py index 9b223162..587f6be1 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -598,22 +598,26 @@ def list_jobs(): pageSize = request.args.get("pageSize", None) pageNumber = request.args.get("pageNumber", None) - if pageSize != None and pageNumber != None: - try: - pageNumber = int(pageNumber) - pageSize = int(pageSize) - - if pageSize not in [10,25,50,100]: + if pageSize is not None or pageNumber is not None: + if pageSize is not None: + try: + pageSize = int(pageSize) + except ValueError: pageSize = 25 + app.logger.error("pageSize cannot be converted to integer, so default (25) will be used") + else: + # if not set, by default + pageSize = 25 - except ValueError: + if pageNumber is not None: + try: + pageNumber = int(pageNumber) + except ValueError: + pageNumber = 0 + app.logger.error("pageNumber cannot be converted to integer, so default (0) will be used") + else: + # if not set, by default pageNumber = 0 - pageSize = 25 - app.logger.error("Wrong pageNumber and/or pageSize") - else: - # if not set, by default - pageNumber = 0 - pageSize = 25 # by default empty job_aux_list = None @@ -690,25 +694,19 @@ def list_job_task(headers,system_name, system_addr,action,task_id,pageSize,pageN app.logger.info(f"Size jobs: {len(jobList)}") # pagination - totalSize = len(jobList) - pageNumber = float(pageNumber) - pageSize = float(pageSize) - - totalPages = int(ceil(float(totalSize) / float(pageSize))) - - if DEBUG_MODE: + if pageNumber is not None and pageSize is not None: + totalSize = len(jobList) + totalPages = int(ceil(float(totalSize) / float(pageSize))) app.logger.debug(f"Total Size: {totalSize} - Total Pages: {totalPages}") - if pageNumber < 0 or pageNumber > totalPages-1: - app.logger.warning(f"pageNumber ({pageNumber}) greater than total pages ({totalPages}), set to default = 0") - pageNumber = 0 - - beg_reg = int(pageNumber * pageSize) - end_reg = int( (pageNumber+1 * pageSize) -1 ) - - app.logger.info(f"Initial reg {beg_reg}, final reg: {end_reg}") + if pageNumber < 0 or pageNumber > totalPages-1: + app.logger.warning(f"pageNumber ({pageNumber}) greater than total pages ({totalPages}), set to default = 0") + pageNumber = 0 - jobList = jobList[beg_reg:end_reg + 1] + beg_reg = pageNumber * pageSize + end_reg = beg_reg + pageSize + app.logger.info(f"Initial reg {beg_reg}, final reg: {end_reg-1}") + jobList = jobList[beg_reg:end_reg] jobs = {} for job_index, jobinfo in enumerate(jobList): @@ -896,7 +894,7 @@ def cancel_job(jobid): return data, 400 -def acct_task(headers, system_name, system_addr, action, task_id): +def acct_task(headers, system_name, system_addr, action, task_id, pageSize, pageNumber): # exec remote command resp = exec_remote_command(headers, system_name, system_addr, action) @@ -918,6 +916,22 @@ def acct_task(headers, system_name, system_addr, action, task_id): return jobs = scheduler.parse_accounting_output(resp["msg"]) + app.logger.info(f"Size jobs: {len(jobs)}") + + # pagination + if pageNumber is not None and pageSize is not None: + totalSize = len(jobs) + totalPages = int(ceil(float(totalSize) / float(pageSize))) + app.logger.debug(f"Total Size: {totalSize} - Total Pages: {totalPages}") + + if pageNumber < 0 or pageNumber > totalPages-1: + app.logger.warning(f"pageNumber ({pageNumber}) greater than total pages ({totalPages}), set to default = 0") + pageNumber = 0 + + beg_reg = pageNumber * pageSize + end_reg = beg_reg + pageSize + app.logger.info(f"Initial reg {beg_reg}, final reg: {end_reg-1}") + jobs = jobs[beg_reg:end_reg] # as it is a json data to be stored in Tasks, the is_json=True update_task(task_id, headers, async_task.SUCCESS, jobs, is_json=True) @@ -960,11 +974,34 @@ def acct(): endtime = request.args.get("endtime","") # check optional parameter jobs=jobidA,jobidB,jobidC jobs = request.args.get("jobs", "") + pageSize = request.args.get("pageSize", None) + pageNumber = request.args.get("pageNumber", None) if jobs != "": v = validate_input(jobs) if v != "": return jsonify(description="Failed to retrieve account information", error=f"'jobs' {v}"), 400 + if pageSize is not None or pageNumber is not None: + if pageSize is not None: + try: + pageSize = int(pageSize) + except ValueError: + pageSize = 25 + app.logger.error("pageSize cannot be converted to integer, so default (25) will be used") + else: + # if not set, by default + pageSize = 25 + + if pageNumber is not None: + try: + pageNumber = int(pageNumber) + except ValueError: + pageNumber = 0 + app.logger.error("pageNumber cannot be converted to integer, so default (0) will be used") + else: + # if not set, by default + pageNumber = 0 + sched_cmd = scheduler.accounting( jobids=jobs.split(','), start_time=starttime, @@ -984,7 +1021,7 @@ def acct(): # asynchronous task creation aTask = threading.Thread(target=acct_task, name=ID, - args=(headers, system_name, system_addr, action, task_id)) + args=(headers, system_name, system_addr, action, task_id, pageSize, pageNumber)) aTask.start() task_url = f"{KONG_URL}/tasks/{task_id}"