Merge dev into stage #249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed

jgu2 wants to merge 29 commits into stage from dev

calliope_app/api/admin.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -9,11 +9,13 @@ @@
         Scenario_Loc_Tech, Scenario_Param, Job_Meta, Carrier
     from api.models.outputs import Run
     from api.models.engage import User_Profile, ComputeEnvironment
+    from api.forms import ComputeEnvironmentModelForm
     class ComputeEnvironmentAdmin(admin.ModelAdmin):
+        form = ComputeEnvironmentModelForm
         filter_horizontal = ("users",)
-        list_display = ['id',  'name', 'full_name', 'is_default', 'solver', 'ncpu', 'memory', 'type', '_users']
+        list_display = ['id',  'name', 'full_name', 'is_default', 'solvers', 'ncpu', 'memory', 'type', '_users']
         @staticmethod
         def _users(instance):
@@ Expand Down @@

calliope_app/api/calliope_utils.py

Large diffs are not rendered by default.

calliope_app/api/engage.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,6 +6,28 @@ @@
     from django.core.mail.message import sanitize_address
+    ENGAGE_SOLVERS = [
+        {
+            "name": "appsi_highs",
+            "pretty_name": "HiGHS",
+            "order": 1,
+            "is_active": True
+        },
+        {
+            "name": "cbc",
+            "pretty_name": "CBC",
+            "order": 2,
+            "is_active": True
+        },
+        {
+            "name": "amplxpress",
+            "pretty_name": "Xpress",
+            "order": 3,
+            "is_active": False
+        }
+    ]
     def aws_ses_configured():
         """
         Check the configuration of AWS SES settings
@@ Expand Down @@

calliope_app/api/forms.py

-Original file line number
+Diff line change
@@ -0,0 +1,15 @@
+    from django import forms
+    from client.widgets import JSONEditorWidget
+    from api.models.engage import ComputeEnvironment
+    class ComputeEnvironmentModelForm(forms.ModelForm):
+        class Meta:
+            model = ComputeEnvironment
+            fields = '__all__'
+            widgets = {
+                'solvers': JSONEditorWidget()
+            }

calliope_app/api/migrations/0068_remove_computeenvironment_solver_and_more.py

-Original file line number
+Diff line change
@@ -0,0 +1,23 @@
+    # Generated by Django 4.2.15 on 2024-09-17 03:26
+    import api.models.engage
+    from django.db import migrations, models
+    class Migration(migrations.Migration):
+        dependencies = [
+            ('api', '0067_auto_20240613_1706'),
+        ]
+        operations = [
+            migrations.RemoveField(
+                model_name='computeenvironment',
+                name='solver',
+            ),
+            migrations.AddField(
+                model_name='computeenvironment',
+                name='solvers',
+                field=models.JSONField(default=api.models.engage.default_solvers),
+            ),
+        ]

calliope_app/api/models/engage.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,8 @@ @@
     from django.urls import reverse
     from django.utils.html import mark_safe
+    from api.engage import ENGAGE_SOLVERS
     logger = logging.getLogger(__name__)
@@ Expand Down Expand Up / @@ -102,6 +104,10 @@ def activate(cls, activation_uuid): @@
             return True
+    def default_solvers():
+        return ENGAGE_SOLVERS
     class ComputeEnvironment(models.Model):
         ENV_TYPES = [
@@ Expand All / @@ -114,11 +120,11 @@ class ComputeEnvironment(models.Model): @@
         full_name = models.CharField(max_length=120)
         is_default = models.BooleanField(default=False)
         type = models.CharField(max_length=60, choices=ENV_TYPES)
-        solver = models.CharField(max_length=60, null=True, blank=True)
         ncpu = models.PositiveSmallIntegerField(null=True, blank=True)
         memory = models.PositiveSmallIntegerField(null=True, blank=True)
         cmd = models.TextField(blank=True, null=True)
         users = models.ManyToManyField(User, related_name="compute_environments", blank=True)
+        solvers = models.JSONField(default=default_solvers)
         class Meta:
             db_table = "compute_environments"
@@ Expand All / @@ -141,4 +147,4 @@ class Meta: @@
             verbose_name_plural = "[Admin] Request Rate Limits"
         def __str__(self):
-            return f"{self.year}, {self.month}, {self.total}"
+            return f"{self.year}, {self.month}, {self.total}"

calliope_app/api/urls.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -133,6 +133,8 @@ @@
         path('upload_outputs/',
              outputs_views.upload_outputs,
              name='upload_outputs'),
+        path('solvers/', outputs_views.solvers,
+             name="solvers"),
         # Bulk Data
         path('upload_locations/',
@@ Expand Down @@

calliope_app/api/views/outputs.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -3,16 +3,13 @@
  
    import io

    import logging

    import os

    import shutil

    import zipfile

    import sys

    from re import match

    from datetime import datetime, timedelta

    from urllib.parse import urljoin

    import requests

    import pandas as pd

    import pint

    from celery import current_app,chain

    from django.views.decorators.csrf import csrf_protect

    @@ -30,19 +27,45 @@
  
    from api.models.calliope import Abstract_Tech, Abstract_Tech_Param, Parameter, Run_Parameter

    from api.models.configuration import (

        Model, ParamsManager, User_File, Location, Technology,

        Tech_Param, Loc_Tech, Loc_Tech_Param, Timeseries_Meta, Carrier, Scenario_Param

        Tech_Param, Loc_Tech, Loc_Tech_Param, Timeseries_Meta, Carrier

    )

    from api.models.engage import ComputeEnvironment

    from api.engage import ENGAGE_SOLVERS

    from api.utils import zip_folder, initialize_units, convert_units, noconv_units

    from batch.managers import AWSBatchJobManager

    from taskmeta.models import CeleryTask, BatchTask, batch_task_status

    from calliope_app.celery import app

    logger = logging.getLogger(__name__)

    @csrf_protect

    def solvers(request):

        env_name = request.GET.get("env_name", None)

        if not env_name:

            env_name = "default"

        flag = True

        try:

            env = ComputeEnvironment.objects.get(name=env_name)

        except ComputeEnvironment.DoesNotExist:

            flag = False

        if (not flag) or (not env.solvers) or (not isinstance(env.solvers, list)):

            solvers = ENGAGE_SOLVERS

        else:

            solvers = env.solvers

        candidates = []

        for solver in solvers:

            is_active = solver.get("is_active", "false")

            if (is_active is True) or (is_active == "true"):

                candidates.append(solver)

        payload = sorted(candidates, key=lambda x: x["order"])

        return HttpResponse(json.dumps(payload), content_type="application/json")

    @csrf_protect

    def build(request):

        """

    @@ -156,13 +179,13 @@ def build(request):
  
                    )

                inputs_path = inputs_path.lower().replace(" ", "-")

                os.makedirs(inputs_path, exist_ok=True)

                run.run_options = []

                for id in parameters.keys():

                    run_parameter= Run_Parameter.objects.get(pk=int(id))

                    run.run_options.append({'root':run_parameter.root,'name':run_parameter.name,'value':parameters[id]})

                # Celery task        

                # Celery task

                async_result = build_model.apply_async(

                    kwargs={

                        "inputs_path": inputs_path,

    @@ -300,7 +323,7 @@ def optimize(request):
  
                    r.batch_job.status = batch_task_status.FAILED

                r.batch_job.save()

                r.save()

            if not all_complete:

                payload = {

                    "status": "BLOCKED",

    @@ -344,7 +367,7 @@ def optimize(request):
  
                        else:

                            logger.info("Found a subsequent gradient model for year %s but it was not built.",next_run.year)

                            break

        # Unknown environment, not supported

        else:

            raise Exception("Failed to submit job, unknown compute environment")

calliope_app/calliope-files/backend/run.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -152,10 +152,10 @@ def run_plan(
  
                # NOTE: pyomo==6.7.0 opt does not have name attribute

                # Disable this for use 'appsi_highs' solver, which does not have 'persistent' in name.

                # if "persistent" in opt.name and persistent is True:

                #     results.attrs["objective_function_value"] = opt.get_model_attr("ObjVal")

                # else:

                results.attrs["objective_function_value"] = backend_model.obj()

                if "appsi_highs" not in solver and  "persistent" in opt.name and persistent is True:

                    results.attrs["objective_function_value"] = opt.get_model_attr("ObjVal")

                else:

                    results.attrs["objective_function_value"] = backend_model.obj()

            else:

                results = xr.Dataset(attrs={"termination_condition": termination})

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge dev into stage #249

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!