Fix/materials myopic

temoa/temoa_model/hybrid_loader.py

@@ -101,6 +101,7 @@ def __init__(self, db_connection: Connection, config: TemoaConfig):
         self.viable_rt: ViableSet | None = None
         self.viable_rpit: ViableSet | None = None
         self.viable_rtt: ViableSet | None = None  # to support scanning LinkedTech
+        self.viable_rtv_eol: ViableSet | None = None  # to support scanning EndOfLifeOutput
         self.efficiency_values: list[tuple] = []
 
         # container for loaded data
@@ -175,6 +176,7 @@ def _build_efficiency_dataset(
             self.viable_rt = filts['rt']
             self.viable_rpit = filts['rpit']
             self.viable_rpto = filts['rpto']
+            self.viable_rtv_eol = filts['rtv_eol']
             self.viable_techs = filts['t']
             self.viable_input_comms = filts['ic']
             self.viable_vintages = filts['v']
@@ -977,10 +979,19 @@ def load_indexed_set(indexed_set: Set, index_value, element, element_validator=N
 
         # EmissionEmbodied
         if self.table_exists('EmissionEmbodied'):
-            raw = cur.execute(
-                'SELECT region, emis_comm, tech, vintage, value '
-                'FROM main.EmissionEmbodied'
-            ).fetchall()
+            if mi:
+                qry = (
+                    'SELECT region, emis_comm, tech, vintage, value FROM main.EmissionEmbodied'
+                    ' WHERE vintage >= ? AND vintage <= ?'
+                )
+                raw = cur.execute(
+                    qry,
+                    (mi.base_year, mi.last_demand_year),
+                ).fetchall()
+            else:
+                raw = cur.execute(
+                    'SELECT region, emis_comm, tech, vintage, value FROM main.EmissionEmbodied'
+                ).fetchall()
             load_element(M.EmissionEmbodied, raw, self.viable_rtv, (0, 2, 3))
 
         # EmissionEndOfLife
@@ -993,17 +1004,27 @@ def load_indexed_set(indexed_set: Set, index_value, element, element_validator=N
 
         # ConstructionInput
         if self.table_exists('ConstructionInput'):
-            raw = cur.execute(
-                'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
-            ).fetchall()
+            if mi:
+                qry = (
+                    'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
+                    ' WHERE vintage >= ? AND vintage <= ?'
+                )
+                raw = cur.execute(
+                    qry,
+                    (mi.base_year, mi.last_demand_year),
+                ).fetchall()
+            else:
+                raw = cur.execute(
+                    'SELECT region, input_comm, tech, vintage, value FROM main.ConstructionInput'
+                ).fetchall()
             load_element(M.ConstructionInput, raw, self.viable_rtv, (0, 2, 3))
 
         # EndOfLifeOutput
         if self.table_exists('EndOfLifeOutput'):
             raw = cur.execute(
                 'SELECT region, tech, vintage, output_comm, value FROM main.EndOfLifeOutput'
             ).fetchall()
-            load_element(M.EndOfLifeOutput, raw, self.viable_rtv, (0, 1, 2))
+            load_element(M.EndOfLifeOutput, raw, self.viable_rtv_eol, (0, 1, 2))
 
         # LinkedTechs
         # Note:  Both of the linked techs must be viable.  As this is non period/vintage

temoa/temoa_model/model_checking/commodity_graph.py

@@ -89,7 +89,7 @@ def generate_graph(
     }
     cap_edges = {
         (tech.ic, tech.name, tech.oc) for tech in network_data.available_techs[region, period]
-        if tech.name in ('Construction','EndOfLife')
+        if tech.name == tech.ic or tech.name == tech.oc
     }
     exc_edges = {
         (tech.ic, tech.name, tech.oc) for tech in network_data.available_techs[region, period]

temoa/temoa_model/model_checking/commodity_network_manager.py

@@ -117,6 +117,7 @@ def build_filters(self) -> dict[str, ViableSet]:
         valid_rpit = set()
         valid_rpto = set()
         valid_t = set()
+        valid_rtv_eol = set()
         valid_input_commodities = set()
         valid_output_commodities = set()
         valid_vintages = set()
@@ -128,10 +129,17 @@ def build_filters(self) -> dict[str, ViableSet]:
                 valid_rpit.add((tech.region, p, tech.ic, tech.name))
                 valid_rpto.add((tech.region, p, tech.name, tech.oc))
                 valid_t.add(tech.name)
+                valid_rtv_eol.add((tech.region, tech.ic, tech.vintage))
                 valid_input_commodities.add(tech.ic)
                 valid_output_commodities.add(tech.oc)
                 valid_vintages.add(tech.vintage)
 
+        for r, p, t, v in self.filtered_data.silent_rptv:
+            valid_rtv.add((r, t, v))
+            valid_rt.add((r, t))
+            valid_t.add(t)
+            valid_vintages.add(v)
+
         filts = {
             'ritvo': ViableSet(
                 elements=valid_ritvo, exception_loc=0, exception_vals=ViableSet.REGION_REGEXES
@@ -148,6 +156,9 @@ def build_filters(self) -> dict[str, ViableSet]:
             'v': ViableSet(elements=valid_vintages),
             'ic': ViableSet(elements=valid_input_commodities),
             'oc': ViableSet(elements=valid_output_commodities),
+            'rtv_eol': ViableSet(
+                elements=valid_rtv_eol, exception_loc=0, exception_vals=ViableSet.REGION_REGEXES
+            ),
         }
         return filts
 

temoa/temoa_model/model_checking/network_model_data.py

@@ -65,6 +65,7 @@ def __init__(self, **kwargs):
             'available_techs'
         )
         self.available_linked_techs: set[LinkedTech] = kwargs.get('available_linked_techs', set())
+        self.silent_rptv: set[str] = kwargs.get('silent_rptv', set())
         # a catch-all for indicators for techs...growth potential
         # dev note:  this is indexed by tech name, and is blind to vintage.  The intended use is in the
         #            network graph, which is also blind to vintage.  So it is interpreted as "at least one"
@@ -82,6 +83,7 @@ def clone(self) -> Self:
             all_commodities=self.physical_commodities.copy(),
             available_techs=self.available_techs.copy(),
             available_linked_techs=self.available_linked_techs.copy(),
+            silent_rptv=self.silent_rptv.copy(),
         )
 
     @property
@@ -179,18 +181,21 @@ def _build_from_db(
     #            re-use some of the hybrid loader code in a clear way.  Not too much overlap, though
     res = NetworkModelData()
     cur = con.cursor()
+    raw = cur.execute('SELECT tech FROM Technology WHERE unlim_cap==1').fetchall()
+    tech_uncap = {t[0] for t in raw}
     raw = cur.execute('SELECT tech FROM Technology WHERE retire==1').fetchall()
     tech_retire = {t[0] for t in raw}
     raw = cur.execute('SELECT DISTINCT region, tech, vintage FROM LifetimeSurvivalCurve').fetchall()
     tech_survival_curve = set(raw)
-    raw = cur.execute('SELECT period FROM TimePeriod').fetchall()
+    raw = cur.execute('SELECT period FROM TimePeriod WHERE flag == "f"').fetchall()
     periods = [p[0] for p in sorted(raw)]
     period_length = {periods[i]: periods[i+1] - periods[i] for i in range(len(periods)-1)}
     periods = periods[:-1]
     raw = cur.execute("SELECT name FROM main.Commodity WHERE flag LIKE '%p%' OR flag = 's' OR flag LIKE '%a%'").fetchall()
     res.physical_commodities = {c[0] for c in raw}
     res.capacity_commodities = set()
     res.exchange_commodities = set()
+    res.silent_rptv = set()
     raw = cur.execute("SELECT Commodity.name FROM Commodity WHERE flag LIKE '%w%'").fetchall()
     waste_comms = {c[0] for c in raw}
     waste_dict = defaultdict(set)
@@ -233,19 +238,20 @@ def _build_from_db(
             '     AND main.MyopicEfficiency.region = main.LifeTimeTech.region '
             '   JOIN TimePeriod '
             '   ON MyopicEfficiency.vintage = TimePeriod.period '
-            # f'   WHERE main.MyopicEfficiency.vintage <= {myopic_index.last_demand_year}'
+            f'   WHERE main.MyopicEfficiency.vintage <= {myopic_index.last_demand_year}'
         )
     raw = cur.execute(query).fetchall()
     # need to exclude the final year which is a non-demand year and should have no tech data
     # This ensures that the periods in this will match the periods in the hybrid loader.
 
     # filter further if myopic
     if myopic_index:
-        periods = {
+        periods = [
             p for p in periods if myopic_index.base_year <= p <= myopic_index.last_demand_year
-        }
+        ]
     techs = defaultdict(set)
     living_techs = set()  # for screening the linked techs below
+    living_rtv = set()
     # filter out the dead ones...
     for element in raw:
         (r, ic, tech, v, oc, lifetime) = element
@@ -267,45 +273,96 @@ def _build_from_db(
                 else:
                     techs[r, p].add(Tech(r, ic, tech, v, oc))
                     living_techs.add(tech)
+                    living_rtv.add((r, tech, v))
                     if ic in source_comms:
                         source_dict[r, p].add(ic)
                     if oc in waste_comms:
                         waste_dict[r, p].add(oc)
 
-            # End of life output
-            if any((
-                p <= v+lifetime < p + period_length[p], # natural eol this period
-                tech in tech_retire and v < p <= v+lifetime - period_length[p], # allowed early retirement
-                (r, tech, v) in tech_survival_curve and v <= p <= v+lifetime
-            )):
-                try:
-                    raw_eol = cur.execute(
-                        'SELECT region, tech, vintage, output_comm FROM EndOfLifeOutput '
-                        f' WHERE region == "{r}" AND tech == "{tech}" AND vintage == {v}'
-                    ).fetchall()
-
-                    for _r, _tech, _v, _oc in raw_eol:
-                        techs[_r, p].add(Tech(_r, _tech, 'EndOfLife', _v, _oc))
-                        source_dict[_r, p].add(_tech)
-                        res.capacity_commodities.add(_tech)
-                        living_techs.add(_tech)
-                        if _oc in waste_comms:
-                            waste_dict[_r, p].add(_oc)
-                except:
-                    # EndOfLifeOutput table did not exist TODO remove this eventually
-                    pass
+    # ExistingCapacity for checking
+    query = (
+        'SELECT region, tech, vintage, capacity FROM main.ExistingCapacity'
+    )
+    raw = cur.execute(query).fetchall()
+    exs_cap = dict()
+    for r, tech, v, cap in raw:
+        exs_cap[r, tech, v] = cap
+
+    # End of life output
+    query = (
+        '  SELECT main.EndOfLifeOutput.region, EndOfLifeOutput.tech, EndOfLifeOutput.vintage, EndOfLifeOutput.output_comm,  '
+        f'  coalesce(main.LifetimeProcess.lifetime, main.LifetimeTech.lifetime, {default_lifetime}) AS lifetime '
+        '   FROM main.EndOfLifeOutput '
+        '    LEFT JOIN main.LifetimeProcess '
+        '       ON main.EndOfLifeOutput.tech = LifetimeProcess.tech '
+        '       AND main.EndOfLifeOutput.vintage = LifetimeProcess.vintage '
+        '       AND main.EndOfLifeOutput.region = LifetimeProcess.region '
+        '    LEFT JOIN main.LifetimeTech '
+        '       ON main.EndOfLifeOutput.tech = main.LifetimeTech.tech '
+        '     AND main.EndOfLifeOutput.region = main.LifeTimeTech.region '
+        '   JOIN TimePeriod '
+        '   ON EndOfLifeOutput.vintage = TimePeriod.period '
+    )
+    raw = cur.execute(query).fetchall()
+    for (r, tech, v, oc, lifetime) in raw:
+        if tech in tech_uncap:
+            # No capacity to retire
+            continue
+        if exs_cap.get((r, tech, v), 0) <= 0:
+            continue
+        for p in periods:
+            if (
+                (p == periods[0] and v + lifetime == p) # retires on start of horizon
+                or (
+                    (r, tech, v) in living_rtv and any((
+                        p <= v+lifetime < p + period_length[p], # natural eol this period
+                        tech in tech_retire and v < p <= v+lifetime - period_length[p], # allowed early retirement
+                        (r, tech, v) in tech_survival_curve and v <= p <= v+lifetime # survival curve retirement
+                    ))
+                )
+            ):
+                techs[r, p].add(Tech(r, tech, tech, v, oc))
+                source_dict[r, p].add(tech)
+                res.capacity_commodities.add(tech)
+                if oc in waste_comms:
+                    waste_dict[r, p].add(oc)
+
+    # Emission end of life
+    query = (
+        '  SELECT main.EmissionEndOfLife.region, EmissionEndOfLife.tech, EmissionEndOfLife.vintage,  '
+        f'  coalesce(main.LifetimeProcess.lifetime, main.LifetimeTech.lifetime, {default_lifetime}) AS lifetime '
+        '   FROM main.EmissionEndOfLife '
+        '    LEFT JOIN main.LifetimeProcess '
+        '       ON main.EmissionEndOfLife.tech = LifetimeProcess.tech '
+        '       AND main.EmissionEndOfLife.vintage = LifetimeProcess.vintage '
+        '       AND main.EmissionEndOfLife.region = LifetimeProcess.region '
+        '    LEFT JOIN main.LifetimeTech '
+        '       ON main.EmissionEndOfLife.tech = main.LifetimeTech.tech '
+        '     AND main.EmissionEndOfLife.region = main.LifeTimeTech.region '
+        '   JOIN TimePeriod '
+        '   ON EmissionEndOfLife.vintage = TimePeriod.period '
+    )
+    raw = cur.execute(query).fetchall()
+    for (r, tech, v, lifetime) in raw:
+        if tech in tech_uncap:
+            # No capacity to retire
+            continue
+        if exs_cap.get((r, tech, v), 0) <= 0:
+            continue
+        if v + lifetime == periods[0]:
+            res.silent_rptv.add((r, periods[0], tech, v))
 
     # Construction input
-    try:
-        raw = cur.execute('SELECT region, input_comm, tech, vintage FROM ConstructionInput').fetchall()
-        for r, ic, tech, v in raw:
-            techs[r, v].add(Tech(r, ic, 'Construction', v, tech))
-            demand_dict[r, v].add(tech)
-            res.capacity_commodities.add(tech)
-            living_techs.add(tech)
-    except:
-        # ConstructionInput table did not exist TODO remove this eventually
-        pass
+    raw = cur.execute('SELECT region, input_comm, tech, vintage FROM ConstructionInput').fetchall()
+    for r, ic, tech, v in raw:
+        if tech in tech_uncap:
+            # No capacity to construct
+            continue
+        if v not in periods:
+            continue
+        techs[r, v].add(Tech(r, ic, tech, v, tech))
+        demand_dict[r, v].add(tech)
+        res.capacity_commodities.add(tech)
 
     res.available_techs = techs
     res.demand_commodities = demand_dict