Cleanup/manual review of feature/cagen-v1 part 1 + 2 minor improvements

.github/workflows/end-to-end-tests.yaml

@@ -3,7 +3,7 @@ name: capgen end-to-end tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [develop]
+    branches: [develop, feature/capgen-v1]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/unit-tests.yaml

@@ -3,7 +3,7 @@ name: capgen unit tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [develop]
+    branches: [develop, feature/capgen-v1]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

capgen/ccpp_datafile.py

@@ -19,6 +19,8 @@
   * ``--suite-list``
   * ``--required-variables`` / ``--input-variables`` /
     ``--output-variables`` / ``--host-variables``
+  * ``--suite-variables`` — the suite-owned (interstitial) variables
+    promoted into ``ccpp_<suite>_data.F90`` (a capgen addition)
   * ``--show`` (pretty-print)
   * ``--separator``, ``--exclude-protected``, ``--line-wrap``, ``--indent``
 
@@ -93,6 +95,10 @@
     {"report": "host_variables", "type": bool,
      "help": ("Return a list of required host model variable "
               "standard names")},
+    {"report": "suite_variables", "type": str,
+     "help": ("Return a list of suite-owned (interstitial) variable "
+              "standard names for suite, <SUITE_NAME>"),
+     "metavar": "SUITE_NAME"},
     {"report": "show", "type": bool,
      "help":
      "Pretty print the database contents to the screen"},
@@ -724,6 +730,37 @@ def _retrieve_variable_list(table, suite_name,
     return sorted(var_set)
 
 
+def _retrieve_suite_variable_list(table, suite_name):
+    """Find and return the sorted standard names of the suite-owned
+    (interstitial) variables promoted for suite <suite_name>.
+
+    Suite-owned variables are those no host table declares: first written
+    by a scheme with ``intent(out)`` and stored in ``ccpp_<suite>_data.F90``.
+    Returns an empty list if <suite_name> has no suite dictionary.
+
+    >>> table = ET.fromstring("<ccpp_datatable version='1.0'><var_dictionaries>"\
+            "<var_dictionary name='fruit' type='suite'><variables>"\
+            "<var name='var_b' local_name='vb'></var>"\
+            "<var name='var_a' local_name='va'></var>"\
+            "</variables></var_dictionary></var_dictionaries></ccpp_datatable>")
+    >>> _retrieve_suite_variable_list(table, 'fruit')
+    ['var_a', 'var_b']
+    >>> _retrieve_suite_variable_list(table, 'veggie')
+    []
+    """
+    result = set()
+    suite_dict = _find_var_dictionary(table, dict_name=suite_name,
+                                      dict_type="suite")
+    if suite_dict is not None:
+        svars = suite_dict.find("variables")
+        if svars is not None:
+            for var in svars:
+                name = var.get("name")
+                if name:
+                    result.add(name)
+    return sorted(result)
+
+
 def datatable_report(datatable, action, sep, exclude_protected=False):
     """Perform a lookup <action> on <datatable> and return the result."""
     if not action:
@@ -770,6 +807,8 @@ def datatable_report(datatable, action, sep, exclude_protected=False):
         result = _retrieve_variable_list(table, "host",
                                          exclude_protected=exclude_protected,
                                          intent_type="host")
+    elif action.action_is("suite_variables"):
+        result = _retrieve_suite_variable_list(table, action.value)
     else:
         result = ''
     if isinstance(result, list):

capgen/generator/datatable.py

@@ -340,7 +340,31 @@ def _build_var_dictionaries(
         suite_d.set('name', suite_resolution.suite_name)
         suite_d.set('type', 'suite')
         suite_d.set('parent', _API_DICT_NAME)
-        ET.SubElement(suite_d, 'variables')
+        s_vars = ET.SubElement(suite_d, 'variables')
+        # Suite-owned (interstitial) variables: promoted from a scheme's
+        # intent(out) that no host table declares, and stored in
+        # ccpp_<suite>_data.F90.  Recorded here so ccpp_datafile.py's
+        # --suite-variables report can enumerate them.
+        for std_name in sorted(suite_resolution.suite_vars):
+            svar = suite_resolution.suite_vars[std_name]
+            v = ET.SubElement(s_vars, 'var')
+            v.set('name', std_name)
+            if svar.local_name:
+                v.set('local_name', svar.local_name)
+            if svar.units:
+                v.set('units', svar.units)
+            if svar.type_:
+                v.set('type', svar.type_)
+            if svar.kind:
+                v.set('kind', svar.kind)
+            if svar.dimensions:
+                v.set('dimensions', ', '.join(svar.dimensions))
+            if svar.source_scheme:
+                v.set('source_scheme', svar.source_scheme)
+            if svar.source_phase:
+                v.set('source_phase', svar.source_phase)
+            if getattr(svar, 'allocatable', False):
+                v.set('allocatable', 'True')
 
         for resolved_group in suite_resolution.groups:
             group_d = ET.SubElement(dicts, 'var_dictionary')

capgen/generator/group_cap.py

@@ -517,21 +517,54 @@ def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str:
         else:
             is_identity = (arg.unit_forward == arg.call_expr)
         if not is_identity:
-            if reverse:
-                bits.append('unit conversion: {} to {}'.format(
-                    arg.kind_scheme or '', arg.kind_host or '',
-                ))
-            else:
-                bits.append('unit conversion: {} to {}'.format(
-                    arg.kind_host or '', arg.kind_scheme or '',
-                ))
+            if (arg.kind_scheme != arg.kind_host):
+                if reverse:
+                    bits.append('type conversion: {} to {}'.format(
+                        arg.kind_scheme or '', arg.kind_host or '',
+                    ))
+                else:
+                    bits.append('type conversion: {} to {}'.format(
+                        arg.kind_host or '', arg.kind_scheme or '',
+                    ))
+            if (arg.unit_scheme != arg.unit_host):
+                if reverse:
+                    bits.append('unit conversion: {} to {}'.format(
+                        arg.unit_scheme or '', arg.unit_host or '',
+                    ))
+                else:
+                    bits.append('unit conversion: {} to {}'.format(
+                        arg.unit_host or '', arg.unit_scheme or '',
+                    ))
     if arg.needs_vert_flip:
         bits.append('vertical flip (top_at_one mismatch)')
     if not bits:
         return ''
     return '! ' + '; '.join(bits)
 
 
+def _log_one_transform(logger, suite_name, group_name, phase, scheme_name,
+                       arg: ResolvedArg, reverse: bool) -> None:
+    """Emit one log line for a value-changing transform on *arg*.
+
+    Reuses :func:`_transform_comment` so the log text is *identical* to the
+    inline comment written into the generated cap.  A suppressed/identity
+    transform (comment == '') logs nothing, matching the cap.
+
+    TEMPORARY level choice: logged at WARNING so transforms show up in a
+    default capgen run (capgen's default log level is WARNING).
+    """
+    comment = _transform_comment(arg, reverse=reverse)
+    if not comment:
+        return
+    desc = comment[1:].strip()  # drop the leading '! '
+    when = 'post-call' if reverse else 'pre-call'
+    logger.warning(
+        "CCPP transform: %s/%s/%s  %s: %s (%s) [%s]  %s",
+        suite_name, group_name, phase, scheme_name,
+        arg.standard_name, arg.scheme_local_name, when, desc,
+    )
+
+
 def _active_required_guard_lines(
     arg: ResolvedArg,
     scheme_name: str,
@@ -688,6 +721,9 @@ def _emit_phase_items(
     phase: str = '',
     errflg_local: Optional[str] = None,
     errmsg_local: Optional[str] = None,
+    logger=None,
+    suite_name: str = '',
+    group_name: str = '',
 ) -> None:
     """Recursively emit Fortran for a list of :data:`PhaseItem` objects.
 
@@ -700,6 +736,7 @@ def _emit_phase_items(
         if isinstance(item, ResolvedCall):
             _emit_one_call(
                 item, indent, lines, phase, errflg_local, errmsg_local,
+                logger=logger, suite_name=suite_name, group_name=group_name,
             )
         elif isinstance(item, ResolvedSubcycle):
             counter = _loop_counter_name(depth)
@@ -711,6 +748,7 @@ def _emit_phase_items(
                 phase=phase,
                 errflg_local=errflg_local,
                 errmsg_local=errmsg_local,
+                logger=logger, suite_name=suite_name, group_name=group_name,
             )
             lines.append('{}end do'.format(indent))
             lines.append('')
@@ -723,8 +761,18 @@ def _emit_one_call(
     phase: str = '',
     errflg_local: Optional[str] = None,
     errmsg_local: Optional[str] = None,
+    logger=None,
+    suite_name: str = '',
+    group_name: str = '',
 ) -> None:
-    """Append Fortran lines for a single scheme call (with transforms + errcheck)."""
+    """Append Fortran lines for a single scheme call (with transforms + errcheck).
+
+    When *logger* is given, each emitted value-changing transform is also
+    logged (via :func:`_log_one_transform`) so a capgen run documents on
+    stdout/stderr the same conversions it writes as inline cap comments.
+    This covers group-cap calls AND the suite-level ``<init>``/``<final>``
+    hooks, which share this emitter (see ``suite_cap.py``).
+    """
     # Pre-call: runtime guard for any non-optional arg whose host declares
     # ``active = (...)``.  Emitted before transforms so an inactive-but-required
     # var bails out with a clear error rather than reading host memory through
@@ -735,9 +783,13 @@ def _emit_one_call(
             errflg_local, errmsg_local, indent,
         ))
 
-    # Pre-call transformations.
+    # Pre-call transformations (and log each on capgen's stdout/stderr).
     for arg in resolved_call.args:
         lines.extend(_pre_call_lines(arg))
+        if logger is not None and arg.unit_forward:
+            _log_one_transform(logger, suite_name, group_name,
+                               resolved_call.phase, resolved_call.scheme_name,
+                               arg, reverse=False)
 
     call_args_exprs = [
         '{}={}'.format(a.scheme_local_name, _call_arg_expr(a))
@@ -761,6 +813,10 @@ def _emit_one_call(
 
     for arg in resolved_call.args:
         lines.extend(_post_call_lines(arg))
+        if logger is not None and arg.unit_backward:
+            _log_one_transform(logger, suite_name, group_name,
+                               resolved_call.phase, resolved_call.scheme_name,
+                               arg, reverse=True)
     lines.append('')
 
 
@@ -851,6 +907,7 @@ def _generate_phase_subroutine(
     phase_items,
     ctrl_entries,
     host_dict,
+    logger=None,
 ) -> List[str]:
     """Generate one phase subroutine for a group cap.
 
@@ -978,11 +1035,15 @@ def _generate_phase_subroutine(
     )
 
     # ---- scheme calls ---------------------------------------------------
+    # Transforms are logged inside _emit_one_call (same text as the inline
+    # cap comments), so group caps and suite <init>/<final> hooks share one
+    # code path.
     _emit_phase_items(
         phase_items, call_indent, lines, depth=1,
         phase=phase,
         errflg_local=errflg_local,
         errmsg_local=errmsg_local,
+        logger=logger, suite_name=suite_name, group_name=group_name,
     )
 
     # ---- post-call state transitions ------------------------------------
@@ -1132,6 +1193,7 @@ def _generate_group_cap(
     resolved_group: ResolvedGroup,
     host_dict,
     trace: bool = False,
+    logger=None,
 ) -> List[str]:
     """Generate the full group cap module source lines.
 
@@ -1240,7 +1302,8 @@ def _generate_group_cap(
     for phase in _GROUP_PHASE_ORDER:
         phase_items = resolved_group.phase_calls.get(phase, [])
         sub_lines = _generate_phase_subroutine(
-            suite_name, group_name, phase, phase_items, ctrl_sig_entries, host_dict
+            suite_name, group_name, phase, phase_items, ctrl_sig_entries,
+            host_dict, logger=logger,
         )
         lines.extend(sub_lines)
         lines.append('')
@@ -1302,6 +1365,7 @@ def write_group_cap(
 
     lines = _generate_group_cap(
         suite_name, group_name, resolved_group, host_dict, trace=trace,
+        logger=logger,
     )
     with open_if_changed(out_path, logger=logger) as fh:
         fh.write('\n'.join(lines) + '\n')

capgen/generator/host_constituents.py

@@ -418,7 +418,7 @@ def _wrap_method_subs(host_dict) -> List[str]:
         'ccpp_const_get_index', 'const_index',
         [
             ('stdname', 'character(len=*), intent(in) :: stdname',
-             'standard_name=stdname'),
+             'standard_name=to_lower(stdname)'),
             ('const_index', 'integer, intent(out) :: const_index',
              'index=const_index'),
         ],
@@ -580,6 +580,7 @@ def _generate_host_constituents(
     lines.append('module {}'.format(_HOST_CONST_MOD))
     lines.append('')
     lines.append('{}use ccpp_kinds, only: kind_phys'.format(_INDENT))
+    lines.append('{}use ccpp_constituent_prop_mod, only: to_lower'.format(_INDENT))
     lines.append('{}use {}, only: &'.format(_INDENT, _CONST_PROP_MOD))
     lines.append('{}{}, &'.format(_INDENT * 2, _CONST_DDT))
     lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_TYPE))

capgen/generator/suite_cap.py

@@ -633,6 +633,7 @@ def _init_lines(
     suite_name: str,
     suite_res: SuiteResolution,
     host_dict=None,
+    logger=None,
 ) -> List[str]:
     """Generate the ``<suite>_init`` framework-setup subroutine lines.
 
@@ -769,7 +770,9 @@ def _init_lines(
     if suite_res.suite_init_call is not None:
         lines.append('')
         from generator.group_cap import _emit_one_call
-        _emit_one_call(suite_res.suite_init_call, i2, lines)
+        _emit_one_call(suite_res.suite_init_call, i2, lines,
+                       logger=logger, suite_name=suite_name,
+                       group_name='(suite init)')
 
     lines += [
         '',
@@ -786,6 +789,7 @@ def _final_lines(
     suite_name: str,
     suite_res: SuiteResolution,
     host_dict=None,
+    logger=None,
 ) -> List[str]:
     """Generate the ``<suite>_final`` framework-teardown subroutine lines.
 
@@ -905,7 +909,9 @@ def _final_lines(
     # transition to UNREGISTERED.  Errflg check follows the call.
     if suite_res.suite_final_call is not None:
         from generator.group_cap import _emit_one_call
-        _emit_one_call(suite_res.suite_final_call, i2, lines)
+        _emit_one_call(suite_res.suite_final_call, i2, lines,
+                       logger=logger, suite_name=suite_name,
+                       group_name='(suite final)')
 
     lines.append(
         '{}ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format(i2, inst_idx)
@@ -1215,6 +1221,7 @@ def _generate_suite_cap(
     scheme_store: SchemeStore,
     host_dict=None,
     trace: bool = False,
+    logger=None,
 ) -> List[str]:
     """Generate the full ``ccpp_<suite>_cap.F90`` module source lines.
 
@@ -1296,10 +1303,10 @@ def _generate_suite_cap(
 
     # Subroutines.  Order: register, init, physics_*, final, state_alloc/dealloc.
     lines.extend(_register_lines(suite_name, suite_res, host_dict))
-    lines.extend(_init_lines(suite_name, suite_res, host_dict))
+    lines.extend(_init_lines(suite_name, suite_res, host_dict, logger=logger))
     for phase in _PHYSICS_PHASES:
         lines.extend(_physics_dispatch_lines(suite_name, phase, suite_res, host_dict))
-    lines.extend(_final_lines(suite_name, suite_res, host_dict))
+    lines.extend(_final_lines(suite_name, suite_res, host_dict, logger=logger))
 
     has_suite_vars = bool(suite_res.suite_vars)
     lines.extend(_suite_state_alloc_lines(suite_name, has_suite_vars))
@@ -1346,6 +1353,7 @@ def write_suite_cap(
 
     lines = _generate_suite_cap(
         suite_name, suite_res, scheme_store, host_dict, trace=trace,
+        logger=logger,
     )
     with open_if_changed(out_path, logger=logger) as fh:
         fh.write('\n'.join(lines) + '\n')