Line numbers

schema_salad/python_codegen.py

@@ -274,10 +274,29 @@ def fromDoc(
         self.serializer.write(
             """
     def save(
-        self, top: bool = False, base_url: str = "", relative_uris: bool = True
-    ) -> Dict[str, Any]:
-        r: Dict[str, Any] = {}
-
+        self, top: bool = False, base_url: str = "", relative_uris: bool = True, keys: Optional[List[Any]] = None
+    ) -> CommentedMap:
+        if keys is None:
+            keys = []
+        r = CommentedMap()
+        doc = doc_line_info
+        for key in keys:
+
+            if isinstance(doc, CommentedMap):
+                doc = doc.get(key)
+            elif isinstance(doc, (CommentedSeq, list)) and isinstance(key, int):
+                if key < len(doc):
+                    doc = doc[key]
+                else:
+                    doc = None
+            else:
+                doc = None
+                break
+        if doc is not None:
+            r._yaml_set_line_col(doc.lc.line, doc.lc.col)
+        line_numbers = get_line_numbers(doc)
+        max_len = get_max_line_num(doc)
+        cols: Dict[int, int] = {}
         if relative_uris:
             for ef in self.extension_fields:
                 r[prefix_url(ef, self.loadingOptions.vocab)] = self.extension_fields[ef]
@@ -301,6 +320,7 @@ def save(
             self.serializer.write(
                 """
         r["class"] = "{class_}"
+        max_len = add_kv(old_doc=doc, new_doc=r, line_numbers=line_numbers, key="class", val=r.get("class"), max_len=max_len, cols=cols)
 """.format(
                     class_=classname
                 )
@@ -395,6 +415,7 @@ def type_loader(
             sub_names: List[str] = list(
                 dict.fromkeys([self.type_loader(i).name for i in type_declaration])
             )
+
             return self.declare_type(
                 TypeDef(
                     "union_of_{}".format("_or_".join(sub_names)),
@@ -566,12 +587,15 @@ def declare_field(
 if self.{safename} is not None:
     u = save_relative_uri(self.{safename}, {baseurl}, {scoped_id}, {ref_scope}, relative_uris)
     r["{fieldname}"] = u
+    max_len = add_kv(old_doc = doc, new_doc = r, line_numbers = line_numbers, key = "{key_1}", val = r.get("{key_2}"), max_len = max_len, cols = cols)
 """.format(
                         safename=self.safe_name(name),
                         fieldname=shortname(name).strip(),
                         baseurl=baseurl,
                         scoped_id=fieldtype.scoped_id,
                         ref_scope=fieldtype.ref_scope,
+                        key_1=self.safe_name(name),
+                        key_2=self.safe_name(name),
                     ),
                     8,
                 )
@@ -581,9 +605,16 @@ def declare_field(
                 fmt(
                     """
 if self.{safename} is not None:
-    r["{fieldname}"] = save(
-        self.{safename}, top=False, base_url={baseurl}, relative_uris=relative_uris
+    saved_val = save(
+        self.{safename}, top=False, base_url={baseurl}, relative_uris=relative_uris, keys = keys + ["{fieldname}"]
     )
+
+    if type(saved_val) == list:
+        if len(saved_val) == 1: # If the returned value is a list of size 1, just save the value in the list
+            saved_val = saved_val[0]
+    r["{fieldname}"] = saved_val
+
+    max_len = add_kv(old_doc = doc, new_doc = r, line_numbers = line_numbers, key = "{fieldname}", val = r.get("{fieldname}"), max_len = max_len, cols = cols)
 """.format(
                         safename=self.safe_name(name),
                         fieldname=shortname(name),

schema_salad/python_codegen_support.py

@@ -27,7 +27,7 @@
 
 from rdflib import Graph
 from rdflib.plugins.parsers.notation3 import BadSyntax
-from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from schema_salad.exceptions import SchemaSaladException, ValidationException
 from schema_salad.fetcher import DefaultFetcher, Fetcher, MemoryCachingFetcher
@@ -43,6 +43,9 @@
 IdxType = MutableMapping[str, Tuple[Any, "LoadingOptions"]]
 
 
+doc_line_info = CommentedMap()
+
+
 class LoadingOptions:
     idx: IdxType
     fileuri: Optional[str]
@@ -203,8 +206,12 @@ def fromDoc(
 
     @abstractmethod
     def save(
-        self, top: bool = False, base_url: str = "", relative_uris: bool = True
-    ) -> Dict[str, Any]:
+        self,
+        top: bool = False,
+        base_url: str = "",
+        relative_uris: bool = True,
+        keys: Optional[List[Any]] = None,
+    ) -> CommentedMap:
         """Convert this object to a JSON/YAML friendly dictionary."""
 
 
@@ -238,26 +245,154 @@ def load_field(val, fieldtype, baseuri, loadingOptions):
 ]
 
 
+def add_kv(
+    old_doc: CommentedMap,
+    new_doc: CommentedMap,
+    line_numbers: Dict[Any, Dict[str, int]],
+    key: str,
+    val: Any,
+    max_len: int,
+    cols: Dict[int, int],
+) -> int:
+    """Add key value pair into Commented Map.
+
+    Function to add key value pair into new CommentedMap given old CommentedMap, line_numbers for each key/val pair in the old CommentedMap,
+    key/val pair to insert, max_line of the old CommentedMap, and max col value taken for each line.
+    """
+    if key in line_numbers:  # If the key to insert is in the original CommentedMap
+        new_doc.lc.add_kv_line_col(key, old_doc.lc.data[key])
+    elif isinstance(val, (int, float, bool, str)):  # If the value is hashable
+        if val in line_numbers:  # If the value is in the original CommentedMap
+            line = line_numbers[val]["line"]
+            if line in cols:
+                col = max(line_numbers[val]["col"], cols[line])
+            else:
+                col = line_numbers[val]["col"]
+            new_doc.lc.add_kv_line_col(key, [line, col, line, col + len(key) + 2])
+            cols[line] = col + len("id") + 2
+        else:  # If neither the key or value is in the original CommentedMap (or value is not hashable)
+            new_doc.lc.add_kv_line_col(key, [max_len, 0, max_len, len(key) + 2])
+            max_len += 1
+    else:  # If neither the key or value is in the original CommentedMap (or value is not hashable)
+        new_doc.lc.add_kv_line_col(key, [max_len, 0, max_len, len(key) + 2])
+        max_len += 1
+    return max_len
+
+
+def get_line_numbers(doc: CommentedMap) -> Dict[Any, Dict[str, int]]:
+    """Get line numbers for kv pairs in CommentedMap.
+
+    For each key/value pair in a CommentedMap, save the line/col info into a dictionary,
+    only save value info if value is hashable.
+    """
+    line_numbers: Dict[Any, Dict[str, int]] = {}
+    if isinstance(doc, dict) or doc is None:
+        return {}
+    for key, value in doc.lc.data.items():
+        line_numbers[key] = {}
+
+        line_numbers[key]["line"] = doc.lc.data[key][0]
+        line_numbers[key]["col"] = doc.lc.data[key][1]
+        if isinstance(value, (int, float, bool, str)):
+            line_numbers[value] = {}
+            line_numbers[value]["line"] = doc.lc.data[key][2]
+            line_numbers[value]["col"] = doc.lc.data[key][3]
+    return line_numbers
+
+
+def get_max_line_num(doc: CommentedMap) -> int:
+    """Get the max line number for a CommentedMap.
+
+    Iterate through the the key with the highest line number until you reach a non-CommentedMap value or empty CommentedMap.
+    """
+    max_line = 0
+    max_key = ""
+    cur = doc
+    while isinstance(cur, CommentedMap) and len(cur) > 0:
+        for key in cur.lc.data.keys():
+            if cur.lc.data[key][2] >= max_line:
+                max_line = cur.lc.data[key][2]
+            max_key = key
+        cur = cur[max_key]
+    return max_line + 1
+
+
 def save(
     val: Any,
     top: bool = True,
     base_url: str = "",
     relative_uris: bool = True,
+    keys: Optional[List[Any]] = None,
 ) -> save_type:
+    """Save a val of any type.
+
+    Recursively calls save method from class if val is of type Saveable. Otherwise, saves val to CommentedMap or CommentedSeq
+    """
+    if keys is None:
+        keys = []
+    doc = doc_line_info
+    for key in keys:
+        if isinstance(doc, CommentedMap):
+            doc = doc.get(key)
+        elif isinstance(doc, (CommentedSeq, list)) and isinstance(key, int):
+            if key < len(doc):
+                doc = doc[key]
+            else:
+                doc = None
+        else:
+            doc = None
+            break
+
     if isinstance(val, Saveable):
-        return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
+        return val.save(
+            top=top, base_url=base_url, relative_uris=relative_uris, keys=keys
+        )
     if isinstance(val, MutableSequence):
-        return [
-            save(v, top=False, base_url=base_url, relative_uris=relative_uris)
-            for v in val
-        ]
+        r = CommentedSeq()
+        r.lc.data = {}
+        for i in range(0, len(val)):
+            new_keys = keys
+            if doc:
+                if i in doc:
+                    r.lc.data[i] = doc.lc.data[i]
+                    new_keys.append(i)
+            r.append(
+                save(
+                    val[i],
+                    top=False,
+                    base_url=base_url,
+                    relative_uris=relative_uris,
+                    keys=new_keys,
+                )
+            )
+        return r
+        # return [
+        #     save(v, top=False, base_url=base_url, relative_uris=relative_uris)
+        #     for v in val
+        # ]
     if isinstance(val, MutableMapping):
-        newdict = {}
+        newdict = CommentedMap()
+        new_keys = keys
         for key in val:
+            if doc:
+                if key in doc:
+                    newdict.lc.add_kv_line_col(key, doc.lc.data[key])
+                    new_keys.append(key)
+
             newdict[key] = save(
-                val[key], top=False, base_url=base_url, relative_uris=relative_uris
+                val[key],
+                top=False,
+                base_url=base_url,
+                relative_uris=relative_uris,
+                keys=new_keys,
             )
         return newdict
+        # newdict = {}
+        # for key in val:
+        #     newdict[key] = save(
+        #         val[key], top=False, base_url=base_url, relative_uris=relative_uris
+        #     )
+        # return newdict
     if val is None or isinstance(val, (int, float, bool, str)):
         return val
     raise Exception("Not Saveable: %s" % type(val))
@@ -697,7 +832,7 @@ def load(self, doc, baseuri, loadingOptions, docRoot=None):
 
 def _document_load(
     loader: _Loader,
-    doc: Union[str, MutableMapping[str, Any], MutableSequence[Any]],
+    doc: Union[CommentedMap, str, MutableMapping[str, Any], MutableSequence[Any]],
     baseuri: str,
     loadingOptions: LoadingOptions,
     addl_metadata_fields: Optional[MutableSequence[str]] = None,
@@ -729,11 +864,22 @@ def _document_load(
             addl_metadata=addl_metadata,
         )
 
-        doc = {
-            k: v
-            for k, v in doc.items()
-            if k not in ("$namespaces", "$schemas", "$base")
-        }
+        # doc = {
+        #     k: v
+        #     for k, v in doc.items()
+        #     if k not in ("$namespaces", "$schemas", "$base")
+        # }
+        doc = copy.copy(doc)
+        if "$namespaces" in doc:
+            doc.pop("$namespaces")
+        if "$schemas" in doc:
+            doc.pop("$schemas")
+        if "$base" in doc:
+            doc.pop("$base")
+
+        if isinstance(doc, CommentedMap):
+            global doc_line_info
+            doc_line_info = doc
 
         if "$graph" in doc:
             loadingOptions.idx[baseuri] = (
@@ -750,7 +896,6 @@ def _document_load(
             loadingOptions.idx[docuri] = loadingOptions.idx[baseuri]
 
         return loadingOptions.idx[baseuri]
-
     if isinstance(doc, MutableSequence):
         loadingOptions.idx[baseuri] = (
             loader.load(doc, baseuri, loadingOptions),

schema_salad/tests/count-lines6-wf_v1_0.cwl

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+
+requirements:
+  - class: ScatterFeatureRequirement
+  - class: MultipleInputFeatureRequirement
+
+inputs:
+    file1: File[]
+    file2: File[]
+
+outputs:
+    count_output:
+      type: int
+      outputSource: step1/output
+
+steps:
+  step1:
+    run: wc3-tool_v1_0.cwl
+    scatter: file1
+    in:
+      file1:
+        source: [file1, file2]
+        linkMerge: merge_nested
+    out: [output]

schema_salad/tests/count-lines6-wf_v1_1.cwl

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.1
+
+requirements:
+  - class: ScatterFeatureRequirement
+  - class: MultipleInputFeatureRequirement
+
+inputs:
+    file1: File[]
+    file2: File[]
+
+outputs:
+    count_output:
+      type: int
+      outputSource: step1/output
+
+steps:
+  step1:
+    run: wc3-tool_v1_1.cwl
+    scatter: file1
+    in:
+      file1:
+        source: [file1, file2]
+        linkMerge: merge_nested
+    out: [output]