Governance model integration and curator updates

Makefile

@@ -2,32 +2,21 @@ CSV := mc2.model.csv
 QC := ./qc_model/mc2_qc.model.csv
 DATA := DataDSP Study FileView PublicationView GrantView ToolView EducationalResource DatasetView DataCatalog
 
-all: collate fix generate-json convert
+all: collate generate-json
 
-qc: collate fix qc_fix qc_convert
+qc: collate qc_convert
 
 collate:
 	@echo "Collating module components..."
 	python update_valid_values.py
 	head -1 modules/consortium/annotationProperty.csv > ${CSV}
 	tail -n +2 -q modules/*/annotationProperty.csv >> ${CSV}
 
-fix:
-	@echo "Modifying columnType to remove string_list"
-	awk -F ',' '{ gsub(/string_list/, ""); gsub(/^integer/, "number", $$10); gsub(/^integer/, "number", $$11); gsub(/^int/, "num", $$10); gsub(/^int/, "num", $$11); gsub(/^bool$$/, "boolean", $$10); gsub(/^bool$$/, "boolean", $$11); print }' OFS=',' ${CSV} > tmp_${CSV}
-	rm ${CSV}
-	mv tmp_${CSV} ${CSV}
-
-qc_fix:
-	awk -F ',' '{ gsub(/matchAtLeastOne.*/, "list like"); print }' OFS=',' ${CSV} > tmp.csv
-	mv tmp.csv ${QC}
-
 convert:
 	schematic schema convert ${CSV}
 
 qc_convert:
 	schematic schema convert ${QC}
 
 generate-json:
-	$(foreach d,$(DATA), schematic schema generate-jsonschema -dms ${CSV} -od . -dt $(d);)
-	rm *.schema.json
+	python create_json_from_model.py ${DATA}

all_valid_values.csv

@@ -8888,31 +8888,6 @@ processLevel,Level 4,
 processLevel,Auxiliary,
 processLevel,Not Applicable,
 processLevel,Metadata,
-duo,IRB,
-duo,HMB,
-duo,PUB,
-duo,US,
-duo,NPOA,
-duo,COL,
-duo,NCU,
-duo,NPUNCU,
-duo,RS,
-duo,TS,
-duo,NRES,
-duo,NPU,
-duo,DUM,
-duo,POA,
-duo,MOR,
-duo,GSO,
-duo,RTN,
-duo,CC,
-duo,NMDS,
-duo,IS,
-duo,GS,
-duo,DS,
-duo,GRU,
-duo,PS,
-duo,Pending Annotation,
 dataTier,Anonymous,
 dataTier,Open,
 dataTier,Controlled,
@@ -9166,12 +9141,17 @@ sourceGeography,YT,
 sourceGeography,ZA,
 sourceGeography,ZM,
 sourceGeography,ZW,
-studyLicense,CC BY 3.0,
-studyLicense,CC BY 4.0,
-studyLicense,CC BY-NC 4.0,
-studyLicense,CC BY-NC 3.0,
-studyLicense,CC BY-SA 4.0,
-studyLicense,CC BY-NC-SA 4.0,
+studyLicense,CC0,
+studyLicense,CC_BY,
+studyLicense,CC_BY_NC,
+studyLicense,CC_BY_ND,
+studyLicense,CC_BY_SA,
+studyLicense,CC_BY_NC_ND,
+studyLicense,CC_BY_NC_SA,
+studyLicense,Apache_2,
+studyLicense,MIT,
+studyLicense,GPL_3,
+studyLicense,BSD_3_Clause,
 ed_topic,Computational Model Development,
 ed_topic,Computational Resource,
 ed_topic,Diversity/Equity/Inclusion,
@@ -10855,18 +10835,50 @@ rtPrimer,Oligo-dT,
 rtPrimer,Poly-dT,
 rtPrimer,Feature barcoding,
 rtPrimer,Random,
+duo,Pending Annotation,
+duo,GRU,
+duo,PS,
+duo,GSO,
+duo,RTN,
+duo,CC,
+duo,NMDS,
+duo,NRES,
+duo,NPU,
+duo,DUM,
+duo,POA,
+duo,NCU,
+duo,NPUNCU,
+duo,NPOA,
+duo,IRB,
+duo,HMB,
+duo,PUB,
+duo,USE,
+duo,COL,
+duo,RST,
+duo,TSM,
+duo,MOR,
+duo,IST,
+duo,GSR,
+duo,DSR,
+duo,DUOPlus1,
+duo,DUOPlus2,
+duo,DUOPlus3,
+duo,DUOPlus4,
+duo,DUOPlus5,
+duo,DUOPlus6,
+duo,DUOPlus7,
 deIdMethod,Manual,
 deIdMethod,Semiautomatic,
 deIdMethod,Automatic,
 deIdMethod,Not Applicable,
-indexDate,Diagnosis Date ,
-indexDate,Enrollment Date,
-indexDate,Collection Date,
-indexDate,Birth Date,
 dataPermission,Agreement,
 dataPermission,Attestation,
 dataPermission,Award,
 dataPermission,Other,
+indexDate,Diagnosis Date ,
+indexDate,Enrollment Date,
+indexDate,Collection Date,
+indexDate,Birth Date,
 tool_accessibility,Open Access,
 tool_accessibility,Open Access (With Restrictions),
 tool_accessibility,Restricted Access,

create_json_from_model.py

@@ -0,0 +1,17 @@
+from synapseclient import Synapse
+from synapseclient.extensions.curator import generate_jsonschema
+import sys
+
+DATA_MODEL_SOURCE = "mc2.model.csv"
+DATA_TYPE = sys.argv[1:] if len(sys.argv) > 1 else None
+OUTPUT_DIRECTORY = "./json_schemas"
+
+syn = Synapse()
+syn.login()
+
+schemas, file_paths = generate_jsonschema(
+    data_model_source=DATA_MODEL_SOURCE,
+    output=OUTPUT_DIRECTORY,
+    data_types=DATA_TYPE,
+    synapse_client=syn,
+)