USG Integration

example_files/python_deps/dependencies.json

@@ -1,7 +1,9 @@
 [
-  { "name": "ThermalNetwork", "version": "0.4.1"},
-  { "name": "urbanopt-ditto-reader", "version": "0.6.4"},
+  { "name": "ThermalNetwork", "version": "0.5.0"},
+  { "name": "git+https://github.com/urbanopt/urbanopt-ditto-reader.git@numpy-update", "version": null},
   { "name": "NREL-disco", "version": "0.5.1"},
-  { "name": "urbanopt-des", "version": "0.1.3"},
-  { "name": "Shapely", "version": "1.8.5"}
+  { "name": "urbanopt-des", "version": "0.2.0"},
+  { "name": "Shapely", "version": "1.8.5"},
+  { "name": "urban-system-generator", "version": "0.1.1"},
+  { "name": "numpy", "version": "2.2.6"}
 ]

lib/uo_cli.rb

@@ -37,7 +37,10 @@ class UrbanOptCLI
         'des_params' => 'Make a DES system parameters config file',
         'des_create' => 'Create a Modelica model',
         'des_run' => 'Run a Modelica DES model',
-        'ghe_size' => 'Run a Ground Heat Exchanger model for sizing'
+        'ghe_size' => 'Run a Ground Heat Exchanger model for sizing',
+        'usg_preprocess' => 'Generate Urban Systems Generator input CSV from GeoJSON feature file',
+        'usg_complete' => 'Fill in missing fields in Urban Systems Generator CSV file using USG model',
+
       }.freeze
 
       def initialize
@@ -374,6 +377,23 @@ def opt_delete
         end
       end
 
+      def opt_usg_preprocess
+        @subopts = Optimist.options do
+          banner "\nURBANopt usg_preprocess:\n \n"
+          opt :feature, "\nProvide the Feature JSON file to include info about each feature\n", type: String, required: true, short: :f
+        end
+      end
+
+      def opt_usg_complete
+        @subopts = Optimist.options do
+          banner "\nURBANopt usg_complete:\n \n"
+          opt :feature, "\nProvide the Feature JSON file to include info about each feature\n", type: String, required: true, short: :f
+          opt :input, "\nProvide the USG partial file CSV generated by the usg_preprocess step\n", type: String, required: true, short: :i
+          opt :output, "\nProvide the desired path for the completed USG input CSV file. If this is not provided, the output file will match \
+                        the input file with '_complete' appended to the end\n", type: String, required: false, short: :o
+        end
+      end
+
       def opt_des_params
         @subopts = Optimist.options do
           banner "\nURBANopt des_params:\n \n"
@@ -1015,10 +1035,12 @@ def self.setup_python_variables
         python_install_path: nil,
         python_path: nil,
         pip_path: nil,
+        des_output_path: nil,
+        disco_path: nil,
         ditto_path: nil,
-        gmt_path: nil,
         ghe_path: nil,
-        des_output_path: nil
+        gmt_path: nil,
+        usg_path: nil
       }
 
       # get location
@@ -1036,11 +1058,12 @@ def self.setup_python_variables
         configs = JSON.parse(File.read(File.join(pvars[:python_install_path], 'python_config.json')), symbolize_names: true)
         pvars[:python_path] = configs[:python_path]
         pvars[:pip_path] = configs[:pip_path]
-        pvars[:ditto_path] = configs[:ditto_path]
-        pvars[:gmt_path] = configs[:gmt_path]
+        pvars[:des_output_path] = configs[:des_output_path]
         pvars[:disco_path] = configs[:disco_path]
+        pvars[:ditto_path] = configs[:ditto_path]
         pvars[:ghe_path] = configs[:ghe_path]
-        pvars[:des_output_path] = configs[:des_output_path]
+        pvars[:gmt_path] = configs[:gmt_path]
+        pvars[:usg_path] = configs[:usg_path]
       end
       return pvars
     end
@@ -1188,19 +1211,21 @@ def self.install_python_dependencies
           mac_path_base = File.join(pvars[:python_install_path], "Miniconda-#{pvars[:miniconda_version]}")
           pvars[:python_path] = File.join(mac_path_base, 'bin', 'python')
           pvars[:pip_path] = File.join(mac_path_base, 'bin', 'pip')
-          pvars[:ditto_path] = File.join(mac_path_base, 'bin', 'ditto_reader_cli')
-          pvars[:gmt_path] = File.join(mac_path_base, 'bin', 'uo_des')
+          pvars[:des_output_path] = File.join(mac_path_base, 'bin', 'des-output')
           pvars[:disco_path] = File.join(mac_path_base, 'bin', 'disco')
+          pvars[:ditto_path] = File.join(mac_path_base, 'bin', 'ditto_reader_cli')
           pvars[:ghe_path] = File.join(mac_path_base, 'bin', 'thermalnetwork')
-          pvars[:des_output_path] = File.join(mac_path_base, 'bin', 'des-output')
+          pvars[:gmt_path] = File.join(mac_path_base, 'bin', 'uo_des')
+          pvars[:usg_path] = File.join(mac_path_base, 'bin', 'usg')
           configs = {
             python_path: pvars[:python_path],
             pip_path: pvars[:pip_path],
-            ditto_path: pvars[:ditto_path],
-            gmt_path: pvars[:gmt_path],
+            des_output_path: pvars[:des_output_path],
             disco_path: pvars[:disco_path],
+            ditto_path: pvars[:ditto_path],
             ghe_path: pvars[:ghe_path],
-            des_output_path: pvars[:des_output_path]
+            gmt_path: pvars[:gmt_path],
+            usg_path: pvars[:usg_path]
           }
         else
           # windows
@@ -1224,20 +1249,22 @@ def self.install_python_dependencies
           windows_path_base = File.join(pvars[:python_install_path], "python-#{pvars[:python_version]}")
           pvars[:python_path] = File.join(windows_path_base, 'python.exe')
           pvars[:pip_path] = File.join(windows_path_base, 'Scripts', 'pip.exe')
-          pvars[:ditto_path] = File.join(windows_path_base, 'Scripts', 'ditto_reader_cli.exe')
-          pvars[:gmt_path] = File.join(windows_path_base, 'Scripts', 'uo_des.exe')
+          pvars[:des_output_path] = File.join(windows_path_base, 'Scripts', 'des-output.exe')
           pvars[:disco_path] = File.join(windows_path_base, 'Scripts', 'disco.exe')
+          pvars[:ditto_path] = File.join(windows_path_base, 'Scripts', 'ditto_reader_cli.exe')
           pvars[:ghe_path] = File.join(windows_path_base, 'Scripts', 'thermalnetwork.exe')
-          pvars[:des_output_path] = File.join(windows_path_base, 'Scripts', 'des-output.exe')
+          pvars[:gmt_path] = File.join(windows_path_base, 'Scripts', 'uo_des.exe')
+          pvars[:usg_path] = File.join(windows_path_base, 'Scripts', 'usg.exe')
 
           configs = {
             python_path: pvars[:python_path],
             pip_path: pvars[:pip_path],
-            ditto_path: pvars[:ditto_path],
-            gmt_path: pvars[:gmt_path],
+            des_output_path: pvars[:des_output_path],
             disco_path: pvars[:disco_path],
+            ditto_path: pvars[:ditto_path],
             ghe_path: pvars[:ghe_path],
-            des_output_path: pvars[:des_output_path]
+            gmt_path: pvars[:gmt_path],
+            usg_path: pvars[:usg_path]
           }
         end
 
@@ -2168,6 +2195,102 @@ def self.install_python_dependencies
       rescue StandardError => e
         puts "\nERROR: #{e.message}"
       end
+    end
+
+    # USG Preprocess
+    if @opthash.command == 'usg_preprocess'
+      # Use the USG CLI to preprocess USG inputs. The output file will be automatically be named the same as the Geojson file +  .csv
+
+      # first check python
+      res = check_python
+      if res[:python] == false
+        puts "\nPython error: #{res[:message]}"
+        abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python  \n")
+      end
+
+      usg_cli_root = "#{res[:pvars][:usg_path].to_s} geojson2csv"
+      usg_cli_addition = ''
+
+      if @opthash.subopts[:feature]
+        usg_cli_addition += " -i #{@opthash.subopts[:feature]}"
+      end
+
+      begin
+        puts "\nRunning system command: #{usg_cli_root + usg_cli_addition}\n"
+        system(usg_cli_root + usg_cli_addition)
+      rescue FileNotFoundError
+        abort("\nFeature File #{@opthash.subopts[:feature]} not Found. Please check the file path and try again.")
+      rescue StandardError => e
+        puts "\nERROR: #{e.message}"
+      end
+    end
+
+    # USG Complete
+    if @opthash.command == 'usg_complete'
+      # Use the USG CLI to complete USG simulations. The input file will be the same as the Geojson file +  .csv
+
+      # first check python
+      res = check_python
+      if res[:python] == false
+        puts "\nPython error: #{res[:message]}"
+        abort("\nPython dependencies are needed to run this workflow. Install with the CLI command: uo install_python  \n")
+      end
+
+      # Step 1 Complete
+      usg_cli_root = "#{res[:pvars][:usg_path].to_s} complete"
+      usg_cli_addition = ''
+
+      if @opthash.subopts[:input]
+        usg_cli_addition += " -i #{@opthash.subopts[:input]}"
+      end
+
+      output_file = ''
+      if @opthash.subopts[:output]
+        usg_cli_addition += " -o #{@opthash.subopts[:output]}"
+        output_file = @opthash.subopts[:output]
+      else
+        # extract output file name from input file name + "_complete.csv"
+        input_file = @opthash.subopts[:input]
+        output_file = input_file.sub('.csv', '_complete.csv')
+        usg_cli_addition += " -o #{output_file}"
+      end
+
+      begin
+        puts "\nRunning system command: #{usg_cli_root + usg_cli_addition}\n"
+        system(usg_cli_root + usg_cli_addition)
+      rescue FileNotFoundError
+        abort("\nInput CSV File #{@opthash.subopts[:input]} not found. Please check the file path and try again.")
+      rescue StandardError => e
+        puts "\nERROR: #{e.message}"
+      end
+
+      # Step 2 - Post Process
+      # this is a temporary step needed to convert headers to newer ResStock schemas
+      usg_cli_root2 = "#{res[:pvars][:usg_path].to_s} process"
+      usg_cli_addition = ''
+
+      # using --no-reports to not write reports
+      usg_cli_addition += " -i #{output_file}"
+      usg_cli_addition += " -o #{output_file.sub('.csv', '_converted.csv')} --no-reports"
+
+      if @opthash.subopts[:feature]
+        usg_cli_addition += " -g #{@opthash.subopts[:feature]}"
+      end
+
+      begin
+        puts "\nRunning system command: #{usg_cli_root2 + usg_cli_addition}\n"
+        system(usg_cli_root2 + usg_cli_addition)
+      rescue FileNotFoundError
+        abort("\nCSV File #{output_file} not found. Please check the file path and try again.")
+      rescue StandardError => e
+        puts "\nERROR: #{e.message}"
+      end
+
+      # Rename final output file to original output file name and delete intermediate file
+      FileUtils.mv("#{output_file.sub('.csv', '_converted.csv')}", output_file)
+
+      # puts
+      puts "\nUSG processing complete. Final output file to use in UO project: #{output_file}\n"
 
     end
 

spec/uo_cli_spec.rb

@@ -871,4 +871,51 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw',
       expect((test_directory_elec / 'run' / 'electrical_scenario' / 'process_status.json').exist?).to be true
     end
   end
+
+  context 'Urban System Generator (USG) workflow' do
+    test_directory_usg = spec_dir / 'test_directory_usg'
+    test_feature_usg = test_directory_usg / 'test_example_project_combined.json'
+
+    before(:all) do
+      # Clean up any existing test directory
+      delete_directory_or_file(test_directory_usg)
+
+      # Create test directory structure
+      system("#{call_cli} create -d --project-folder #{test_directory_usg}")
+
+      # Copy the test feature file to use for USG testing
+      FileUtils.cp(spec_dir / 'spec_files' / 'test_example_project_combined.json', test_feature_usg)
+    end
+
+    after(:all) do
+      delete_directory_or_file(test_directory_usg)
+    end
+
+    it 'successfully preprocesses and completes USG workflow' do
+      # Expected output files
+      csv_file = test_directory_usg / 'test_example_project_combined.csv'
+      complete_csv_file = test_directory_usg / 'usg_buildstock_mappings.csv'
+
+      # Step 1: Run USG preprocess to generate CSV from GeoJSON
+      system("#{call_cli} usg_preprocess --feature #{test_feature_usg}")
+
+      # Assert that the CSV file was created
+      expect(csv_file.exist?).to be true
+      expect(csv_file.size > 0).to be true
+
+      # Step 2: Run USG complete to process the CSV file
+      system("#{call_cli} usg_complete --input #{csv_file} --output #{complete_csv_file} --feature #{test_feature_usg}")
+
+      # Assert that the complete CSV file was created
+      expect(complete_csv_file.exist?).to be true
+      expect(complete_csv_file.size > 0).to be true
+
+      # Verify the CSV files contain expected headers/content
+      csv_content = File.read(csv_file)
+      expect(csv_content).to include('Building')
+
+      complete_csv_content = File.read(complete_csv_file)
+      expect(complete_csv_content).to include('Building')
+    end
+  end
 end