From e149bf1af3c97a402d5cbd879894b77992377184 Mon Sep 17 00:00:00 2001 From: sharath-abingdon Date: Mon, 9 Sep 2024 11:53:37 +0100 Subject: [PATCH 1/3] Import socsmusic data --- lib/import/socsmusic/options.rb | 128 +++++++++++ lib/import/socsmusic/socsmusic_lessons.rb | 48 ++++ lib/import/socsmusicimport.rb | 261 ++++++++++++++++++++++ utils/importsocsmusicdata | 87 ++++++++ 4 files changed, 524 insertions(+) create mode 100644 lib/import/socsmusic/options.rb create mode 100644 lib/import/socsmusic/socsmusic_lessons.rb create mode 100644 lib/import/socsmusicimport.rb create mode 100644 utils/importsocsmusicdata diff --git a/lib/import/socsmusic/options.rb b/lib/import/socsmusic/options.rb new file mode 100644 index 00000000..45b89b56 --- /dev/null +++ b/lib/import/socsmusic/options.rb @@ -0,0 +1,128 @@ +require 'optparse' +require 'optparse/date' + +class Options + attr_reader :event_category_name, :verbose, :start_date, :end_date + + def initialize + @event_category_name = "Lesson" + @verbose = false + @start_date = nil + @end_date = nil + + parser = OptionParser.new do |opts| + opts.banner = "Usage: music_import.rb [options]" + + opts.on("-c", "--category [EVENTCATEGORY]", + "Specify the name of the event category", + "to be used for all the fixtures.", + "Defaults to \"Music\".") do |name| + @event_category_name = name + end + + opts.on("-s", "--start_date [DATE]", "Specify the start date (e.g. 2024-09-05)") do |date| + @start_date = Date.parse(date) + end + + opts.on("-e", "--end_date [DATE]", "Specify the end date (e.g. 2024-09-06)") do |date| + @end_date = Date.parse(date) + end + + opts.on("-v", "--verbose", + "Run with verbose output") do + @verbose = true + end + + opts.on("-h", "--help", "Show this message") do + puts opts + exit + end + end + + parse_options(parser) + end + + private + + def parse_options(parser) + begin + parser.parse! + rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e + puts e.message + puts parser + exit 1 + rescue SystemExit + # Allow SystemExit to propagate for clean exit + raise + rescue StandardError => e + puts "An error occurred: #{e.message}" + puts parser + exit 1 + end + end +end + +scheduler@devxronos:~/Work/Coding/scheduler/lib/import/socsmusic$ cat options.rb +require 'optparse' +require 'optparse/date' + +class Options + attr_reader :event_category_name, :verbose, :start_date, :end_date + + def initialize + @event_category_name = "Lesson" + @verbose = false + @start_date = nil + @end_date = nil + + parser = OptionParser.new do |opts| + opts.banner = "Usage: music_import.rb [options]" + + opts.on("-c", "--category [EVENTCATEGORY]", + "Specify the name of the event category", + "to be used for all the fixtures.", + "Defaults to \"Music\".") do |name| + @event_category_name = name + end + + opts.on("-s", "--start_date [DATE]", "Specify the start date (e.g. 2024-09-05)") do |date| + @start_date = Date.parse(date) + end + + opts.on("-e", "--end_date [DATE]", "Specify the end date (e.g. 2024-09-06)") do |date| + @end_date = Date.parse(date) + end + + opts.on("-v", "--verbose", + "Run with verbose output") do + @verbose = true + end + + opts.on("-h", "--help", "Show this message") do + puts opts + exit + end + end + + parse_options(parser) + end + + private + + def parse_options(parser) + begin + parser.parse! + rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e + puts e.message + puts parser + exit 1 + rescue SystemExit + # Allow SystemExit to propagate for clean exit + raise + rescue StandardError => e + puts "An error occurred: #{e.message}" + puts parser + exit 1 + end + end +end \ No newline at end of file diff --git a/lib/import/socsmusic/socsmusic_lessons.rb b/lib/import/socsmusic/socsmusic_lessons.rb new file mode 100644 index 00000000..84c6468a --- /dev/null +++ b/lib/import/socsmusic/socsmusic_lessons.rb @@ -0,0 +1,48 @@ +# app/models/music_fixture.rb +class MusicFixture + attr_reader :lesson_id, :instrument, :title, :starts_at, :ends_at, :location, :staff_id, :pupil_id, :attendance + def initialize(xml_node) + @lesson_id = xml_node.at_xpath('lessonid').text + start_date = xml_node.at_xpath('startdate').text + start_time = xml_node.at_xpath('starttime').text + end_time = xml_node.at_xpath('endtime').text + @instrument = xml_node.at_xpath('instrument').text + @title = xml_node.at_xpath('title').text + @starts_at = Time.parse("#{start_date} #{start_time}") + @ends_at = Time.parse("#{start_date} #{end_time}") + @location = xml_node.at_xpath('location').text + @staff_id = xml_node.at_xpath('staffid').text + @pupil_id = xml_node.at_xpath('pupilid').text + @attendance = xml_node.at_xpath('attendance').text + end + def home_location + @location + end + def away? + # If there's a need to define "away" logic, it can be based on location or other attributes + false + end +end +# app/models/music_fixture_set.rb +class MusicFixtureSet + attr_reader :fixtures + def initialize(xml, options = {}) + @fixtures = xml.xpath('//lesson').map { |node| MusicFixture.new(node) } + @options = options + end + def empty? + @fixtures.empty? + end + def fixtures_on(date) + @fixtures.select { |f| f.starts_at.to_date == date } + end + def last_date + @fixtures.map(&:starts_at).max.to_date + end + def instruments + @fixtures.map(&:instrument).uniq + end + def home_locations + @fixtures.map(&:home_location).uniq + end +end \ No newline at end of file diff --git a/lib/import/socsmusicimport.rb b/lib/import/socsmusicimport.rb new file mode 100644 index 00000000..f2053e82 --- /dev/null +++ b/lib/import/socsmusicimport.rb @@ -0,0 +1,261 @@ +require_relative '../../config/environment' + +require_relative 'common/xmlimport' +require_relative 'socs/element_engine' +require_relative 'socs/location_engine' +require_relative 'socs/property_engine' +require_relative 'socsmusic/options' +require_relative 'socsmusic/socsmusic_lessons' + +MUSIC_IMPORT_DIR = 'import/socsmusic/Current' + +# Initialize engines and options +element_engine = ElementEngine.new +location_engine = LocationEngine.new +property_engine = PropertyEngine.new +options = Options.new +DUMMY_SOURCE_ID_VALUE = 111111 + +# Function to fetch email (for pupil or staff) from API +def get_email(id) + base_url = "https://my.abingdon.org.uk/ma-api/scheduler-api/getEmail/486f74" + url = URI("#{base_url}/#{id}") + + response = Net::HTTP.get_response(url) + if response.is_a?(Net::HTTPSuccess) + json_response = JSON.parse(response.body) + return json_response['email'] # Assuming API returns email in this format + else + puts "Failed to fetch email for id #{id}" + return nil + end +end + +# Find event source for Music +eventsource = Eventsource.find_by(name: "SOCS MUSIC") +unless eventsource + puts "Eventsource Music not found" + exit 1 +end + +# Find event category +eventcategory = Eventcategory.find_by(name: options.event_category_name) +unless eventcategory + puts "Eventcategory #{options.event_category_name} not found." + exit 2 +end + +# Load XML +full_dir_path = Rails.root.join(MUSIC_IMPORT_DIR) +xml_file_path = File.expand_path("data.xml", full_dir_path) + +# Ensure the file exists +unless File.exist?(xml_file_path) + puts "XML file not found at #{xml_file_path}" + exit 3 +end + +xml = Nokogiri::XML(File.open(xml_file_path)) + +fixture_set = MusicFixtureSet.new(xml, options) + +if fixture_set.empty? + puts "No music fixtures found." +else + puts "Got #{fixture_set.fixtures.count} fixtures" if options.verbose + + start_date = options.start_date + end_date = options.end_date || fixture_set.last_date + + if end_date < start_date + puts "End date (#{end_date}) is less than start date (#{start_date}) - aborting." + exit 4 + end + + events_created = 0 + events_deleted = 0 + + (start_date..end_date).each do |date| + puts "Processing #{date.to_s(:dmy)}" if options.verbose + + # Fetch existing events + existing_events = Event.events_on(date, nil, nil, eventsource, nil, nil, true) + .includes(:commitments).to_a + + # Determine desired events + wanted = fixture_set.fixtures_on(date) + + wanted.each do |fixture| + property_element = property_engine.find(fixture.home_location) + + # Calculate owner based on options and property ownership + calculated_owner = 2 + + # Find existing event by source ID (lesson_id) + existing_event = existing_events.detect { |e| e.source_id == fixture.lesson_id } + + if existing_event + # Update existing event + do_save = false + + if existing_event.body != fixture.title + puts "Changing \"#{existing_event.body}\" to \"#{fixture.title}\"" if options.verbose + existing_event.body = fixture.title + do_save = true + end + + if existing_event.eventcategory != eventcategory + existing_event.eventcategory = eventcategory + do_save = true + end + + if existing_event.starts_at != fixture.starts_at + existing_event.starts_at = fixture.starts_at + do_save = true + end + + if existing_event.ends_at != fixture.ends_at + existing_event.ends_at = fixture.ends_at + do_save = true + end + + if existing_event.all_day != fixture.away? + existing_event.all_day = fixture.away? + do_save = true + end + + if existing_event.owner != calculated_owner + existing_event.owner = calculated_owner + do_save = true + end + + existing_event.save! if do_save + existing_events.delete(existing_event) + else + # Create new event + new_event = Event.create!( + body: fixture.title, + eventcategory: eventcategory, + eventsource: eventsource, + starts_at: fixture.starts_at, + ends_at: fixture.ends_at, + all_day: fixture.away?, + source_id: fixture.lesson_id + ) + puts "Created Event ID: #{new_event.id}" + events_created += 1 + existing_event = new_event + end + + # Fetch pupil email and handle pupil element + pupil_email = get_email(fixture.pupil_id) + next unless pupil_email + + pupil = Pupil.find_by(email: pupil_email) + unless pupil + puts "Pupil with email #{pupil_email} not found, skipping." + next + end + + pupil_element = pupil.element # Assuming each pupil has an associated element record + unless pupil_element + pupil_element = Element.create!(entity: pupil) + puts "Created Element for Pupil #{pupil.name}" + end + + # Fetch staff email and handle staff element + staff_email = get_email(fixture.staff_id) + next unless staff_email + + staff = Staff.find_by(email: staff_email) + unless staff + puts "Staff with email #{staff_email} not found, skipping." + next + end + + staff_element = staff.element # Assuming each staff has an associated element record + unless staff_element + staff_element = Element.create!(entity: staff) + puts "Created Element for Staff #{staff.name}" + end + + # Handle subject element + instrument_name = fixture.instrument.strip.downcase + + # Special case: If the instrument name is Violin/Viola, we should map it to Violin + if instrument_name == 'violin/viola' + instrument_name = 'violin' + end + + subject = Subject.find_by('LOWER(name) = ?', instrument_name) + unless subject + puts "Subject with name '#{instrument_name}' not found, skipping subject association." + next + end + + subject_element = subject.element # Assuming each subject has an associated element record + unless subject_element + subject_element = Element.create!(entity: subject) + puts "Created Element for Subject #{subject.name}" + end + + # *** Begin Property Integration *** + # Assuming "Music lesson" is the property you want to assign to all events + property = Property.find_by(name: 'Music lesson') + unless property + puts "Property 'Music lesson' not found, skipping." + next + end + + property_element = property.element # Assuming each property has an associated element record + unless property_element + property_element = Element.create!(entity: property) + puts "Created Element for Property #{property.name}" + end + # *** End Property Integration *** + + # Collect element IDs for pupil, staff, subject, and property + element_ids = [pupil_element.id, staff_element.id, subject_element.id, property_element.id] + + # Identify commitments to destroy + commitments_to_destroy = [] + existing_event.commitments.each do |commitment| + if (commitment.source_id == DUMMY_SOURCE_ID_VALUE) && + !element_ids.include?(commitment.element_id) + commitments_to_destroy << commitment + end + end + + # Add new commitments + element_ids.each do |element_id| + unless existing_event.commitments.detect { |commitment| commitment.element_id == element_id } + new_commitment = existing_event.commitments.new({ + element_id: element_id, + source_id: DUMMY_SOURCE_ID_VALUE + }) + # If needed, set approval status or other attributes here + new_commitment.save! + puts "Created new Commitment for Element ID #{element_id} and Event ID #{existing_event.id}" + end + end + + # Destroy invalid commitments + commitments_to_destroy.each do |commitment| + puts "Destroying Commitment ID #{commitment.id} for Element ID #{commitment.element_id}" + commitment.destroy + end + end + + # Delete surplus events + if existing_events.any? + puts "Deleting #{existing_events.count} events on #{date.to_s(:dmy)}" if options.verbose + events_deleted += existing_events.count + existing_events.each(&:destroy) + end + end + + puts "#{events_created} events created and #{events_deleted} events deleted." if options.verbose + # location_engine.list_missing if options.list_missing +end + +exit 0 \ No newline at end of file diff --git a/utils/importsocsmusicdata b/utils/importsocsmusicdata new file mode 100644 index 00000000..bae13241 --- /dev/null +++ b/utils/importsocsmusicdata @@ -0,0 +1,87 @@ +#!/bin/bash +. /etc/profile +. ~/etc/socsauth +. ~/etc/whichsystem +. ~/.rvm/environments/scheduler + +if [ "a$SCHEDULER_DIR" == "a" ]; then + echo "SCHEDULER_DIR environment variable needs to be set." + exit 1 +fi +if [ ! -d $SCHEDULER_DIR/import/socsmusic ]; then + echo "$SCHEDULER_DIR/import/socs does not seem to be a directory." + exit 2 +fi + +# Use today's date as the start date +STARTDATE=$(date "+%d %b %y") +# Set the end date to 14 days after the start date (2 weeks) +ENDDATE=$(date --date='+14 days' "+%d %b %y") + +cd $SCHEDULER_DIR/import/socsmusic +rm -f Incoming/* + +if [ -f PreFetched/data.xml ]; then + echo "Using pre-fetched data.xml" + cp PreFetched/data.xml Incoming +else + if [ "a$SOCS_MUSIC_API_URL" == "a" ]; then + echo "SOCS_MUSIC_API_URL environment variable needs to be set." + exit 3 + fi + if [ "a$SOCS_MUSIC_APIKEY" == "a" ]; then + echo "SOCS_MUSIC_APIKEY environment variable needs to be set." + exit 4 + fi + + # Initialize or empty the data.xml file + echo "" > Incoming/data.xml + + # Loop through each date from start date to end date + CURRENTDATE="$STARTDATE" + + while [ "$(date -d "$CURRENTDATE" "+%s")" -le "$(date -d "$ENDDATE" "+%s")" ]; do + + # Manually encode the startdate (replace spaces with %20) + ENCODED_STARTDATE=$(echo "$CURRENTDATE" | sed 's/ /%20/g') + + FULL_URL="$SOCS_MUSIC_API_URL?ID=2&key=$SOCS_MUSIC_APIKEY&data=musiclessons&startdate=$ENCODED_STARTDATE" + #echo "Full URL: $FULL_URL" + + # Fetch data for the current date + curl -G --silent "$FULL_URL" --output Incoming/tmp_data.xml 2> curl_errors.log + + if [ $? -ne 0 ]; then + echo "Failed to fetch SOCS API data for $CURRENTDATE." + cat curl_errors.log + exit 5 + fi + + # Print the API response for debugging + #echo "Response from API for $CURRENTDATE:" + #cat Incoming/tmp_data.xml + + # Append the fetched data to the main data.xml file + cat Incoming/tmp_data.xml >> Incoming/data.xml + + # Move to the next day + CURRENTDATE=$(date --date="$CURRENTDATE +1 day" "+%d %b %y") + done + + # Close the root tag in the data.xml file + echo "" >> Incoming/data.xml + rm -f Incoming/tmp_data.xml +fi + +# Move and archive the data files +rm -f Current/* +mv Incoming/* Current +TARGET_DIR="Archive/$(date +%Y/%m/%d)" +mkdir -p "$TARGET_DIR" +cp Current/* "$TARGET_DIR" +bzip2 -f "$TARGET_DIR"/*.xml + +# Import the new data into Scheduler +$SCHEDULER_DIR/lib/import/socsmusicimport.rb --start_date="$STARTDATE" --end_date="$ENDDATE" | tee import.log +cp import.log "$TARGET_DIR" +bzip2 -f "$TARGET_DIR/import.log" \ No newline at end of file From 708274d0f43d1f2d20b4f4402621c6c645171ad7 Mon Sep 17 00:00:00 2001 From: sharath-abingdon Date: Thu, 12 Sep 2024 16:04:33 +0100 Subject: [PATCH 2/3] Updated code to fetch from scheduler DB --- lib/import/socsmusicimport.rb | 77 ++++++++++++++++------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/lib/import/socsmusicimport.rb b/lib/import/socsmusicimport.rb index f2053e82..5df48c7e 100644 --- a/lib/import/socsmusicimport.rb +++ b/lib/import/socsmusicimport.rb @@ -1,3 +1,4 @@ +#!/usr/bin/env ruby require_relative '../../config/environment' require_relative 'common/xmlimport' @@ -16,21 +17,6 @@ options = Options.new DUMMY_SOURCE_ID_VALUE = 111111 -# Function to fetch email (for pupil or staff) from API -def get_email(id) - base_url = "https://my.abingdon.org.uk/ma-api/scheduler-api/getEmail/486f74" - url = URI("#{base_url}/#{id}") - - response = Net::HTTP.get_response(url) - if response.is_a?(Net::HTTPSuccess) - json_response = JSON.parse(response.body) - return json_response['email'] # Assuming API returns email in this format - else - puts "Failed to fetch email for id #{id}" - return nil - end -end - # Find event source for Music eventsource = Eventsource.find_by(name: "SOCS MUSIC") unless eventsource @@ -94,6 +80,9 @@ def get_email(id) # Find existing event by source ID (lesson_id) existing_event = existing_events.detect { |e| e.source_id == fixture.lesson_id } + # Initialize element_ids array + element_ids = [] + if existing_event # Update existing event do_save = false @@ -147,13 +136,10 @@ def get_email(id) existing_event = new_event end - # Fetch pupil email and handle pupil element - pupil_email = get_email(fixture.pupil_id) - next unless pupil_email - - pupil = Pupil.find_by(email: pupil_email) + # Fetch pupil and handle pupil element + pupil = Pupil.find_by(school_id: fixture.pupil_id) unless pupil - puts "Pupil with email #{pupil_email} not found, skipping." + puts "Pupil with school ID #{fixture.pupil_id} not found, skipping." next end @@ -163,13 +149,13 @@ def get_email(id) puts "Created Element for Pupil #{pupil.name}" end - # Fetch staff email and handle staff element - staff_email = get_email(fixture.staff_id) - next unless staff_email + # Add pupil element to element_ids array + element_ids << pupil_element.id if pupil_element - staff = Staff.find_by(email: staff_email) + # Fetch staff and handle staff element + staff = Staff.find_by(user_code: fixture.staff_id) unless staff - puts "Staff with email #{staff_email} not found, skipping." + puts "Staff with user code #{fixture.staff_id} not found, skipping." next end @@ -179,28 +165,35 @@ def get_email(id) puts "Created Element for Staff #{staff.name}" end + # Add staff element to element_ids array + element_ids << staff_element.id if staff_element + # Handle subject element instrument_name = fixture.instrument.strip.downcase # Special case: If the instrument name is Violin/Viola, we should map it to Violin - if instrument_name == 'violin/viola' - instrument_name = 'violin' - end + if instrument_name == 'violin/viola' + instrument_name = 'violin' + end + + if instrument_name == 'drums' + instrument_name = 'drum' + end subject = Subject.find_by('LOWER(name) = ?', instrument_name) unless subject puts "Subject with name '#{instrument_name}' not found, skipping subject association." - next - end - - subject_element = subject.element # Assuming each subject has an associated element record - unless subject_element - subject_element = Element.create!(entity: subject) - puts "Created Element for Subject #{subject.name}" + else + subject_element = subject.element # Assuming each subject has an associated element record + unless subject_element + subject_element = Element.create!(entity: subject) + puts "Created Element for Subject #{subject.name}" + end + # Add subject element to element_ids array if subject_element is valid + element_ids << subject_element.id if subject_element end # *** Begin Property Integration *** - # Assuming "Music lesson" is the property you want to assign to all events property = Property.find_by(name: 'Music lesson') unless property puts "Property 'Music lesson' not found, skipping." @@ -212,10 +205,11 @@ def get_email(id) property_element = Element.create!(entity: property) puts "Created Element for Property #{property.name}" end - # *** End Property Integration *** - # Collect element IDs for pupil, staff, subject, and property - element_ids = [pupil_element.id, staff_element.id, subject_element.id, property_element.id] + # Add property element to element_ids array + element_ids << property_element.id if property_element + + # *** End Property Integration *** # Identify commitments to destroy commitments_to_destroy = [] @@ -255,7 +249,6 @@ def get_email(id) end puts "#{events_created} events created and #{events_deleted} events deleted." if options.verbose - # location_engine.list_missing if options.list_missing end -exit 0 \ No newline at end of file +exit 0 From 9ca925c6cbd7a7703a94d5d3e25c2c7c9f05fa7d Mon Sep 17 00:00:00 2001 From: sharath-abingdon Date: Fri, 13 Sep 2024 09:34:28 +0100 Subject: [PATCH 3/3] Migrations --- db/20240911151440_add_school_staff_ids.rb | 6 ++++++ lib/import/misimport/mispupil.rb | 6 ++++-- lib/import/misimport/misstaff.rb | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 db/20240911151440_add_school_staff_ids.rb diff --git a/db/20240911151440_add_school_staff_ids.rb b/db/20240911151440_add_school_staff_ids.rb new file mode 100644 index 00000000..bf3ac6a2 --- /dev/null +++ b/db/20240911151440_add_school_staff_ids.rb @@ -0,0 +1,6 @@ +class AddSchoolStaffIds < ActiveRecord::Migration[5.2] + def change + add_column :pupils, :school_id, :string, default: "", null: false + add_column :staffs, :user_code, :string, default: "", null: false + end + end \ No newline at end of file diff --git a/lib/import/misimport/mispupil.rb b/lib/import/misimport/mispupil.rb index 88331896..6892ca27 100644 --- a/lib/import/misimport/mispupil.rb +++ b/lib/import/misimport/mispupil.rb @@ -19,7 +19,8 @@ class MIS_Pupil < MIS_Record :email, :house_name, :current, - :datasource_id + :datasource_id, + :school_id ] FIELDS_TO_UPDATE = [ @@ -29,7 +30,8 @@ class MIS_Pupil < MIS_Record :known_as, :email, :house_name, - :current + :current, + :school_id ] def force_save diff --git a/lib/import/misimport/misstaff.rb b/lib/import/misimport/misstaff.rb index f3dcac28..1205b63b 100644 --- a/lib/import/misimport/misstaff.rb +++ b/lib/import/misimport/misstaff.rb @@ -14,7 +14,8 @@ class MIS_Staff < MIS_Record :email, :active, :current, - :datasource_id] + :datasource_id, + :user_code] FIELDS_TO_UPDATE = [:name, :initials, :surname, @@ -22,7 +23,8 @@ class MIS_Staff < MIS_Record :forename, :email, :active, - :current] + :current, + :user_code] # # The MIS-specific code should override everything below here.