diff --git a/app/models/account/provider_import_adapter.rb b/app/models/account/provider_import_adapter.rb index 61a095e324..e40e736786 100644 --- a/app/models/account/provider_import_adapter.rb +++ b/app/models/account/provider_import_adapter.rb @@ -771,6 +771,7 @@ def find_pending_transaction(date:, amount:, currency:, source:, date_window: 8) OR (transactions.extra -> 'enable_banking' ->> 'pending')::boolean = true OR (transactions.extra -> 'akahu' ->> 'pending')::boolean = true OR (transactions.extra -> 'up' ->> 'pending')::boolean = true + OR (transactions.extra -> 'mercury' ->> 'pending')::boolean = true SQL .order(date: :desc) # Prefer most recent pending transaction @@ -820,6 +821,7 @@ def find_pending_transaction_fuzzy(date:, amount:, currency:, source:, merchant_ OR (transactions.extra -> 'enable_banking' ->> 'pending')::boolean = true OR (transactions.extra -> 'akahu' ->> 'pending')::boolean = true OR (transactions.extra -> 'up' ->> 'pending')::boolean = true + OR (transactions.extra -> 'mercury' ->> 'pending')::boolean = true SQL # If merchant_id is provided, prioritize matching by merchant @@ -892,6 +894,7 @@ def find_pending_transaction_low_confidence(date:, amount:, currency:, source:, OR (transactions.extra -> 'enable_banking' ->> 'pending')::boolean = true OR (transactions.extra -> 'akahu' ->> 'pending')::boolean = true OR (transactions.extra -> 'up' ->> 'pending')::boolean = true + OR (transactions.extra -> 'mercury' ->> 'pending')::boolean = true SQL # For low confidence, require BOTH merchant AND name match (stronger signal needed) diff --git a/app/models/mercury_entry/processor.rb b/app/models/mercury_entry/processor.rb index a18508111a..ee0bed387e 100644 --- a/app/models/mercury_entry/processor.rb +++ b/app/models/mercury_entry/processor.rb @@ -36,7 +36,8 @@ def process name: name, source: "mercury", merchant: merchant, - notes: notes + notes: notes, + extra: extra ) rescue ArgumentError => e # Re-raise validation errors (missing required fields, invalid data) @@ -114,6 +115,17 @@ def merchant end end + def extra + meta = { "pending" => pending? } + meta["kind"] = data[:kind] if data[:kind].present? + meta["counterparty_id"] = data[:counterpartyId] if data[:counterpartyId].present? + { "mercury" => meta } + end + + def pending? + data[:status] == "pending" + end + def amount parsed_amount = case data[:amount] when String diff --git a/app/models/mercury_item/importer.rb b/app/models/mercury_item/importer.rb index 277ea5e2a5..f507f1ad6c 100644 --- a/app/models/mercury_item/importer.rb +++ b/app/models/mercury_item/importer.rb @@ -209,25 +209,54 @@ def fetch_and_store_transactions(mercury_account) begin existing_transactions = mercury_account.raw_transactions_payload.to_a - # Build set of existing transaction IDs for efficient lookup - existing_ids = existing_transactions.map do |tx| - tx.with_indifferent_access[:id] - end.to_set - - # Filter to ONLY truly new transactions (skip duplicates) - # Transactions are immutable on the bank side, so we don't need to update them - new_transactions = transactions_data[:transactions].select do |tx| - next false unless tx.is_a?(Hash) - - tx_id = tx.with_indifferent_access[:id] - tx_id.present? && !existing_ids.include?(tx_id) + # Build a map of existing transaction IDs for efficient lookup + existing_by_id = existing_transactions.index_by { |tx| tx.with_indifferent_access[:id] } + + new_transactions = [] + updated_transactions = [] + + transactions_data[:transactions].each do |tx| + next unless tx.is_a?(Hash) + + tx_data = tx.with_indifferent_access + tx_id = tx_data[:id] + next unless tx_id.present? + + if existing_by_id.key?(tx_id) + existing = existing_by_id[tx_id].with_indifferent_access + # Mercury reuses the same transaction ID when a pending entry posts. + # Replace the stored copy so the processor clears the pending flag. + if existing[:status] == "pending" && tx_data[:status] != "pending" + existing_by_id[tx_id] = tx + updated_transactions << tx_id + end + else + new_transactions << tx + end end - if new_transactions.any? - Rails.logger.info "MercuryItem::Importer - Storing #{new_transactions.count} new transactions (#{existing_transactions.count} existing, #{transactions_data[:transactions].count - new_transactions.count} duplicates skipped) for account #{mercury_account.account_id}" - mercury_account.upsert_mercury_transactions_snapshot!(existing_transactions + new_transactions) + if new_transactions.any? || updated_transactions.any? + merged = existing_by_id.values + new_transactions + DebugLogEntry.capture( + category: "provider_sync", + level: "info", + message: "Storing #{new_transactions.count} new, #{updated_transactions.count} status-updated transactions for account #{mercury_account.account_id}", + source: self.class.name, + provider_key: "mercury", + family: mercury_item.family, + metadata: { account_id: mercury_account.account_id, new_count: new_transactions.count, updated_count: updated_transactions.count } + ) + mercury_account.upsert_mercury_transactions_snapshot!(merged) else - Rails.logger.info "MercuryItem::Importer - No new transactions to store (all #{transactions_data[:transactions].count} were duplicates) for account #{mercury_account.account_id}" + DebugLogEntry.capture( + category: "provider_sync", + level: "info", + message: "No new or updated transactions for account #{mercury_account.account_id}", + source: self.class.name, + provider_key: "mercury", + family: mercury_item.family, + metadata: { account_id: mercury_account.account_id } + ) end rescue => e Rails.logger.error "MercuryItem::Importer - Failed to store transactions for account #{mercury_account.account_id}: #{e.message}" diff --git a/app/models/transaction.rb b/app/models/transaction.rb index 040a1ea695..e359e5db82 100644 --- a/app/models/transaction.rb +++ b/app/models/transaction.rb @@ -94,7 +94,7 @@ def exchange_rate_must_be_valid INTERNAL_MOVEMENT_LABELS = [ "Transfer", "Sweep In", "Sweep Out", "Exchange" ].freeze # Providers that support pending transaction flags - PENDING_PROVIDERS = %w[simplefin plaid lunchflow enable_banking akahu up].freeze + PENDING_PROVIDERS = %w[simplefin plaid lunchflow enable_banking akahu up mercury].freeze # Pre-computed SQL fragment for subqueries that check if a transaction (aliased as "t") is pending. # Stored as a constant so static analysis can verify it contains no user input. diff --git a/test/models/mercury_account/processor_test.rb b/test/models/mercury_account/processor_test.rb new file mode 100644 index 0000000000..e01a53e00f --- /dev/null +++ b/test/models/mercury_account/processor_test.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require "test_helper" + +class MercuryAccount::ProcessorTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @item = MercuryItem.create!(family: @family, name: "Mercury", token: "tok") + end + + # --------------------------------------------------------------------------- + # balance update + # --------------------------------------------------------------------------- + + test "updates account balance from mercury_account current_balance" do + account = create_account("Checking") + mercury_account = create_mercury_account("acc_001", balance: 12_345.67, account: account) + + MercuryAccount::Processor.new(mercury_account).process + + assert_in_delta 12_345.67, account.reload.balance, 0.01 + end + + test "negates balance for CreditCard accounts" do + account = @family.accounts.create!( + name: "Mercury Credit", balance: 0, currency: "USD", + accountable: CreditCard.new + ) + mercury_account = create_mercury_account("acc_credit", balance: 500.0, account: account) + + MercuryAccount::Processor.new(mercury_account).process + + assert_in_delta(-500.0, account.reload.balance, 0.01) + end + + test "sets cash_balance equal to balance for depository" do + account = create_account("Savings") + mercury_account = create_mercury_account("acc_002", balance: 3_000.0, account: account) + + MercuryAccount::Processor.new(mercury_account).process + + assert_in_delta 3_000.0, account.reload.cash_balance, 0.01 + end + + # --------------------------------------------------------------------------- + # no linked account + # --------------------------------------------------------------------------- + + test "returns nil without error when no linked account" do + mercury_account = @item.mercury_accounts.create!( + name: "Unlinked", account_id: "acc_unlinked", currency: "USD", current_balance: 100 + ) + + result = nil + assert_nothing_raised do + result = MercuryAccount::Processor.new(mercury_account).process + end + assert_nil result + end + + # --------------------------------------------------------------------------- + # transaction processing delegation + # --------------------------------------------------------------------------- + + test "processes transactions stored in raw_transactions_payload" do + account = create_account("Checking") + mercury_account = create_mercury_account("acc_003", balance: 0, account: account, + raw_transactions: [ + { "id" => "tx_a", "amount" => 100.0, "status" => "sent", + "bankDescription" => "Deposit", "createdAt" => "2024-06-01T00:00:00Z", + "postedAt" => "2024-06-01T00:00:00Z" } + ] + ) + + assert_difference "account.entries.count", 1 do + MercuryAccount::Processor.new(mercury_account).process + end + end + + private + + def create_account(name) + @family.accounts.create!( + name: name, balance: 0, currency: "USD", + accountable: Depository.new(subtype: "checking") + ) + end + + def create_mercury_account(account_id, balance:, account:, raw_transactions: []) + ma = @item.mercury_accounts.create!( + name: account_id, account_id: account_id, currency: "USD", + current_balance: balance, + raw_transactions_payload: raw_transactions + ) + AccountProvider.create!(provider: ma, account: account) + ma + end +end diff --git a/test/models/mercury_entry/processor_test.rb b/test/models/mercury_entry/processor_test.rb new file mode 100644 index 0000000000..11e315238b --- /dev/null +++ b/test/models/mercury_entry/processor_test.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +require "test_helper" + +class MercuryEntry::ProcessorTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @account = @family.accounts.create!( + name: "Mercury Checking", balance: 0, currency: "USD", + accountable: Depository.new(subtype: "checking") + ) + @item = MercuryItem.create!( + family: @family, name: "Mercury", token: "test_token" + ) + @mercury_account = @item.mercury_accounts.create!( + name: "Mercury Checking", account_id: "acc_001", currency: "USD", current_balance: 0 + ) + AccountProvider.create!(provider: @mercury_account, account: @account) + end + + # --------------------------------------------------------------------------- + # happy-path posted transaction + # --------------------------------------------------------------------------- + + test "imports a posted transaction with correct sign conversion" do + assert_difference "@account.entries.count", 1 do + process(tx(amount: 150.00, status: "sent")) + end + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert entry + # Mercury positive = inflow; Sure convention negates it → negative + assert entry.amount.negative? + assert_in_delta(-150.0, entry.amount.to_f, 0.01) + end + + test "expense (negative Mercury amount) becomes positive outflow in Sure" do + assert_difference "@account.entries.count", 1 do + process(tx(amount: -75.50, status: "sent")) + end + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert entry + assert entry.amount.positive? + assert_in_delta 75.5, entry.amount.to_f, 0.01 + end + + # --------------------------------------------------------------------------- + # name resolution + # --------------------------------------------------------------------------- + + test "prefers counterpartyNickname over counterpartyName over bankDescription" do + process(tx(counterparty_nickname: "Nick", counterparty_name: "Full Name", bank_description: "Bank Desc")) + assert_equal "Nick", @account.entries.last.name + end + + test "falls back to counterpartyName when nickname absent" do + process(tx(counterparty_name: "Acme Corp")) + assert_equal "Acme Corp", @account.entries.last.name + end + + test "falls back to bankDescription when no counterparty" do + process(tx(bank_description: "ACH Credit")) + assert_equal "ACH Credit", @account.entries.last.name + end + + # --------------------------------------------------------------------------- + # date resolution + # --------------------------------------------------------------------------- + + test "uses postedAt when present" do + process(tx(posted_at: "2024-03-15T00:00:00Z", created_at: "2024-03-10T00:00:00Z")) + assert_equal Date.new(2024, 3, 15), @account.entries.last.date + end + + test "falls back to createdAt when postedAt absent" do + process(tx(posted_at: nil, created_at: "2024-03-10T00:00:00Z")) + assert_equal Date.new(2024, 3, 10), @account.entries.last.date + end + + # --------------------------------------------------------------------------- + # notes + # --------------------------------------------------------------------------- + + test "concatenates note and details with separator" do + process(tx(note: "Office supplies", details: "Q1 restock")) + assert_equal "Office supplies - Q1 restock", @account.entries.last.notes + end + + test "note alone stored without separator" do + process(tx(note: "Reimbursement")) + assert_equal "Reimbursement", @account.entries.last.notes + end + + # --------------------------------------------------------------------------- + # extra metadata: pending, kind, counterpartyId + # --------------------------------------------------------------------------- + + test "marks pending transactions in extra" do + process(tx(status: "pending")) + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert entry + assert entry.entryable.extra.dig("mercury", "pending"), "pending flag must be true" + end + + test "posted transactions have pending=false in extra" do + process(tx(status: "sent")) + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert_equal false, entry.entryable.extra.dig("mercury", "pending") + end + + test "stores transaction kind in extra" do + process(tx(kind: "externalTransfer")) + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert_equal "externalTransfer", entry.entryable.extra.dig("mercury", "kind") + end + + test "stores counterpartyId in extra" do + process(tx(counterparty_id: "cpty_abc123")) + + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert_equal "cpty_abc123", entry.entryable.extra.dig("mercury", "counterparty_id") + end + + test "does not store nil kind in extra" do + process(tx) + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert_nil entry.entryable.extra.dig("mercury", "kind") + end + + # --------------------------------------------------------------------------- + # skipped statuses + # --------------------------------------------------------------------------- + + test "skips failed transactions" do + assert_no_difference "@account.entries.count" do + result = process(tx(status: "failed")) + assert_nil result + end + end + + # --------------------------------------------------------------------------- + # idempotency + # --------------------------------------------------------------------------- + + test "does not create duplicate entries on re-process" do + process(tx) + assert_no_difference "@account.entries.count" do + process(tx) + end + end + + # --------------------------------------------------------------------------- + # merchant creation + # --------------------------------------------------------------------------- + + test "creates merchant from counterpartyName" do + process(tx(counterparty_name: "Stripe Inc")) + entry = @account.entries.find_by(external_id: "mercury_tx_001", source: "mercury") + assert entry.entryable.merchant.present? + assert_equal "Stripe Inc", entry.entryable.merchant.name + end + + # --------------------------------------------------------------------------- + # missing linked account + # --------------------------------------------------------------------------- + + test "returns nil when mercury_account has no linked account" do + AccountProvider.where(provider: @mercury_account).destroy_all + + assert_no_difference "@account.entries.count" do + result = process(tx) + assert_nil result + end + end + + private + + def process(transaction_data) + MercuryEntry::Processor.new(transaction_data, mercury_account: @mercury_account).process + end + + def tx( + id: "tx_001", + amount: 100.0, + status: "sent", + counterparty_name: nil, + counterparty_nickname: nil, + counterparty_id: nil, + bank_description: "Test Transaction", + kind: nil, + note: nil, + details: nil, + posted_at: "2024-06-01T12:00:00Z", + created_at: "2024-06-01T10:00:00Z" + ) + { + "id" => id, + "amount" => amount, + "status" => status, + "counterpartyName" => counterparty_name, + "counterpartyNickname" => counterparty_nickname, + "counterpartyId" => counterparty_id, + "bankDescription" => bank_description, + "kind" => kind, + "note" => note, + "details" => details, + "postedAt" => posted_at, + "createdAt" => created_at + } + end +end diff --git a/test/models/mercury_item/importer_test.rb b/test/models/mercury_item/importer_test.rb new file mode 100644 index 0000000000..e42b9389e6 --- /dev/null +++ b/test/models/mercury_item/importer_test.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +require "test_helper" + +class MercuryItem::ImporterTest < ActiveSupport::TestCase + setup do + @family = families(:dylan_family) + @item = MercuryItem.create!(family: @family, name: "Mercury", token: "tok") + @provider = mock + @provider.stubs(:get_accounts).returns({ accounts: [] }) + @provider.stubs(:get_account_transactions).returns({ transactions: [] }) + end + + # --------------------------------------------------------------------------- + # account discovery + # --------------------------------------------------------------------------- + + test "creates unlinked mercury_account records for newly discovered accounts" do + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_001", "Business Checking") ] }) + + assert_difference "@item.mercury_accounts.count", 1 do + run_import + end + + acct = @item.mercury_accounts.find_by(account_id: "acc_001") + assert acct + assert_equal "Business Checking", acct.name + assert_equal "USD", acct.currency + end + + test "does not duplicate existing unlinked account records on re-import" do + @item.mercury_accounts.create!(name: "Existing", account_id: "acc_001", currency: "USD") + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_001", "Business Checking") ] }) + + assert_no_difference "@item.mercury_accounts.count" do + run_import + end + end + + test "updates current_balance on linked accounts" do + account, mercury_account = create_linked_account("acc_balance") + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_balance", "Checking", balance: 5_000.0) ] }) + @provider.stubs(:get_account_transactions).with("acc_balance", anything).returns({ transactions: [] }) + + run_import + + assert_in_delta 5_000.0, mercury_account.reload.current_balance, 0.01 + end + + # --------------------------------------------------------------------------- + # transaction deduplication + # --------------------------------------------------------------------------- + + test "appends new transactions and skips duplicate ids" do + _account, mercury_account = create_linked_account("acc_dedup", + raw_transactions: [ tx_payload("tx_old") ]) + + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_dedup", "Checking") ] }) + @provider.stubs(:get_account_transactions).with("acc_dedup", anything).returns({ + transactions: [ tx_payload("tx_old"), tx_payload("tx_new") ] + }) + + run_import + + ids = mercury_account.reload.raw_transactions_payload.map { |tx| tx["id"] } + assert_includes ids, "tx_old" + assert_includes ids, "tx_new" + assert_equal 2, ids.uniq.size + end + + # --------------------------------------------------------------------------- + # sync window + # --------------------------------------------------------------------------- + + test "uses 90-day window for account with no stored transactions" do + create_linked_account("acc_first") + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_first", "Checking") ] }) + + captured_start = nil + @provider.stubs(:get_account_transactions).with do |_id, opts| + captured_start = opts[:start_date] + true + end.returns({ transactions: [] }) + + run_import + + assert_not_nil captured_start + assert captured_start >= 91.days.ago.to_date, + "first-sync start date must be within 90 days" + end + + test "uses last_synced_at minus 7 days when account has existing transactions" do + ten_days_ago = 10.days.ago + _account, mercury_account = create_linked_account("acc_resync", + raw_transactions: [ tx_payload("existing_tx") ]) + + @item.stubs(:last_synced_at).returns(ten_days_ago) + @provider.stubs(:get_accounts).returns({ accounts: [ account_payload("acc_resync", "Checking") ] }) + + captured_start = nil + @provider.stubs(:get_account_transactions).with do |_id, opts| + captured_start = opts[:start_date] + true + end.returns({ transactions: [] }) + + run_import + + expected = (ten_days_ago - 7.days).to_date + assert_equal expected, captured_start.to_date + end + + # --------------------------------------------------------------------------- + # auth error handling + # --------------------------------------------------------------------------- + + test "marks item requires_update on 401 from Mercury API" do + @provider.stubs(:get_accounts).raises( + Provider::Mercury::MercuryError.new("Unauthorized", :unauthorized) + ) + + run_import + + assert @item.reload.requires_update? + end + + private + + def run_import + MercuryItem::Importer.new(@item, mercury_provider: @provider).import + end + + def create_linked_account(account_id, raw_transactions: []) + account = @family.accounts.create!( + name: account_id, balance: 0, currency: "USD", + accountable: Depository.new(subtype: "checking") + ) + mercury_account = @item.mercury_accounts.create!( + name: account_id, account_id: account_id, currency: "USD", + current_balance: 0, raw_transactions_payload: raw_transactions + ) + AccountProvider.create!(provider: mercury_account, account: account) + [ account, mercury_account ] + end + + def account_payload(id, name, balance: 1_000.0) + { id: id, name: name, nickname: nil, legalBusinessName: nil, + currentBalance: balance, availableBalance: balance, + status: "active", type: "checking", kind: "checking" } + end + + def tx_payload(id, amount: 50.0, status: "sent") + { "id" => id, "amount" => amount, "status" => status, + "bankDescription" => "Test", "createdAt" => "2024-06-01T00:00:00Z", + "postedAt" => "2024-06-01T00:00:00Z" } + end +end