fix dex price e6 overflow with bigint math

Author: andrewnguyen22

fsm/dex.go

@@ -706,13 +706,19 @@ func (s *StateMachine) getPrice(batch *lib.DexBatch) (p *lib.DexPrice, err lib.E
 	if batch.PoolSize == 0 || batch.CounterPoolSize == 0 {
 		return nil, ErrInvalidLiquidityPool()
 	}
+	priceNumerator := new(big.Int).Mul(new(big.Int).SetUint64(batch.PoolSize), big.NewInt(1_000_000))
+	price := priceNumerator.Div(priceNumerator, new(big.Int).SetUint64(batch.CounterPoolSize))
+	// DexPrice stores e6-scaled price as uint64; reject unrepresentable values instead of wrapping.
+	if !price.IsUint64() {
+		return nil, ErrInvalidLiquidityPool()
+	}
 	// exit with the dex price
 	return &lib.DexPrice{
 		LocalChainId:  s.Config.ChainId,
 		RemoteChainId: batch.Committee,
 		LocalPool:     batch.PoolSize,
 		RemotePool:    batch.CounterPoolSize,
-		E6ScaledPrice: batch.PoolSize * 1_000_000 / batch.CounterPoolSize,
+		E6ScaledPrice: price.Uint64(),
 	}, nil
 }
 

fsm/dex_test.go

@@ -3453,6 +3453,24 @@ func clearLocks(t *testing.T, chain1, chain2 StateMachine, chain1Id, chain2Id ui
 	require.NoError(t, chain2.Delete(KeyForNextBatch(chain1Id)))
 }
 
+func TestGetPriceUsesBigIntForE6ScaledPrice(t *testing.T) {
+	sm := newTestStateMachine(t)
+	sm.Config.ChainId = 10
+
+	price, err := sm.getPrice(&lib.DexBatch{
+		Committee:       1,
+		PoolSize:        24515572443456,
+		CounterPoolSize: 50998020079,
+	})
+	require.NoError(t, err)
+	require.NotNil(t, price)
+	require.Equal(t, uint64(24515572443456), price.LocalPool)
+	require.Equal(t, uint64(50998020079), price.RemotePool)
+	require.Equal(t, uint64(480716161), price.E6ScaledPrice)
+	// Previous uint64 multiplication overflow produced this incorrect wrapped value.
+	require.NotEqual(t, uint64(119001254), price.E6ScaledPrice)
+}
+
 var _ lib.RCManagerI = new(MockRCManager)
 
 // MockRCManager is a minimal mock implementation of lib.RCManagerI for testing