From 392b5717cbdf5c3e257dbd278ea0906e727ac0ba Mon Sep 17 00:00:00 2001 From: Michael Podtserkovskii Date: Mon, 23 Sep 2024 01:41:42 +0100 Subject: [PATCH] Implement Freetrade parser --- cgt_calc/args_parser.py | 7 + cgt_calc/main.py | 1 + cgt_calc/parsers/__init__.py | 7 + cgt_calc/parsers/freetrade.py | 197 ++++++++++++++++++ tests/test_data/freetrade/expected_output.txt | 29 +++ tests/test_data/freetrade/transactions.csv | 21 ++ tests/test_data/raw/expected_output.txt | 1 + .../schwab_cash_merger/expected_output.txt | 1 + .../test_run_with_example_files_output.txt | 1 + ...aresight_files_no_balance_check_output.txt | 1 + .../trading212_2024/expected_output.txt | 1 + tests/test_freetrade.py | 29 +++ 12 files changed, 296 insertions(+) create mode 100644 cgt_calc/parsers/freetrade.py create mode 100644 tests/test_data/freetrade/expected_output.txt create mode 100644 tests/test_data/freetrade/transactions.csv create mode 100644 tests/test_freetrade.py diff --git a/cgt_calc/args_parser.py b/cgt_calc/args_parser.py index ef713c94..a0c73864 100644 --- a/cgt_calc/args_parser.py +++ b/cgt_calc/args_parser.py @@ -74,6 +74,13 @@ def create_parser() -> argparse.ArgumentParser: nargs="?", help="folder containing reports from Sharesight in CSV format", ) + parser.add_argument( + "--freetrade", + type=str, + default=None, + nargs="?", + help="file containing the exported transactions from Freetrade in CSV format", + ) parser.add_argument( "--exchange-rates-file", diff --git a/cgt_calc/main.py b/cgt_calc/main.py index 0b027f0e..4d071948 100755 --- a/cgt_calc/main.py +++ b/cgt_calc/main.py @@ -835,6 +835,7 @@ def main() -> int: args.mssb, args.sharesight, args.raw, + args.freetrade, ) converter = CurrencyConverter(args.exchange_rates_file) initial_prices = InitialPrices(read_initial_prices(args.initial_prices)) diff --git a/cgt_calc/parsers/__init__.py b/cgt_calc/parsers/__init__.py index f751eaf6..9deda260 100644 --- a/cgt_calc/parsers/__init__.py +++ b/cgt_calc/parsers/__init__.py @@ -11,6 +11,7 @@ from cgt_calc.const import DEFAULT_INITIAL_PRICES_FILE from cgt_calc.exceptions import UnexpectedColumnCountError +from cgt_calc.parsers.freetrade import read_freetrade_transactions from cgt_calc.resources import RESOURCES_PACKAGE from .mssb import read_mssb_transactions @@ -56,6 +57,7 @@ def read_broker_transactions( mssb_transactions_folder: str | None, sharesight_transactions_folder: str | None, raw_transactions_file: str | None, + freetrade_transactions_file: str | None, ) -> list[BrokerTransaction]: """Read transactions for all brokers.""" transactions = [] @@ -93,6 +95,11 @@ def read_broker_transactions( else: print("INFO: No raw file provided") + if freetrade_transactions_file is not None: + transactions += read_freetrade_transactions(freetrade_transactions_file) + else: + print("INFO: No freetrade file provided") + transactions.sort(key=lambda k: k.date) return transactions diff --git a/cgt_calc/parsers/freetrade.py b/cgt_calc/parsers/freetrade.py new file mode 100644 index 00000000..97f03068 --- /dev/null +++ b/cgt_calc/parsers/freetrade.py @@ -0,0 +1,197 @@ +"""Freetrade parser.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from typing import Final + +from cgt_calc.exceptions import ParsingError +from cgt_calc.model import ActionType, BrokerTransaction + +COLUMNS: Final[list[str]] = [ + "Title", + "Type", + "Timestamp", + "Account Currency", + "Total Amount", + "Buy / Sell", + "Ticker", + "ISIN", + "Price per Share in Account Currency", + "Stamp Duty", + "Quantity", + "Venue", + "Order ID", + "Order Type", + "Instrument Currency", + "Total Shares Amount", + "Price per Share", + "FX Rate", + "Base FX Rate", + "FX Fee (BPS)", + "FX Fee Amount", + "Dividend Ex Date", + "Dividend Pay Date", + "Dividend Eligible Quantity", + "Dividend Amount Per Share", + "Dividend Gross Distribution Amount", + "Dividend Net Distribution Amount", + "Dividend Withheld Tax Percentage", + "Dividend Withheld Tax Amount", +] + + +class FreetradeTransaction(BrokerTransaction): + """Represents a single Freetrade transaction.""" + + def __init__(self, header: list[str], row_raw: list[str], filename: str): + """Create transaction from CSV row.""" + row = dict(zip(header, row_raw)) + action = action_from_str(row["Type"], row["Buy / Sell"], filename) + + symbol = row["Ticker"] if row["Ticker"] != "" else None + if symbol is None and action not in [ActionType.TRANSFER, ActionType.INTEREST]: + raise ParsingError(filename, f"No symbol for action: {action}") + + # I believe GIA account at Freetrade can be only in GBP + if row["Account Currency"] != "GBP": + raise ParsingError(filename, "Non-GBP accounts are unsupported") + + # Convert all numbers in GBP using Freetrade rates + if action in [ActionType.SELL, ActionType.BUY]: + quantity = Decimal(row["Quantity"]) + price = Decimal(row["Price per Share"]) + amount = Decimal(row["Total Shares Amount"]) + currency = row["Instrument Currency"] + if currency != "GBP": + fx_rate = Decimal(row["FX Rate"]) + price /= fx_rate + amount /= fx_rate + currency = "GBP" + elif action == ActionType.DIVIDEND: + # Total amount before US tax withholding + amount = Decimal(row["Dividend Gross Distribution Amount"]) + quantity, price = None, None + currency = row["Instrument Currency"] + if currency != "GBP": + # FX Rate is not defined for dividends, + # but we can use base one as there's no fee + amount /= Decimal(row["Base FX Rate"]) + currency = "GBP" + elif action in [ActionType.TRANSFER, ActionType.INTEREST]: + amount = Decimal(row["Total Amount"]) + quantity, price = None, None + currency = "GBP" + else: + raise ParsingError( + filename, f"Numbers parsing unimplemented for action: {action}" + ) + + if row["Type"] == "FREESHARE_ORDER": + price = Decimal(0) + amount = Decimal(0) + + amount_negative = action == ActionType.BUY or row["Type"] == "WITHDRAWAL" + if amount is not None and amount_negative: + amount *= -1 + + super().__init__( + date=datetime.fromisoformat(row["Timestamp"]).date(), + action=action, + symbol=symbol, + description=f"{row["Title"]} {action}", + quantity=quantity, + price=price, + fees=Decimal("0"), # not implemented + amount=amount, + currency=currency, + broker="Freetrade", + ) + + +def action_from_str(type: str, buy_sell: str, filename: str) -> ActionType: + """Infer action type.""" + if type == "INTEREST_FROM_CASH": + return ActionType.INTEREST + if type == "DIVIDEND": + return ActionType.DIVIDEND + if type in ["TOP_UP", "WITHDRAWAL"]: + return ActionType.TRANSFER + if type in ["ORDER", "FREESHARE_ORDER"]: + if buy_sell == "BUY": + return ActionType.BUY + if buy_sell == "SELL": + return ActionType.SELL + + raise ParsingError(filename, f"Unknown buy_sell: {buy_sell}") + + raise ParsingError(filename, f"Unknown type: {type}") + + +def validate_header(header: list[str], filename: str) -> None: + """Check if header is valid.""" + for actual in header: + if actual not in COLUMNS: + msg = f"Unknown column {actual}" + raise ParsingError(filename, msg) + + +@dataclass +class StockSplit: + """Info about stock split.""" + + symbol: str + date: date + factor: int + + +STOCK_SPLIT_INFO = [ + StockSplit(symbol="GOOGL", date=datetime(2022, 7, 18).date(), factor=20), + StockSplit(symbol="TSLA", date=datetime(2022, 8, 25).date(), factor=3), + StockSplit(symbol="NDAQ", date=datetime(2022, 8, 29).date(), factor=3), +] + + +# Pretend there was no split +def _handle_stock_split(transaction: BrokerTransaction) -> BrokerTransaction: + for split in STOCK_SPLIT_INFO: + if ( + transaction.symbol == split.symbol + and transaction.action == ActionType.SELL + and transaction.date > split.date + ): + if transaction.quantity: + transaction.quantity /= split.factor + if transaction.price: + transaction.price *= split.factor + + return transaction + + +def read_freetrade_transactions(transactions_file: str) -> list[BrokerTransaction]: + """Parse Freetrade transactions from a CSV file.""" + try: + with Path(transactions_file).open(encoding="utf-8") as file: + lines = list(csv.reader(file)) + header = lines[0] + validate_header(header, str(file)) + lines = lines[1:] + # HACK: reverse transactions to avoid negative balance issues + # the proper fix would be to use datetime in BrokerTransaction + lines.reverse() + transactions: list[BrokerTransaction] = [ + _handle_stock_split(FreetradeTransaction(header, row, str(file))) + for row in lines + ] + if len(transactions) == 0: + print(f"WARNING: no transactions detected in file {file}") + return transactions + except FileNotFoundError: + print( + f"WARNING: Couldn't locate Freetrade transactions file({transactions_file})" + ) + return [] diff --git a/tests/test_data/freetrade/expected_output.txt b/tests/test_data/freetrade/expected_output.txt new file mode 100644 index 00000000..e922162c --- /dev/null +++ b/tests/test_data/freetrade/expected_output.txt @@ -0,0 +1,29 @@ +INFO: No schwab file provided +INFO: No schwab Equity Award JSON file provided +INFO: No trading212 folder provided +INFO: No mssb folder provided +INFO: No sharesight file provided +INFO: No raw file provided +First pass completed +Final portfolio: +Final balance: + Freetrade: 10278.93 (GBP) +Dividends: £1.78 +Dividend taxes: £0.00 +Interest: £9.02 +Disposal proceeds: £4262.50 + + +Second pass completed +Portfolio at the end of 2023/2024 tax year: +For tax year 2023/2024: +Number of disposals: 4 +Disposal proceeds: £4262.50 +Allowable costs: £4006.53 +Capital gain: £460.18 +Capital loss: £204.21 +Total capital gain: £255.97 +Taxable capital gain: £0 + +Generate calculations report +All done! diff --git a/tests/test_data/freetrade/transactions.csv b/tests/test_data/freetrade/transactions.csv new file mode 100644 index 00000000..1ea8708b --- /dev/null +++ b/tests/test_data/freetrade/transactions.csv @@ -0,0 +1,21 @@ +Title,Type,Timestamp,Account Currency,Total Amount,Buy / Sell,Ticker,ISIN,Price per Share in Account Currency,Stamp Duty,Quantity,Venue,Order ID,Order Type,Instrument Currency,Total Shares Amount,Price per Share,FX Rate,Base FX Rate,FX Fee (BPS),FX Fee Amount,Dividend Ex Date,Dividend Pay Date,Dividend Eligible Quantity,Dividend Amount Per Share,Dividend Gross Distribution Amount,Dividend Net Distribution Amount,Dividend Withheld Tax Percentage,Dividend Withheld Tax Amount +S&P 500,ORDER,2024-01-16T10:01:02.811Z,GBP,2930.36,SELL,SPXP,IE00B3YCGJ38,732.59000000,0.00,4.00000000,London Stock Exchange,PEJJSM8NABDD,BASIC,GBP,2930.36,732.59000000,,,0,,,,,,,,, +Alphabet,ORDER,2024-01-12T17:26:04.797Z,GBP,310.66,SELL,GOOGL,US02079K3059,112.01515004,0.00,2.78980120,Multiple,2RSI45ZGENMJ,MARKET,USD,398.05,142.68172100,1.28126511,1.27374999,59,1.84,,,,,,,, +Nasdaq,ORDER,2024-01-12T17:25:47.793Z,GBP,280.80,SELL,NDAQ,US6311031081,44.37543198,0.00,6.36545916,Multiple,EKWK55H45G1W,MARKET,USD,359.79,56.52113100,1.28125507,1.27374000,59,1.67,,,,,,,, +Tesla,ORDER,2024-01-12T17:25:31.041Z,GBP,740.63,SELL,TSLA,US88160R1014,171.26705002,0.00,4.35010704,Multiple,L5C6WVOML0DJ,MARKET,USD,949.11,218.18110300,1.28144618,1.27392999,59,4.40,,,,,,,, +Nasdaq,DIVIDEND,2023-12-22T17:11:00.000Z,GBP,0.93,,NDAQ,US6311031081,,,6.36545916,,,,USD,,,,0.78491703,0,0.00,2023-12-07,2023-12-22,6.36545916,0.22000000,1.40,1.19,15,0.21 +Interest,INTEREST_FROM_CASH,2023-10-16T00:00:00.000Z,GBP,9.02,,,,,,,,,,,,,,,,,,,,,,,, +Nasdaq,DIVIDEND,2022-03-25T02:35:00.000Z,GBP,0.74,,NDAQ,US6311031081,,,2.12181972,,,,USD,,,,0.76110000,0,0.00,2022-03-10,2022-03-25,2.12181972,0.54000000,1.15,0.97,15,0.18 +Tesla,ORDER,2022-02-01T15:14:23.014Z,GBP,5.59,BUY,TSLA,US88160R1014,678.77225238,0.00,0.00819126,Drivewealth,AMN762KCC2G3,MARKET,USD,7.50,915.61000000,1.34403451,1.35011000,45,0.03,,,,,,,, +S&P 500,ORDER,2022-02-01T15:09:49.696Z,GBP,1256.78,BUY,SPXP,IE00B3YCGJ38,628.39000000,0.00,2.00000000,London Stock Exchange,PYJAGIY7EMQN,MARKET,GBP,1256.78,628.39000000,,,,,,,,,,,, +Nasdaq,ORDER,2022-02-01T15:04:35.355Z,GBP,139.79,BUY,NDAQ,US6311031081,132.51921688,0.00,1.05011185,Multiple,9AFQXD6BXUZ2,MARKET,USD,187.76,178.80000000,1.34316842,1.34924000,45,0.63,,,,,,,, +Alphabet,ORDER,2022-02-01T15:04:18.478Z,GBP,139.80,BUY,GOOGL,US02079K3059,2014.06123574,0.00,0.06909919,Drivewealth,GPXKDQQGE2RY,MARKET,USD,187.69,2716.24000000,1.34265076,1.34872000,45,0.63,,,,,,,, +Tesla,ORDER,2022-02-01T15:03:12.817Z,GBP,464.38,BUY,TSLA,US88160R1014,674.85659078,0.00,0.68503443,Drivewealth,JA2UNVA48W11,MARKET,USD,623.58,910.29000000,1.34281004,1.34888000,45,2.08,,,,,,,, +S&P 500,ORDER,2022-01-31T10:05:09.670Z,GBP,1245.72,BUY,SPXP,IE00B3YCGJ38,622.86000000,0.00,2.00000000,London Stock Exchange,2OWT86SBH5EE,BASIC,GBP,1245.72,622.86000000,,,,,,,,,,,, +Nasdaq,ORDER,2022-01-28T20:49:47.645Z,GBP,139.80,BUY,NDAQ,US6311031081,129.85814875,0.00,1.07170787,Multiple,A5ZKDJ5LTLVE,MARKET,USD,186.37,173.90000000,1.33321342,1.33924000,45,0.63,,,,,,,, +Alphabet,ORDER,2022-01-28T20:49:12.609Z,GBP,139.79,BUY,GOOGL,US02079K3059,1976.96093258,0.00,0.07039087,Drivewealth,BZSWP4MI4D57,MARKET,USD,186.35,2647.36000000,1.33310392,1.33913000,45,0.63,,,,,,,, +Tesla,ORDER,2022-01-28T20:47:15.462Z,GBP,464.29,BUY,TSLA,US88160R1014,622.88226129,0.00,0.74205035,Drivewealth,6I8P755U1BPY,MARKET,USD,618.87,834.00000000,1.33293468,1.33896000,45,2.08,,,,,,,, +Top up,TOP_UP,2022-01-28T20:41:58.209Z,GBP,10000.00,,,,,,,,,,,,,,,,,,,,,,,, +Tesla,ORDER,2021-12-14T15:19:00.899Z,GBP,10.62,BUY,TSLA,US88160R1014,716.14212813,0.00,0.01475964,Drivewealth,Y5283HQTZIGJ,MARKET,USD,13.97,946.50000000,1.31687727,1.32283000,45,0.05,,,,,,,, +ThredUp,ORDER,2021-11-11T15:18:22.892Z,GBP,10.63,SELL,TDUP,US88556E1029,10.68000000,0.00,1.00000000,"New York Stock Exchange, Inc.",4E9L0RXWBMMP,MARKET,USD,14.14,14.14000000,1.32916445,1.32321000,45,0.05,,,,,,,, +ThredUp,FREESHARE_ORDER,2021-11-10T18:11:43.475Z,GBP,11.04,BUY,TDUP,US88556E1029,11.03772436,0.00,1.00000000,New York Stock Exchange,O52KRZI3BRAQ,,USD,4554.89,14.60000000,1.32265000,1.32265000,45,0.00,,,,,,,, diff --git a/tests/test_data/raw/expected_output.txt b/tests/test_data/raw/expected_output.txt index 6c340b88..49aa0016 100644 --- a/tests/test_data/raw/expected_output.txt +++ b/tests/test_data/raw/expected_output.txt @@ -3,6 +3,7 @@ INFO: No schwab Equity Award JSON file provided INFO: No trading212 folder provided INFO: No mssb folder provided INFO: No sharesight file provided +INFO: No freetrade file provided First pass completed Final portfolio: AMZN: 210.00 diff --git a/tests/test_data/schwab_cash_merger/expected_output.txt b/tests/test_data/schwab_cash_merger/expected_output.txt index bfc907a1..ebbdeff6 100644 --- a/tests/test_data/schwab_cash_merger/expected_output.txt +++ b/tests/test_data/schwab_cash_merger/expected_output.txt @@ -5,6 +5,7 @@ INFO: No trading212 folder provided INFO: No mssb folder provided INFO: No sharesight file provided INFO: No raw file provided +INFO: No freetrade file provided First pass completed Final portfolio: Final balance: diff --git a/tests/test_data/test_run_with_example_files_output.txt b/tests/test_data/test_run_with_example_files_output.txt index 7ff5fdb6..c53c55a6 100644 --- a/tests/test_data/test_run_with_example_files_output.txt +++ b/tests/test_data/test_run_with_example_files_output.txt @@ -3,6 +3,7 @@ INFO: No schwab Equity Award JSON file provided Parsing tests/test_data/trading212/from_2020-09-11_to_2021-04-02.csv INFO: No sharesight file provided INFO: No raw file provided +INFO: No freetrade file provided First pass completed Final portfolio: GE: 1.00 diff --git a/tests/test_data/test_run_with_sharesight_files_no_balance_check_output.txt b/tests/test_data/test_run_with_sharesight_files_no_balance_check_output.txt index 057ca387..32e775ed 100644 --- a/tests/test_data/test_run_with_sharesight_files_no_balance_check_output.txt +++ b/tests/test_data/test_run_with_sharesight_files_no_balance_check_output.txt @@ -3,6 +3,7 @@ INFO: No schwab Equity Award JSON file provided INFO: No trading212 folder provided INFO: No mssb folder provided INFO: No raw file provided +INFO: No freetrade file provided First pass completed Final portfolio: FX:ETH: 2.59 diff --git a/tests/test_data/trading212_2024/expected_output.txt b/tests/test_data/trading212_2024/expected_output.txt index 5df9659a..76607c7c 100644 --- a/tests/test_data/trading212_2024/expected_output.txt +++ b/tests/test_data/trading212_2024/expected_output.txt @@ -4,6 +4,7 @@ Parsing tests/test_data/trading212_2024/transactions.csv INFO: No mssb folder provided INFO: No sharesight file provided INFO: No raw file provided +INFO: No freetrade file provided First pass completed Final portfolio: Final balance: diff --git a/tests/test_freetrade.py b/tests/test_freetrade.py new file mode 100644 index 00000000..ed7da416 --- /dev/null +++ b/tests/test_freetrade.py @@ -0,0 +1,29 @@ +"""Test Freetrade support.""" + +from pathlib import Path +import subprocess +import sys + + +def test_run_with_freetrade_file() -> None: + """Runs the script and verifies it doesn't fail.""" + cmd = [ + sys.executable, + "-m", + "cgt_calc.main", + "--year", + "2023", + "--freetrade", + "tests/test_data/freetrade/transactions.csv", + "--no-pdflatex", + ] + result = subprocess.run(cmd, check=True, capture_output=True) + assert result.stderr == b"", "Run with example files generated errors" + expected_file = Path("tests") / "test_data" / "freetrade" / "expected_output.txt" + expected = expected_file.read_text() + cmd_str = " ".join([param if param else "''" for param in cmd]) + assert result.stdout.decode("utf-8") == expected, ( + "Run with example files generated unexpected outputs, " + "if you added new features update the test with:\n" + f"{cmd_str} > {expected_file}" + )