✨ Implemented mgm-hurry export_trades

mgm-hurry

@@ -50,7 +50,7 @@ class MGMHurry:
     Attributes:
         monigomani_config   The monigomani_config object
         freqtrade_cli       The FreqtradeCli object
-        logger             The logging function out of MoniGoManiLogger
+        logger              The logging function out of MoniGoManiLogger
     """
     monigomani_config: MoniGoManiConfig
     freqtrade_cli: FreqtradeCli
@@ -131,6 +131,14 @@ class MGMHurry:
                 {'name': 'Yes, Dry-Run please', 'value': 1},
                 {'name': 'Yes, for real!', 'value': 2}
             ]
+        }, {
+            'type': 'confirm',
+            'name': 'export_trades_backtest',
+            'message': '💨 Do you want to export trades from backtest now?'
+        }, {
+            'type': 'confirm',
+            'name': 'export_trades_hyperopt',
+            'message': '💨 Do you want to export trades from hyperopt now?'
         }])
 
         if answers == {}:
@@ -211,6 +219,51 @@ class MGMHurry:
         if answers.get('do_backtest') is True:
             self.backtest()
 
+        if answers.get('export_trades_backtest') is True:
+            last_result = prompt(questions=[{
+                'type': 'list',
+                'name': 'last_result',
+                'message': 'Do you want to export trades from the latest backtest results ?',
+                'choices': [{'name': 'Yes', 'value': 1}, {'name': 'No', 'value': 0}]}
+            ])
+
+            input_file = None
+            if last_result.get('last_result') == 0:
+                input_file = prompt(questions=[{
+                        'type': 'input',
+                        'name': 'input_file',
+                        'message': 'Please specify the filename (.json) you want to use ?'
+                }])
+                input_file = input_file.get('input_file')
+
+            self.export_trades_backtest(input_file_name=input_file)
+
+        if answers.get('export_trades_hyperopt') is True:
+            last_result = prompt(questions=[{
+                'type': 'list',
+                'name': 'last_result',
+                'message': 'Do you want to export trades from the latest hyperopt results ?',
+                'choices': [{'name': 'Yes', 'value': 1}, {'name': 'No', 'value': 0}]}
+            ])
+
+            input_file = None
+            if last_result.get('last_result') == 0:
+                input_file = prompt(questions=[{
+                        'type': 'input',
+                        'name': 'input_file',
+                        'message': 'Please specify the filename (.fthypt) you want to use ?'
+                }])
+                input_file = input_file.get('input_file')
+
+            epoch_choice = prompt(questions=[{
+                'type': 'input',
+                'name': 'ho_epoch',
+                'message': 'Choose the epoch which you want to export trades from (0 = export all epochs)',
+                'filter': lambda val: int(val)
+                }])
+
+            self.export_trades_hyperopt(input_file_name=input_file, epoch=epoch_choice.get('ho_epoch'))
+
         if answers.get('start_trading') > 0:
             if answers.get('start_trading') == 2:
                 self.start_trader(False)
@@ -810,7 +863,7 @@ class MGMHurry:
 
         fthypt_name = None
         if fthypt is not None:
-            fthypt_name = self.freqtrade_cli.parse_fthypt_name(fthypt_name=fthypt)
+            fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=fthypt)
 
         if epoch == 0:
             self.logger.error(Color.red('🤷 Please pass the epoch number through. '
@@ -1053,7 +1106,7 @@ class MGMHurry:
         fthypt_name = None
         fthypt_file_path = None
         if fthypt is not None:
-            fthypt_name = self.freqtrade_cli.parse_fthypt_name(fthypt_name=fthypt)
+            fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=fthypt)
             fthypt_file_path = f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}'
 
         if output_file_name is None:
@@ -1073,6 +1126,83 @@ class MGMHurry:
                                                     message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Results ⬇️',
                                                     results_paths=[output_file_path])
 
+    def export_trades_backtest(self, strategy: str = 'MoniGoManiHyperStrategy',
+                   output_file_name: str = None, input_file_name: str = None) -> None:
+        """
+        Export the detailed trades from backtest result to an easy to interpret/sort/filter '.csv' SpreadSheet.
+
+        :param strategy: Name of the used Strategy, defaults to 'MoniGoManiHyperStrategy'
+        :param output_file_name: (str, Optional) Custom filename for the CsvTrades '.csv' file
+            that will be stored in the format: 'CsvTrades-<output_file_name>.csv'.
+            Defaults to 'StrategyName + Current DateTime'.
+        :param input_file_name: (str, Optional) '.fthypt' file to export results from.. If 'true' is passed,
+            an interactive prompt will launch to pick an '.fthypt' file of choice. Defaults to latest
+        """
+
+        self.logger.info(Color.title(f'👉 Exporting Backtest Trades to SpreadSheet (.csv) format.'))
+
+        fthypt_name = None
+        fthypt_file_path = None
+        if input_file_name is not None:
+            fthypt_name = self.freqtrade_cli.parse_backtest_filename(bt_filename=input_file_name)
+            fthypt_file_path = f'{self.basedir}/user_data/backtest_results/{fthypt_name}'
+
+        if output_file_name is None:
+            output_file_name = f'MoniGoManiHyperStrategy-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
+        csv_results_file_name = f'CsvTrades-{output_file_name}'
+        output_file_path = f'{self.basedir}/user_data/csv_results/{csv_results_file_name}.csv'
+
+        command = (f'python3 {self.basedir}/user_data/mgm_tools/ExportCsvBacktestTrades.py -o {output_file_path}')
+
+        if fthypt_name is not None:
+            command += f' -i {fthypt_file_path}'
+
+        self.monigomani_cli.run_command(command=command)
+
+        MoniGoManiLogger(self.basedir).post_message(username=self.monigomani_config.config['username'],
+                                                    message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Trades ⬇️',
+                                                    results_paths=[output_file_path])
+
+    def export_trades_hyperopt(self, strategy: str = 'MoniGoManiHyperStrategy',
+                   output_file_name: str = None, input_file_name: str = None, epoch: int = None) -> None:
+        """
+        Export the detailed trades from hyperopt results to an easy to interpret/sort/filter '.csv' SpreadSheet.
+
+        :param strategy: Name of the used Strategy, defaults to 'MoniGoManiHyperStrategy'
+        :param output_file_name: (str, Optional) Custom filename for the CsvTrades '.csv' file
+            that will be stored in the format: 'CsvTrades-<output_file_name>.csv'.
+            Defaults to 'StrategyName + Current DateTime'.
+        :param input_file_name: (str, Optional) '.fthypt' file to export results from.. If 'true' is passed,
+            an interactive prompt will launch to pick an '.fthypt' file of choice. Defaults to latest
+        """
+
+        self.logger.info(Color.title(f'👉 Exporting HyperOpt Trades to SpreadSheet (.csv) format.'))
+
+        fthypt_name = None
+        fthypt_file_path = None
+        if input_file_name is not None:
+            fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=input_file_name)
+            fthypt_file_path = f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}'
+
+        if output_file_name is None:
+            output_file_name = f'MoniGoManiHyperStrategy-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}'
+        csv_results_file_name = f'CsvTrades-{output_file_name}'
+        output_file_path = f'{self.basedir}/user_data/csv_results/{csv_results_file_name}.csv'
+
+        command = (f'python3 {self.basedir}/user_data/mgm_tools/ExportCsvHyperoptTrades.py -o {output_file_path}')
+
+        if fthypt_name is not None:
+            command += f' -i {fthypt_file_path}'
+
+        if epoch is not None and epoch > 0:
+            command += f' -n {epoch}'
+
+        self.monigomani_cli.run_command(command=command)
+
+        MoniGoManiLogger(self.basedir).post_message(username=self.monigomani_config.config['username'],
+                                                    message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Trades ⬇️',
+                                                    results_paths=[output_file_path])
+
     def export_results(self):
         """
         Export the results that are selected. Creates a '.zip' archive of the various files created by HyperOpt runs after the user selects which HyperOpt run they would like to export. 

user_data/mgm_tools/ExportCsvBacktestTrades.py

@@ -0,0 +1,78 @@
+import getopt
+import os
+import sys
+from pathlib import Path
+
+import pandas as pd
+import rapidjson
+from pandas import json_normalize
+
+
+def ExportCsvBacktestTrades(input_file, output_file):
+    basedir = os.getcwd()
+
+    # Fetch the latest '.json' filename from last_result
+    if input_file is None or input_file == '':
+        last_result_file = Path(f'{basedir}/user_data/backtest_results/.last_result.json')
+        with last_result_file.open('r') as f:
+            data = [rapidjson.loads(line) for line in f]
+        last_rf = json_normalize(data, max_level=1)
+
+        results_file = Path(f'{basedir}/user_data/backtest_results/{last_rf["latest_backtest"].loc[0]}')
+    else:
+        results_file = Path(input_file)
+
+    run_id = results_file.name.split('.')[0]
+
+    # Open and normalize '.json' to dataframe
+    with results_file.open('r') as f:
+        data = [rapidjson.loads(line) for line in f]
+    backtest = json_normalize(data, max_level=0)
+
+    # Define result dataframe columns
+    list_of_columns = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration', 
+                       'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open']
+
+    backtest_result = json_normalize(backtest["strategy"], max_level=1)
+    trades = pd.DataFrame.from_dict(backtest_result.iloc[0, 0])
+
+    if len(trades) > 0:
+        trades = trades.loc[:, list_of_columns]
+        trades['stake_amount'] = trades['stake_amount'].apply(lambda x: round(x, 3))
+        trades['amount'] = trades['amount'].apply(lambda x: round(x, 3))
+        trades['trade_duration'] = trades['trade_duration'].apply(lambda x: round(x / 3600, 2))
+        trades['open_rate'] = trades['open_rate'].apply(lambda x: round(x, 3))
+        trades['close_rate'] = trades['close_rate'].apply(lambda x: round(x, 3))
+        trades['profit_ratio'] = trades['profit_ratio'].apply(lambda x: round(x * 100, 2))
+        trades['profit_abs'] = trades['profit_abs'].apply(lambda x: round(x, 3))
+        trades.insert(0, 'run_id', run_id)
+
+        # Export result as '.csv' file for readable result
+        if output_file is None or output_file == '':
+            output_file = f'{basedir}/user_data/csv_results/{run_id}_trades.csv'
+
+        trades.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8')
+
+
+def main(argv):
+    input_file = ''
+    output_file = ''
+    try:
+        opts, args = getopt.getopt(argv, 'h:i:o:', ['cfile=', 'ifile=', 'ofile='])
+    except getopt.GetoptError:
+        print('ExportCsvBacktestTrades.py -i <input_file> -o <output_file>')
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            print('ExportCsvBacktestTrades.py -i <input_file> -o <output_file>')
+            sys.exit()
+        elif opt in ('-i', '--ifile', '--input_file'):
+            input_file = arg
+        elif opt in ('-o', '--ofile', '--output_file'):
+            output_file = arg
+
+    ExportCsvBacktestTrades(input_file, output_file)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

user_data/mgm_tools/ExportCsvHyperoptTrades.py

@@ -0,0 +1,96 @@
+import getopt
+import os
+import sys
+from pathlib import Path
+
+import rapidjson
+from pandas import DataFrame, json_normalize
+
+
+def ExportCsvHyperoptTrades(config_file, input_file, output_file, epoch_n):
+    # Load the 'mgm-config' file as an object and parse it as a dictionary
+    basedir = os.getcwd()
+
+    # Fetch the latest '.fthypt' filename from last_result
+    if input_file is None or input_file == '':
+        last_result_file = Path(f'{basedir}/user_data/hyperopt_results/.last_result.json')
+        with last_result_file.open('r') as f:
+            data = [rapidjson.loads(line) for line in f]
+        last_rf = json_normalize(data, max_level=1)
+
+        results_file = Path(f'{basedir}/user_data/hyperopt_results/{last_rf["latest_hyperopt"].loc[0]}')
+    else:
+        results_file = Path(input_file)
+
+    run_id = results_file.name.split('.')[0]
+
+    # Open '.fthypt' file and normalize '.json' to dataframe
+    with results_file.open('r') as f:
+        data = [rapidjson.loads(line) for line in f]
+    hyperopt_results = json_normalize(data, max_level=2)
+
+    # Filter results
+    if epoch_n != 0:
+        # Filter chosen epoch only
+        hyperopt_results = hyperopt_results.loc[hyperopt_results['current_epoch'] == epoch_n]
+    else:
+        # Filter out epochs without profit
+        hyperopt_results = hyperopt_results.loc[hyperopt_results['total_profit'] > 0]
+
+    # Define result dataframe columns
+    list_of_columns = ['epoch', 'pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration',
+                       'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open']
+    results_df = DataFrame(columns=list_of_columns)
+
+    # Populate results df with selected values + rearrange format
+    for idx, row in hyperopt_results.iterrows():
+        trades = json_normalize(row['results_metrics.trades'], max_level=1)
+        trades["epoch"] = row["current_epoch"]
+        results_df = results_df.append(trades)
+
+    if len(results_df) > 0:
+        results_df = results_df.loc[:, list_of_columns]
+        results_df['stake_amount'] = results_df['stake_amount'].apply(lambda x: round(x, 3))
+        results_df['amount'] = results_df['amount'].apply(lambda x: round(x, 3))
+        results_df['trade_duration'] = results_df['trade_duration'].apply(lambda x: round(x / 3600, 2))
+        results_df['open_rate'] = results_df['open_rate'].apply(lambda x: round(x, 3))
+        results_df['close_rate'] = results_df['close_rate'].apply(lambda x: round(x, 3))
+        results_df['profit_ratio'] = results_df['profit_ratio'].apply(lambda x: round(x * 100, 2))
+        results_df['profit_abs'] = results_df['profit_abs'].apply(lambda x: round(x, 3))
+        results_df.insert(0, 'run_id', run_id)
+
+        # Export result as '.csv' file for readable result
+        if output_file is None or output_file == '':
+            output_file = f'{basedir}/user_data/csv_results/{run_id}_trades.csv'
+
+        results_df.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8')
+
+
+def main(argv):
+    input_file = ''
+    output_file = ''
+    config_file = f'{os.getcwd()}/user_data/mgm-config.json'
+    epoch = 0
+    try:
+        opts, args = getopt.getopt(argv, 'hc:i:o:n:', ['cfile=', 'ifile=', 'ofile='])
+    except getopt.GetoptError:
+        print('ExportCsvHyperoptTrades.py -c <mgm_config_file> -i <input_file> -o <output_file> -n <epoch>')
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            print('ExportCsvHyperoptTrades.py -c <mgm_config_file> -i <input_file> -o <output_file> -n <epoch>')
+            sys.exit()
+        elif opt in ('-c', '--cfile', '--config_file'):
+            config_file = arg
+        elif opt in ('-i', '--ifile', '--input_file'):
+            input_file = arg
+        elif opt in ('-o', '--ofile', '--output_file'):
+            output_file = arg
+        elif opt in ('-n'):
+            epoch = int(arg)
+
+    ExportCsvHyperoptTrades(config_file, input_file, output_file, epoch)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

user_data/mgm_tools/mgm_hurry/FreqtradeCli.py

@@ -356,9 +356,9 @@ def choose_fthypt_file(self) -> str:
         answers = prompt(questions=questions)
         return answers.get('fthypt_file')
 
-    def parse_fthypt_name(self, fthypt_name: str) -> str:
+    def parse_hyperopt_filename(self, fthypt_name: str) -> str:
         """
-        Helper method to parse the '.fthypt' filename provided/asked by the user
+        Helper method to parse the hyperopt '.fthypt' filename provided/asked by the user
 
         :param fthypt_name: '.fthypt' filename provided by the user
         :return: fthypt_name usable for the code
@@ -370,9 +370,26 @@ def parse_fthypt_name(self, fthypt_name: str) -> str:
         elif os.path.isfile(f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}'):
             return fthypt_name
         else:
-            self.cli_logger.warning(Color.yellow('🤷 Provided fthypt file not exist, please select fthypt file:'))
+            self.cli_logger.warning(Color.yellow('🤷 Provided fthypt file not exist, please select a new file:'))
             return self.choose_fthypt_file()
 
+    def parse_backtest_filename(self, bt_filename: str) -> str:
+        """
+        Helper method to parse the backtest '.json' filename provided/asked by the user
+
+        :param bt_filename: '.json' filename provided by the user
+        :return: bt_filename usable for the code
+        """
+        if bt_filename is True or bt_filename.lower() == 'true':
+            return self.choose_backtest_results_file()
+        elif os.path.isfile(f'{self.basedir}/user_data/backtest_results/{bt_filename}.json'):
+            return f'{bt_filename}.json'
+        elif os.path.isfile(f'{self.basedir}/user_data/backtest_results/{bt_filename}'):
+            return bt_filename
+        else:
+            self.cli_logger.warning(Color.yellow('🤷 Provided backtest file not exist, please select a new file:'))
+            return self.choose_backtest_results_file()
+
     def choose_backtest_results_file(self, choose_results: bool = True) -> str:
         """
         Interactive prompt to choose a 'backtest-result-<timestamp>.json' file.