From ce3a0f1b601007c6cf8b3ea671acec3abd6a1e65 Mon Sep 17 00:00:00 2001 From: justinpolygon <123573436+justinpolygon@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:37:02 -0800 Subject: [PATCH] Fix lint --- .../hunting-anomalies/build-lookup-table.py | 62 +++---- .../hunting-anomalies/gui-lookup-table.py | 162 +++++++++++------- .../hunting-anomalies/query-lookup-table.py | 46 ++--- 3 files changed, 151 insertions(+), 119 deletions(-) diff --git a/examples/tools/hunting-anomalies/build-lookup-table.py b/examples/tools/hunting-anomalies/build-lookup-table.py index c173d58d..a2de6ca8 100644 --- a/examples/tools/hunting-anomalies/build-lookup-table.py +++ b/examples/tools/hunting-anomalies/build-lookup-table.py @@ -5,13 +5,13 @@ import json # Directory containing the daily CSV files -data_dir = './aggregates_day/' +data_dir = "./aggregates_day/" # Initialize a dictionary to hold trades data trades_data = defaultdict(list) # List all CSV files in the directory -files = sorted([f for f in os.listdir(data_dir) if f.endswith('.csv')]) +files = sorted([f for f in os.listdir(data_dir) if f.endswith(".csv")]) print("Starting to process files...") @@ -22,15 +22,13 @@ df = pd.read_csv(file_path) # For each stock, store the date and relevant data for _, row in df.iterrows(): - ticker = row['ticker'] - date = pd.to_datetime(row['window_start'], unit='ns').date() - trades = row['transactions'] - close_price = row['close'] # Ensure 'close' column exists in your CSV - trades_data[ticker].append({ - 'date': date, - 'trades': trades, - 'close_price': close_price - }) + ticker = row["ticker"] + date = pd.to_datetime(row["window_start"], unit="ns").date() + trades = row["transactions"] + close_price = row["close"] # Ensure 'close' column exists in your CSV + trades_data[ticker].append( + {"date": date, "trades": trades, "close_price": close_price} + ) print("Finished processing files.") print("Building lookup table...") @@ -42,38 +40,40 @@ # Convert records to DataFrame df_ticker = pd.DataFrame(records) # Sort records by date - df_ticker.sort_values('date', inplace=True) - df_ticker.set_index('date', inplace=True) + df_ticker.sort_values("date", inplace=True) + df_ticker.set_index("date", inplace=True) # Calculate the percentage change in close_price - df_ticker['price_diff'] = df_ticker['close_price'].pct_change() * 100 # Multiply by 100 for percentage + df_ticker["price_diff"] = ( + df_ticker["close_price"].pct_change() * 100 + ) # Multiply by 100 for percentage # Shift trades to exclude the current day from rolling calculations - df_ticker['trades_shifted'] = df_ticker['trades'].shift(1) + df_ticker["trades_shifted"] = df_ticker["trades"].shift(1) # Calculate rolling average and standard deviation over the previous 5 days - df_ticker['avg_trades'] = df_ticker['trades_shifted'].rolling(window=5).mean() - df_ticker['std_trades'] = df_ticker['trades_shifted'].rolling(window=5).std() + df_ticker["avg_trades"] = df_ticker["trades_shifted"].rolling(window=5).mean() + df_ticker["std_trades"] = df_ticker["trades_shifted"].rolling(window=5).std() # Store the data in the lookup table for date, row in df_ticker.iterrows(): # Convert date to string for JSON serialization - date_str = date.strftime('%Y-%m-%d') + date_str = date.strftime("%Y-%m-%d") # Ensure rolling stats are available - if pd.notnull(row['avg_trades']) and pd.notnull(row['std_trades']): + if pd.notnull(row["avg_trades"]) and pd.notnull(row["std_trades"]): lookup_table[ticker][date_str] = { - 'trades': row['trades'], - 'close_price': row['close_price'], - 'price_diff': row['price_diff'], - 'avg_trades': row['avg_trades'], - 'std_trades': row['std_trades'] + "trades": row["trades"], + "close_price": row["close_price"], + "price_diff": row["price_diff"], + "avg_trades": row["avg_trades"], + "std_trades": row["std_trades"], } else: # Store data without rolling stats if not enough data points lookup_table[ticker][date_str] = { - 'trades': row['trades'], - 'close_price': row['close_price'], - 'price_diff': row['price_diff'], - 'avg_trades': None, - 'std_trades': None + "trades": row["trades"], + "close_price": row["close_price"], + "price_diff": row["price_diff"], + "avg_trades": None, + "std_trades": None, } print("Lookup table built successfully.") @@ -82,13 +82,13 @@ lookup_table = {k: v for k, v in lookup_table.items()} # Save the lookup table to a JSON file -with open('lookup_table.json', 'w') as f: +with open("lookup_table.json", "w") as f: json.dump(lookup_table, f, indent=4) print("Lookup table saved to 'lookup_table.json'.") # Save the lookup table to a file for later use -with open('lookup_table.pkl', 'wb') as f: +with open("lookup_table.pkl", "wb") as f: pickle.dump(lookup_table, f) print("Lookup table saved to 'lookup_table.pkl'.") diff --git a/examples/tools/hunting-anomalies/gui-lookup-table.py b/examples/tools/hunting-anomalies/gui-lookup-table.py index ee2fc43b..df58746c 100644 --- a/examples/tools/hunting-anomalies/gui-lookup-table.py +++ b/examples/tools/hunting-anomalies/gui-lookup-table.py @@ -12,27 +12,28 @@ PORT = 8888 # Load the lookup_table -with open('lookup_table.pkl', 'rb') as f: +with open("lookup_table.pkl", "rb") as f: lookup_table = pickle.load(f) + class handler(http.server.SimpleHTTPRequestHandler): def do_GET(self): # Parse the path and query parameters parsed_path = urlparse(self.path) path = parsed_path.path query_params = parse_qs(parsed_path.query) - - if path == '/': + + if path == "/": # Handle the root path # Get the date parameter if provided - date_param = query_params.get('date', [None])[0] - + date_param = query_params.get("date", [None])[0] + # Get all dates from the lookup table all_dates = set() for ticker_data in lookup_table.values(): all_dates.update(ticker_data.keys()) all_dates = sorted(all_dates) - + # If date is None, get the latest date from the lookup table if date_param is None: if all_dates: @@ -41,109 +42,131 @@ def do_GET(self): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - html_content = '

No data available.

' + html_content = ( + "

No data available.

" + ) self.wfile.write(html_content.encode()) return else: latest_date = date_param - + # Ensure latest_date is in all_dates if latest_date not in all_dates: # Handle the case where the provided date is invalid self.send_response(400) self.send_header("Content-type", "text/html") self.end_headers() - error_html = f'

Error: No data available for date {latest_date}

' + error_html = f"

Error: No data available for date {latest_date}

" self.wfile.write(error_html.encode()) return - + # Now, get the anomalies for the latest_date anomalies = [] for ticker, date_data in lookup_table.items(): if latest_date in date_data: data = date_data[latest_date] - trades = data['trades'] - avg_trades = data['avg_trades'] - std_trades = data['std_trades'] + trades = data["trades"] + avg_trades = data["avg_trades"] + std_trades = data["std_trades"] if ( - avg_trades is not None and - std_trades is not None and - std_trades > 0 + avg_trades is not None + and std_trades is not None + and std_trades > 0 ): z_score = (trades - avg_trades) / std_trades threshold_multiplier = 3 # Adjust as needed if z_score > threshold_multiplier: - anomalies.append({ - 'ticker': ticker, - 'date': latest_date, - 'trades': trades, - 'avg_trades': avg_trades, - 'std_trades': std_trades, - 'z_score': z_score, - 'close_price': data['close_price'], - 'price_diff': data['price_diff'] - }) + anomalies.append( + { + "ticker": ticker, + "date": latest_date, + "trades": trades, + "avg_trades": avg_trades, + "std_trades": std_trades, + "z_score": z_score, + "close_price": data["close_price"], + "price_diff": data["price_diff"], + } + ) # Sort anomalies by trades in descending order - anomalies.sort(key=lambda x: x['trades'], reverse=True) + anomalies.sort(key=lambda x: x["trades"], reverse=True) # Generate the HTML to display the anomalies self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() # Build the HTML content - html_content = 'Anomalies for {}'.format(latest_date) - html_content += '

Anomalies for {}

'.format(latest_date) + html_content = 'Anomalies for {}'.format( + latest_date + ) + html_content += '

Anomalies for {}

'.format( + latest_date + ) # Add navigation links (prev and next dates) current_index = all_dates.index(latest_date) prev_date = all_dates[current_index - 1] if current_index > 0 else None - next_date = all_dates[current_index + 1] if current_index < len(all_dates) - 1 else None - html_content += '

' + next_date = ( + all_dates[current_index + 1] + if current_index < len(all_dates) - 1 + else None + ) + html_content += "

" if prev_date: - html_content += 'Previous Date '.format(prev_date) + html_content += 'Previous Date '.format( + prev_date + ) if next_date: html_content += 'Next Date '.format(next_date) - html_content += '

' + html_content += "

" # Display the anomalies in a table - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' - html_content += '' + html_content += ( + '
TickerTradesAvg TradesStd DevZ-scoreClose PricePrice DiffChart
' + ) + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" + html_content += "" for anomaly in anomalies: - html_content += '' - html_content += ''.format(anomaly['ticker']) - html_content += ''.format(anomaly['trades']) - html_content += ''.format(anomaly['avg_trades']) - html_content += ''.format(anomaly['std_trades']) - html_content += ''.format(anomaly['z_score']) - html_content += ''.format(anomaly['close_price']) - html_content += ''.format(anomaly['price_diff']) + html_content += "" + html_content += "".format(anomaly["ticker"]) + html_content += "".format(anomaly["trades"]) + html_content += "".format(anomaly["avg_trades"]) + html_content += "".format(anomaly["std_trades"]) + html_content += "".format(anomaly["z_score"]) + html_content += "".format(anomaly["close_price"]) + html_content += "".format(anomaly["price_diff"]) # Add a link to the chart - html_content += ''.format(anomaly['ticker'], latest_date) - html_content += '' + html_content += ( + ''.format( + anomaly["ticker"], latest_date + ) + ) + html_content += "" html_content += '
TickerTradesAvg TradesStd DevZ-scoreClose PricePrice DiffChart
{}{}{:.2f}{:.2f}{:.2f}{:.2f}{:.2f}
{}{}{:.2f}{:.2f}{:.2f}{:.2f}{:.2f}View Chart
View Chart
' - html_content += '
' + html_content += "
" self.wfile.write(html_content.encode()) - elif path == '/chart': + elif path == "/chart": # Handle the chart page # Get 'ticker' and 'date' from query parameters - ticker = query_params.get('ticker', [None])[0] - date = query_params.get('date', [None])[0] + ticker = query_params.get("ticker", [None])[0] + date = query_params.get("date", [None])[0] if ticker is None or date is None: # Return an error page self.send_response(400) self.send_header("Content-type", "text/html") self.end_headers() - error_html = '

Error: Missing ticker or date parameter

' + error_html = "

Error: Missing ticker or date parameter

" self.wfile.write(error_html.encode()) else: # Fetch minute aggregates for the ticker and date - client = RESTClient(trace=True) # POLYGON_API_KEY environment variable is used + client = RESTClient( + trace=True + ) # POLYGON_API_KEY environment variable is used try: aggs = [] date_from = date @@ -166,7 +189,7 @@ def do_GET(self): agg.open, agg.high, agg.low, - agg.close + agg.close, ] data.append(new_record) # Generate the HTML for the chart page @@ -239,10 +262,15 @@ def do_GET(self): - """ % (json.dumps(data), ticker, date, ticker) + """ % ( + json.dumps(data), + ticker, + date, + ticker, + ) self.send_response(200) self.send_header("Content-type", "text/html") - self.send_header('Access-Control-Allow-Origin', '*') + self.send_header("Access-Control-Allow-Origin", "*") self.end_headers() self.wfile.write(chart_html.encode()) except Exception as e: @@ -250,12 +278,15 @@ def do_GET(self): self.send_response(500) self.send_header("Content-type", "text/html") self.end_headers() - error_html = '

Error fetching data: {}

'.format(str(e)) + error_html = "

Error fetching data: {}

".format( + str(e) + ) self.wfile.write(error_html.encode()) else: # Serve files from the current directory super().do_GET() + def run_server(): with socketserver.TCPServer(("", PORT), handler) as httpd: print("serving at port", PORT) @@ -266,5 +297,6 @@ def run_server(): httpd.shutdown() httpd.server_close() -if __name__ == '__main__': + +if __name__ == "__main__": run_server() diff --git a/examples/tools/hunting-anomalies/query-lookup-table.py b/examples/tools/hunting-anomalies/query-lookup-table.py index 4037a031..38bb86cf 100644 --- a/examples/tools/hunting-anomalies/query-lookup-table.py +++ b/examples/tools/hunting-anomalies/query-lookup-table.py @@ -2,12 +2,12 @@ import argparse # Parse command-line arguments -parser = argparse.ArgumentParser(description='Anomaly Detection Script') -parser.add_argument('date', type=str, help='Target date in YYYY-MM-DD format') +parser = argparse.ArgumentParser(description="Anomaly Detection Script") +parser.add_argument("date", type=str, help="Target date in YYYY-MM-DD format") args = parser.parse_args() # Load the lookup_table -with open('lookup_table.pkl', 'rb') as f: +with open("lookup_table.pkl", "rb") as f: lookup_table = pickle.load(f) # Threshold for considering an anomaly (e.g., 3 standard deviations) @@ -23,33 +23,33 @@ for ticker, date_data in lookup_table.items(): if target_date_str in date_data: data = date_data[target_date_str] - trades = data['trades'] - avg_trades = data['avg_trades'] - std_trades = data['std_trades'] - if ( - avg_trades is not None and - std_trades is not None and - std_trades > 0 - ): + trades = data["trades"] + avg_trades = data["avg_trades"] + std_trades = data["std_trades"] + if avg_trades is not None and std_trades is not None and std_trades > 0: z_score = (trades - avg_trades) / std_trades if z_score > threshold_multiplier: - anomalies.append({ - 'ticker': ticker, - 'date': target_date_str, - 'trades': trades, - 'avg_trades': avg_trades, - 'std_trades': std_trades, - 'z_score': z_score, - 'close_price': data['close_price'], - 'price_diff': data['price_diff'] - }) + anomalies.append( + { + "ticker": ticker, + "date": target_date_str, + "trades": trades, + "avg_trades": avg_trades, + "std_trades": std_trades, + "z_score": z_score, + "close_price": data["close_price"], + "price_diff": data["price_diff"], + } + ) # Sort anomalies by trades in descending order -anomalies.sort(key=lambda x: x['trades'], reverse=True) +anomalies.sort(key=lambda x: x["trades"], reverse=True) # Print the anomalies with aligned columns print(f"\nAnomalies Found for {target_date_str}:\n") -print(f"{'Ticker':<10}{'Trades':>10}{'Avg Trades':>15}{'Std Dev':>10}{'Z-score':>10}{'Close Price':>12}{'Price Diff':>12}") +print( + f"{'Ticker':<10}{'Trades':>10}{'Avg Trades':>15}{'Std Dev':>10}{'Z-score':>10}{'Close Price':>12}{'Price Diff':>12}" +) print("-" * 91) for anomaly in anomalies: print(