daytrade.py

# -*- coding: utf-8 -*-
"""daytrade.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1aravbFmBdESeZuvC9kwktRwUDOPvGgtt

Small python script that plots the difference between close and open prices of s&p stocks
"""

import os
import random
from time import sleep
import pprint
from dotenv import load_dotenv
from datetime import datetime, timezone
import json

import pandas as pd
import matplotlib.pyplot as plt
import plotly.graph_objects as go
import plotly.express as px
import seaborn as sns
import numpy as np

import yfinance as yahooFinance
import yahoo_fin.stock_info as si
from pytickersymbols import PyTickerSymbols
import robin_stocks.robinhood as r

tzname = 'America/New_York'

## Hyper parameters
history = "2y"
buy_trigger = 3                           # times standard deviation
sell_trigger = 1                          # times the avg cost of the security to grow before we sell
security_age = 15                         # number of days to hold the security before we cut the losses
lockin_gains_factor = 1000                # times the orignal amount to grow before we lockin the gains.
mean_type = "+ve"                         # only consider stocks with +ve mean of ND. These stocks have been growing over the period of time
max_stocks_to_buy = 5                     # number of stocks to buy at buy trigger. We can change this value to be 
                                          # more adaptive based on market cap of the security and other parameters.
backtest_iterations = 1                   # number of backtests to run
backtest_days = random.sample(range(100, 700), backtest_iterations) # starting days for back testing
backtest_days.sort()
prefer_beta = True
above_beta_mean = False
stop_falling_knife = False
index_stocks = "SP500"                    # Other legal names are NASDAQ, DOW, SP500

starting_balance = 10000                  # seed money to start investing
current_account = original_balance = starting_balance
portfolio = {}
stks = {}

# Display test results. Debugging Tools
print_final_portfolio = False             # Prints the portfolio list at the end of each backtest iteration
plot_every_test_graph = True              # Prints the model performance during the back end against s&P500
plot_summary_graph = True                 # prints the summary graph
dump_all_trades = True                   # dumps all sells at the end of the trade. Use it sparingly, with iteration set to 1

# Deploy the initial amount gradually
tranche_pct = 100                         # of amount to deploy
tranche_period = 1                        # calender days between deployment

tests = [
    {
        "Name": "Test 1",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 15,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 2",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 15,
        "lockin_gains_factor": 1.1,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 3",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 1000,
        "lockin_gains_factor": 1.1,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 4",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 1000,
        "lockin_gains_factor": 1.1,
        "prefer_beta": False,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 5",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 90,
        "lockin_gains_factor": 1.1,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 6",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 90,
        "lockin_gains_factor": 1.1,
        "prefer_beta": False,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
    {
        "Name": "Test 7",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 90,
        "lockin_gains_factor": 1.1,
        "prefer_beta": True,
        "mean_type": "",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 8",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 90,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": True,
        "stop_falling_knife": False,
     },
      {
        "Name": "Test 9",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 20,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": True,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 10",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 20,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": False,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 11",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 90,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": True,
        "stop_falling_knife": True,
     },
     {
        "Name": "Test 12",
        "buy_trigger": 2,
        "sell_trigger": 1,
        "security_age": 20,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "+ve",
        "above_beta_mean": True,
        "stop_falling_knife": False,
     },
     {
        "Name": "Test 13 (DayTrade)",
        "buy_trigger": 6,
        "sell_trigger": 1,
        "security_age": 1,
        "lockin_gains_factor": 1000,
        "prefer_beta": True,
        "mean_type": "",
        "above_beta_mean": True,
        "stop_falling_knife": False,
     },
]

tests_to_run = [13]

pp = pprint.PrettyPrinter(indent=4)

stock_indexes = {"SP500": '^GSPC', 'NASDAQ': '^IXIC', 'DOW': '^DJI'}

assert index_stocks in stock_indexes
indices = list(stock_indexes.values())

if index_stocks == "SP500":
  stocks = si.tickers_sp500()
  if 'FRC' in stocks:
      stocks.remove('FRC')
elif index_stocks == "NASDAQ":
  stock_data = PyTickerSymbols()
  stocks = stock_data.get_nasdaq_100_nyc_yahoo_tickers()
elif index_stocks == "DOW":
  stocks = si.tickers_dow()

stocks_ts = pd.DataFrame()
delta1 = pd.DataFrame()
delta2 = pd.DataFrame()
indices_ts = pd.DataFrame()

for idx in indices:
    data = yahooFinance.Ticker(idx).history(period=history)
    d = data.copy()[['High', 'Low', 'Open', "Close"]]
    indices_ts[idx+"_High"] = d['High']
    indices_ts[idx+"_Low"] = d['Low']
    indices_ts[idx+"_Open"] = d["Open"]
    indices_ts[idx+"_Close"] = d["Close"]

for stock in stocks:
    data = yahooFinance.Ticker(stock).history(period=history)
    d = data.copy()[['High', 'Low', "Open", "Close"]]
    stocks_ts[stock+"_High"] = d['High']
    stocks_ts[stock+"_Low"] = d['Low']
    stocks_ts[stock+"_Open"] = d["Open"]
    stocks_ts[stock+"_Close"] = d["Close"]
    m = (d['Open'] + d['Close']) / 2
    delta1[stock] = (m - d['Low']) * 100/m
    delta2[stock] = (d['High'] - m) * 100/m

    stocks_ts = stocks_ts.copy()
    delta1 = delta1.copy()
    delta2 = delta2.copy()

indices_ts.index = indices_ts.index.tz_convert(tzname)
stocks_ts.index = stocks_ts.index.tz_convert(tzname)

for stock in stocks[:1]:
    fig = go.Figure(data=[go.Candlestick(x=d.index,
                open=d['High'],
                high=d['Low'],
                low=d['High'],
                close=d['Low'])])
    fig.update_layout(
      title=stock,
      yaxis_title=stock +' Stock',
      shapes = [dict(
          x0='2022-12-09', x1='2022-12-09', y0=0, y1=1, xref='x', yref='paper',
          line_width=2)],
      annotations=[dict(
          x='2022-12-09', y=0.05, xref='x', yref='paper',
          showarrow=False, xanchor='left', text='Increase Period Begins')]
    )

    #fig.show()

    delta1['Date'] = delta1.index
    fig = px.line(delta1, x="Date", y=stock, title="Lows Price change", markers=True) 
    #fig.show()

    x = pd.Series(delta1[stock])
    ax = x.plot.kde(figsize=(30,8))

    delta2['Date'] = delta2.index
    fig = px.line(delta2, x="Date", y=stock, title="High Price change", markers=True) 
    #fig.show()

    x = pd.Series(delta2[stock])
    ax = x.plot.kde(figsize=(30,8))

    #sns.set(rc={'figure.figsize':(25.7,8.27)})
    #sns.lineplot(x="Date",y=stock, data=stocks_spread, markers=True, err_style="bars")
    #sns.despine();

#print(delta1.describe())
std_delta1 = delta1.describe().loc['std']
mean_delta1 = delta1.describe().loc['mean']
std_delta1 = pd.DataFrame.from_dict(std_delta1)
std_delta1.columns = ['std_delta1',]

mean = pd.DataFrame.from_dict(mean_delta1)
mean.columns = ['mean',]

fig = px.line(std_delta1, title="Daily Low Price", markers=True)
#fig.show()

#print(delta1.describe())
std_delta2 = delta2.describe().loc['std']
mean_delta2 = delta2.describe().loc['mean']
std_delta2 = pd.DataFrame.from_dict(std_delta2)
std_delta2.columns = ['std_delta2',]

mean = pd.DataFrame.from_dict(mean_delta2)
mean.columns = ['mean',]

fig = px.line(std_delta2, title="Daily High Price", markers=True)
#fig.show()

# Determine what stocks are in buy zone and what stocks are in sell zone
# Start date of the investing

def getsellbuy(portfolio, security_profit, security_loss):
  latest = {}
  index = stks['A'].index[0]
  for s in stocks:
      latest[s] = (stks[s].loc[index]["Close"] - stocks_ts.loc[index][s+"_Open"])* 100 /stocks_ts.loc[index][s+"_Open"]

  latest = pd.Series(latest)
  pd.DataFrame.from_dict(latest)
  latest_diff = pd.DataFrame.from_dict(latest)
  latest_diff.columns = ['diff',]

  latest_diff['std_delta1'] = std_delta1['std_delta1']
  latest_diff['std_delta2'] = std_delta2['std_delta2']

  # Find stocks that are in buy range
  # also lets avoid outliers too. If the fall is too steep, we don't want to consider it
  latest_diff['buy'] = np.where(((latest_diff['diff'] < 0) &
                                 (latest_diff['diff'] <  -1 * buy_trigger * std_delta1['std_delta1']) &
                                 (latest_diff['diff'] > -(buy_trigger + 2) * std_delta1['std_delta1'])), True, False)

  # Find stocks that are in sell range
  latest_diff['sell'] = False #np.where((latest_diff['diff'] >= std['std']), True, False)
  profit = loss = 0
  for s in stocks:
    if s in portfolio:
      latest_diff.at[s, 'buy'] = False
      t = index - pd.Timestamp(portfolio[s]['date']).tz_convert(tzname)
      avg = sum(portfolio[s]['costs'])/len(portfolio[s]['costs'])
      if avg + sell_trigger * std_delta1['std_delta1'][s] * avg / 100 <= stks[s].loc[index, "Close"]:
        # If the avg cost of the security has grown more than std
        profit = len(portfolio[s]['costs']) * stks[s].loc[index, "Close"] - sum(portfolio[s]['costs'])
        if dump_all_trades:
          print("Selling %s at profit. closing price %f. profit %f days %d" %
                (s, stks[s].loc[index, "Close"], profit, t.days), portfolio[s])
        security_profit.loc[len(security_profit.index)] = {'days':t.days, 'profit':profit}
        latest_diff.at[s, 'sell'] = True
      elif t.days > security_age or \
             latest_diff['diff'][s] < -(buy_trigger + 2) * std_delta1['std_delta1'][s]:

        # if the security has aged for certain days, cut the losses
        loss = len(portfolio[s]['costs']) * stks[s].loc[index, "Close"] - sum(portfolio[s]['costs'])
        if dump_all_trades:
          print("Dumping %s because of age. closing price %f. loss %f" %
                (s, stks[s].loc[index, "Close"], loss), portfolio[s])
        if loss > 0:
          security_profit.loc[len(security_profit.index)] = {'days':t.days, 'profit':loss}
        else:
          security_loss.loc[len(security_loss.index)] = {'loss':loss}
        latest_diff.at[s, 'sell'] = True
      
  
  # Print which stocks are buy and which are sell
  #print("Stock to Buy on " + str(index))
  #print(latest_diff[latest_diff['buy']])
  #print()
  #print("Stocks to Sell on " + str(index))
  #print(latest_diff[latest_diff['sell']])

  sell_stocks = latest_diff[latest_diff['sell']]
  buy_stocks = latest_diff[latest_diff['buy']]
  buy_stocks = buy_stocks.sort_values('std_delta2', ascending=False)
  return sell_stocks, buy_stocks

# We assume that we invest $100 in each stock that is in the buy zone
# and sell all stocks in the sell zone

# We will start back testing from year back. We can always change the starting point and tune the model.

def calculate_networth():
  networth = current_account
  for stock, value in portfolio.items():
    networth += value['shares'] * stks[stock].loc[stks[stock].index[0]]["Close"]
  return networth

def do_one_backtesting():
  global portfolio
  global original_balance
  global current_account

  stocks_bought = pd.DataFrame(columns=stocks)
  stocks_sold = pd.DataFrame(columns=stocks)
  stocks_profit = pd.DataFrame(columns=['days', 'profit'])
  stocks_loss = pd.DataFrame(columns=['loss'])

  original_balance = starting_balance
  portfolio = {}
  capital_to_be_deployed = original_balance

  idx = 0
  price_movement = []
  cash_inhand = []

  if os.path.exists("holdings.json"):
      with open("holdings.json", "r") as f:
          portfolio = json.load(f)

  if os.path.exists("cash_balance.json"):
      with open("cash_balance.json", "r") as f:
          c = {}
          c = json.load(f)
          current_account = c['cash']  

  hrs = r.get_market_today_hours('XNYS')
  #while pd.to_datetime(datetime.now(timezone.utc)) < pd.to_datetime(hrs['closes_at']) and \
        #pd.to_datetime(datetime.now(timezone.utc)) > pd.to_datetime(hrs['opens_at']):
  while True:
      # Load test data
      try:
          for i in stocks:
              try:
                  data = yahooFinance.Ticker(i).history("1d")
                  stks[i] = pd.DataFrame()
                  d = data.copy()[['High', 'Low', "Open", "Close"]]
                  stks[i]['High'] = d['High']
                  stks[i]['Low'] = d['Low']
                  stks[i]['Open'] = d['Open']
                  stks[i]['Close'] = d['Close']
                  stks[i].index = stks[i].index.tz_convert(tzname)
              except:
                  pass

          backtest_start_date = stks[i].index[0]
          sell, buy = getsellbuy(portfolio, stocks_profit, stocks_loss)
      except:
          # yahoo api changes the index from utc or localized value between market open and close
          # in that case we may get an exception. Wait for 60 seconds and try again
          sleep(60)
          continue
   
      # process the stocks that are marked sell
      for st in sell.iterrows():
        stock = st[0].split('_')[0]
        if stock in portfolio:
          current_account += portfolio[stock]['shares'] * stks[stock].loc[backtest_start_date]["Close"]
          stocks_sold.loc[backtest_start_date, stock] = portfolio[stock]['shares'] * stks[stock].loc[backtest_start_date]["Close"]
          portfolio.pop(stock)

      # buy stocks that are marked by. We are buying max_stocks_to_buy number of stocks
      # TODO: The number of stocks to be must be adaptive. Will come up with some
      # algorithm based on:
      # 1. Market capitalization
      # 2. Beta
      # and other criteria
      # The goal is to put the money to work
      for st in buy.iterrows():
        stock = st[0].split('_')[0]
        try:
            earning_calls_date=pd.to_datetime(r.get_earnings(st)[-2]['call']['datetime']).tz_convert('America/New_York')
            delta = pd.Timestamp.utcnow() - earning_calls_date
            if delta.days > 0 and delta.days < 4:
                # skip the stock for few days after earnings call
                continue
        except:
            pass
        if current_account > max_stocks_to_buy * stks[stock].loc[backtest_start_date]["Close"]:
          current_account -= max_stocks_to_buy * stks[stock].loc[backtest_start_date]["Close"]
          if not stock in portfolio:
              assert portfolio.get(stock, None) == None
              portfolio[stock] = {'shares': 0, 'costs':[], 'date': str(backtest_start_date)}
          portfolio[stock]['shares'] += max_stocks_to_buy
          stocks_bought.loc[backtest_start_date, stock] = max_stocks_to_buy * stks[stock].loc[backtest_start_date]["Close"]
          for i in range(max_stocks_to_buy):
            portfolio[stock]['costs'].append(stks[stock].loc[backtest_start_date]["Close"])

      # lock in the gains after 10% increase of networth
      nw = calculate_networth()
      price_movement.append(nw)
      cash_inhand.append(current_account)
      if nw > original_balance * lockin_gains_factor:
          #print(backtest_start_date, calculate_networth(), current_account, portfolio)
          for stock, value in portfolio.items():
            current_account += value['shares'] * stks[stock].loc[backtest_start_date]["Close"]
            stocks_sold.loc[backtest_start_date, stock] = stks[stock].loc[backtest_start_date]["Close"]
          portfolio = {}
          original_balance = current_account
          #print(current_account, portfolio)

      with open("holdings.json", "w") as f:
          json.dump(portfolio, f, indent=4, sort_keys=True, default=str)

      with open("cash_balance.json", "w") as f:
          json.dump({"cash": current_account}, f)
      print("=======================================================")
      print(datetime.now())
      print("Cash: %d" % current_account)
      print("Holdings:")
      networth = current_account
      total_gain = 0;
      for s, v in portfolio.items():
          networth += v['shares'] * stks[s]["Close"][0]
          total_gain += v['shares'] * (stks[s]["Close"][0] - v['costs'][0])
          gain = v['shares'] * (stks[s]["Close"][0] - v['costs'][0])
          print("%s(%d)\t%s\t%f\t%f\t%f" % (s, v['shares'], str(v['date']), v['costs'][0], stks[s]["Close"][0], gain))
      print("Completed one iteration with unrealized total gain: %f and networth: %f" % (total_gain, networth))
      print("=======================================================")
      sleep(15 * 60)

  #if dump_all_trades:
    #print(stocks_profit)
    #print(stocks_loss)
  print("Profit Distribution")
  print("===================")
  print(stocks_profit.describe())
  print()
  print("Loss Distribution")
  print("=================")
  print(stocks_loss.describe())
  print("Total Profit %f and Total Loss %f" % (stocks_profit['profit'].sum(), stocks_loss['loss'].sum()))
  return calculate_networth(), current_account, price_movement, cash_inhand

def run_backtest():
  nws = {}
  sp_ret = {}
  hrs = r.get_market_today_hours('XNYS')
  while True:
    if hrs['is_open'] == False:
        sleep(15 * 60)
        hrs = r.get_market_today_hours('XNYS')
        continue
    tradingdays_togoback = 1
    networth, current, pm, cih = do_one_backtesting()
    # normalize s&p500 for starting balance
    idx = pd.Timestamp("%d-%d-%d" %(stks['A'].index[0].year, stks['A'].index[0].month, stks['A'].index[0].day))
    #idx_list = list(indices_ts.loc[indices_ts.index[-tradingdays_togoback:], "^GSPC_Close"] * starting_balance/indices_ts.loc[indices_ts.index[-tradingdays_togoback], "^GSPC_Close"])
    idx_list = list(indices_ts.loc[idx:]["^GSPC_Close"] * starting_balance/indices_ts.loc[idx, "^GSPC_Close"])
    pm_pct = (pm[-1]-starting_balance)*100/starting_balance
    d = pd.DataFrame({'Portfolio_Performance':pm, "Cash_In_Hand": cih})
    idx_pct = (idx_list[-1] - idx_list[0])*100/idx_list[0]
    print("Total networth: %d (Cash %d) after going back %d days (%s)" % (networth, current_account, 15, idx))
    print("Model (%f)%% vs S&P Performance (%f)%%" % (pm_pct, idx_pct))
    if print_final_portfolio:
      pp.pprint(portfolio)
    nws[i] = networth
    sp_ret[i] = idx_list[-1]
    if plot_every_test_graph:
      fig = px.line(d, title="Model (%f)%% vs %s Performance (%f)%% starting at %s" % (pm_pct, index_stocks, idx_pct, idx), markers=True)
      fig.show()

  if plot_summary_graph:
    plt.bar(nws.keys(), nws.values(), color="green", label='Model')
    plt.bar(sp_ret.keys(), sp_ret.values(), color="blue", label=index_stocks)
    plt.title("Networth at the end of each iteration")
    plt.xlabel('Days') 
    plt.ylabel('Networth')
    plt.legend()
    plt.show()
    print(sum(nws.values())/len(nws))
    idx_list = []
    for i in backtest_days:
      idx_list.append(indices_ts.index[-i])
    summary = pd.DataFrame(data={"Model":nws.values(), index_stocks:sp_ret.values()}, 
                           index=list(idx_list))
    fig = px.bar(summary, title="Model vs %s Backtesting Results. Seed money $%f" % (index_stocks, starting_balance), barmode="group")
    fig.show()

login = r.login(os.environ['robin_username'], os.environ['robin_password'], store_session=True, by_sms=True)

for i in tests_to_run:
  t = tests[i-1]
  buy_trigger = t['buy_trigger']    # times standard deviation
  sell_trigger = t['sell_trigger']   # times the avg cost of the security to grow before we sell
  security_age = t['security_age']  # number of days to hold the security before we cut the losses
  lockin_gains_factor = t['lockin_gains_factor'] # times the orignal amount to grow before we lockin the gains.
  prefer_beta = t['prefer_beta']
  mean_type = t["mean_type"]
  stop_falling_knife = t["stop_falling_knife"]
  above_beta_mean = t["above_beta_mean"]
  print("Test Parameters for " + t['Name'])
  print("=================================")
  pp.pprint(t)
  run_backtest()
  print()
  print("==================================\n")