#Setting up Selenium.
import os
import random
import pandas as pd
import keyboard, time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from datetime import datetime, timedelta

service = Service()
options = Options()

options.add_argument("--disable-blink-features")
options.add_argument("--disable-blink-features=AutomationControlled")
driver = webdriver.Chrome(service=service, options=options)

#Opening a browser and pausing to log in.
ticker = "AAPL"
reference_url = "https://stockanalysis.com/stocks/" + ticker + "/financials/"
driver.get(reference_url)

#Pause here to log in...
print("Slect FMP from dropdown menu if doing financials")
input("Press enter to continue.")

nasdaq_list = pd.read_csv('screener-stocks.csv')
# print(nasdaq_list)
ticker_list = nasdaq_list['Symbol'].tolist()

def scrape_financials(url):
    driver.get(url)
    time.sleep(15 + random.random() * 10)
    download_button = driver.find_element(By.XPATH, "//*[contains(text(), 'Download')]")
    download_button.click()
    time.sleep(2 + random.random() * 2)
    download_button = driver.find_element(By.XPATH, "//*[contains(text(), 'Download to CSV')]")
    download_button.click()
    time.sleep(2 + random.random() * 2)

failure_list = []

for ticker in ticker_list: 
    print("Processing " + ticker)
    time.sleep(20)
    try:
        income_url = "https://stockanalysis.com/stocks/" + ticker.lower() + "/financials/?p=quarterly"
        # income_url = "https://stockanalysis.com/quote/otc/" + ticker.lower() + "/financials/?p=quarterly"
        scrape_financials(income_url)
    except Exception as e:
        print("Income statement not found!")
        failure_list.append(ticker)
        print(e)
    else:
        try:
            balance_sheet_url = "https://stockanalysis.com/stocks/" + ticker.lower() + "/financials/balance-sheet/?p=quarterly"
            # balance_sheet_url = "https://stockanalysis.com/quote/otc/" + ticker.lower() + "/financials/balance-sheet/?p=quarterly"
            scrape_financials(balance_sheet_url)
        except Exception as e:
            print("Balance sheet not found!")
            failure_list.append(ticker)
            print(e)
        else:
            try:
                cash_flow_url = "https://stockanalysis.com/stocks/" + ticker.lower() + "/financials/cash-flow-statement/?p=quarterly"
                # cash_flow_url = "https://stockanalysis.com/quote/otc/" + ticker.lower() + "/financials/cash-flow-statement/?p=quarterly"
                scrape_financials(cash_flow_url)
                success_list.append(ticker)
            except Exception as e:
                print("Cash flow not found!")
                failure_list.append(ticker)
                print(e)