!pip install refinitiv.dataplatform
        
        Copy
    

    	
            !pip install sklearn
        
        Copy
    

    	
            !pip install xgboost
        
        Copy
    

    	
            !pip install transformers
        
        Copy
    

    	
            !pip install openpyxl
        
        Copy
    

    	
            !pip install plotly
        
        Copy
    

    	
            
import configparser
import datetime
import pandas as pd
import numpy as np
from numpy import mean
from numpy import std
import os
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
 
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
import xgboost as xgb
from xgboost import XGBClassifier
from sklearn.decomposition import PCA
from transformers import BertTokenizer, BertForSequenceClassification
import torch
 
from sklearn import metrics
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_validate
from sklearn.model_selection import RepeatedStratifiedKFold
from scipy.stats import norm
 
import warnings
warnings.filterwarnings("ignore")
 
# import eikon package and read app key from a text file
import refinitiv.dataplatform as rdp
app_key = open("app_key.txt","r").read()
 
rdp.open_desktop_session(app_key)

        
        Copy
    

    	
            
cfg = configparser.ConfigParser()
cfg.read('rdp.cfg',encoding='utf-8')
 
APP_KEY = cfg['RDP']['app_key']
RDP_LOGIN = cfg['RDP']['rdp_login']
RDP_PASSWORD = cfg['RDP']['rdp_password']
 
session = rdp.open_platform_session(
    APP_KEY,
    rdp.GrantPassword(
        username = RDP_LOGIN,
        password = RDP_PASSWORD
    )
)
session.open()

        
        Copy
    

    	
            
#build search query with the specified filters
MnA = rdp.search(
    view = rdp.SearchViews.DealsMergersAndAcquisitions,
    
    #specify filtering properties
    filter = "((AcquirerCompanyName ne 'Creditors' and AcquirerCompanyName ne 'Shareholder') and (TargetCountry eq 'US' or TargetCountry eq 'UK')"
    + "and TransactionValueIncludingNetDebtOfTarget ge 100 and TargetPublicStatus eq 'Public')"
    + "and (TransactionStatus eq 'Completed' or TransactionStatus eq 'Pending' or TransactionStatus eq 'Withdrawn')"
    + "and (FormOfTransactionName xeq 'Merger' or FormOfTransactionName xeq 'Acquisition') and (TransactionAnnouncementDate le 2021-11-15 and TransactionAnnouncementDate ge 2020-09-15)",
    
    #select only the required fields and order them based on announcement date
    #then specify number of items to be 10000, default value is 100
    select = 'TransactionAnnouncementDate, TargetCompanyName, TargetRIC',
    order_by = 'TransactionAnnouncementDate desc',
    top = 10000)
 
#remove companies which doesn't have RIC
MnA = MnA.dropna(subset = ['TargetRIC']).reset_index(drop = True)
 
print(f'Number of M&A deals for the specified period is {len(MnA)}')
MnA.head()

        
        Copy
    

    	
            
#create list of RICs
rics = MnA['TargetRIC'].to_list()
rics = [rics[i][0] for i in range(len(rics))]
 
#create list of announcement dates including one for 30 days prior to the announcement
dates = pd.DataFrame(MnA['TransactionAnnouncementDate'])
dates.insert(loc = len(dates.columns), column = 'AD-30', value = pd.to_datetime(dates['TransactionAnnouncementDate']) - datetime.timedelta(30))
dates.insert(loc = len(dates.columns), column = 'rics', value = rics)
 
dates_30 = dates['AD-30'].dt.strftime('%Y-%m-%d').to_list()

        
        Copy
    

    	
            
#specify variables
fields = ["TR.TRBCIndustry", "TR.F.MktCap", "TR.F.ReturnAvgTotEqPctTTM",
           "TR.F.IncAftTaxMargPctTTM", "TR.F.GrossProfMarg","TR.F.NetIncAfterMinIntr","TR.F.TotCap","TR.F.OpMargPctTTM",
           "TR.F.ReturnCapEmployedPctTTM","TR.F.NetCashFlowOp", "TR.F.LeveredFOCF", "TR.F.TotRevenue",
          "TR.F.TotRevenue(SDate = -1Y)","TR.F.TotRevenue(SDate = -2Y)","TR.F.TotRevenue(SDate = -3Y)", "TR.F.TotAssets","TR.F.CurrRatio","TR.F.WkgCaptoTotAssets",
           "TR.PriceToBVPerShare","TR.PriceToSalesPerShare",'TR.F.EBITDA',"TR.EV","TR.EVToSales","TR.F.TotShHoldEq",
           "TR.F.DebtTot","TR.F.NetDebttoTotCap","TR.TotalDebtToEV","TR.F.NetDebtPerShr", "TR.F.CashCashEquivTot"]
 
#create empty lists and dataframe to store requested values
target_data = pd.DataFrame()
error_target = []
error_target_dates = []
 
for i in range(len(rics)):
    try:
        #get data for fields as of the specified date 
        df, err = rdp.legacy.get_data(rics[i], fields = fields , parameters = {'SDate': dates_30[i]})
        #add anoouncement date to the resulting dataframe
        df.insert(loc = 1, column = 'AD', value = pd.to_datetime(dates['TransactionAnnouncementDate'][i]))
        #append company data to the main dataframe
        target_data = pd.concat([target_data, df], ignore_index = True, axis = 0)
    #if error is returned, store ric and request date into a separate list
    except:
        error_target.append(rics[i])
        error_target_dates.append(dates_30[i])
        continue

        
        Copy
    

    	
            
#run the data request code above for the companies in the error list
for i in range(len(error_target)):
    df, err =rdp.legacy.get_data(error_target[i], fields = fields , parameters = {'SDate': error_target_dates[i]})
    target_data = pd.concat([target_data, df], ignore_index = True, axis = 0)

        
        Copy
    

    	
            
#convert announcement date into date format  
target_data['AD'] = target_data['AD'].apply(lambda a: pd.to_datetime(a).date()) 
 
#drop some of the field as specified in my previous article
target_data = target_data.drop(columns = ['TRBC Industry Name', 'Market Capitalization', 'Income after Tax Margin - %, TTM', 'Return on Capital Employed - %, TTM', 
                                    'Net Cash Flow from Operating Activities', 'Working Capital to Total Assets', 'Enterprise Value To Sales (Daily Time Series Ratio)',
                                    'Net Debt to Total Capital', 'Total Debt To Enterprise Value (Daily Time Series Ratio)','Return on Average Total Equity - %, TTM','Total Assets'])
 
#remove NAs, insert a column for the date on 30 days prior to the M&A announcement 
target_data.dropna(inplace = True)
target_data.insert(loc = 1, column = 'AD-30', value = target_data['AD'] - datetime.timedelta(30))
target_data.reset_index(drop = True, inplace = True)
target_data.head()

        
        Copy
    

    	
            
target_data.to_excel("mergerdata/target_data.xlsx")
#read target data from the stored excel file
target_data = pd.read_excel("mergerdata/target_data.xlsx").drop(columns = ['Unnamed: 0'])

        
        Copy
    

    	
            
def peers(RIC, date):
    '''
    Get peer group for an individual RIC along with required variables for the models
    
    Dependencies
    ------------
    Python library 'refinitiv.dataplatform' version 1.0.0a8.post1
    Python library 'pandas' version 1.3.3
    
    Parameters
    -----------
        Input:
            RIC (str): Refinitiv Identification Number (RIC) of a stock
            date (str): Date as of which peer group and variables are requested - in yyyy-mm-dd
        Output:
            peer_group (DataFrame): Dataframe of 50 peer companies along with requested variables
    
    '''
    # specify variables for the request
    fields = ["TR.F.GrossProfMarg","TR.F.NetIncAfterMinIntr","TR.F.TotCap","TR.F.OpMargPctTTM","TR.F.LeveredFOCF", "TR.F.TotRevenue",
          "TR.F.TotRevenue(SDate = -1Y)","TR.F.TotRevenue(SDate = -2Y)","TR.F.TotRevenue(SDate = -3Y)","TR.F.CurrRatio",
           "TR.PriceToBVPerShare","TR.PriceToSalesPerShare",'TR.F.EBITDA',"TR.EV","TR.F.TotShHoldEq",
              "TR.F.DebtTot","TR.F.NetDebtPerShr", "TR.F.CashCashEquivTot"]
 
    #search for peers
    instruments = 'SCREEN(U(IN(Peers("{}"))))'.format(RIC)
    
    #request variable data for each peer
    peer_group, error = rdp.legacy.get_data(instruments = instruments, fields = fields, parameters = {'SDate': date})
    
    return peer_group

        
        Copy
    

    	
            
#store rics and dates into separate lists
target_rics = target_data['Instrument'].to_list()
target_dates = target_data['AD-30'].dt.strftime('%Y-%m-%d').to_list()
 
#create empty lists for error data and a dataframe to store selected peers
no_peers = []
no_dates = []
peer_data = pd.DataFrame()
 
for i in range(len(target_rics)):
    try:
        #request Peer function for each target company in the lits
        vals = peers(target_rics[i], target_dates[i])
        #drop peers with missing values
        vals.dropna(inplace = True)
        #add a column for 30 days prior to the M&A announcement
        vals.insert(loc = 1, column = 'AD-30', value = target_dates[i])
        #append target company's peer data to the main dataframe of all peers
        peer_data = pd.concat([peer_data, vals], ignore_index = True, axis = 0)
        
    #if error is returned, store ric and request date in a separate list    
    except:
        no_peers.append(target_rics[i])
        no_dates.append(target_dates[i])
        continue

        
        Copy
    

    	
            
#run the data request code above for the companies in the error list
for i in range(len(no_peers)):
    try:
        vals = peers(no_peers[i], no_dates[i])
        vals.dropna(inplace = True)
        peer_data = pd.concat([peer_data, vals], ignore_index = True, axis = 0)
    except:
        continue

        
        Copy
    

    	
            
peer_data.to_excel('mergerdata/peer_data.xlsx')
 
#read peer data from the excel file
peer_data = pd.read_excel('mergerdata/peer_data.xlsx').drop(columns = ['Unnamed: 0'])
peer_data.insert(loc = 2, column = 'AD', value =  pd.to_datetime(peer_data['AD-30']) + datetime.timedelta(30))
peer_data.head()

        
        Copy
    

    	
            
#add labels
target_data['Label'] = 1
peer_data['Label'] = 0
 
#merge the target and non-target datasets
all_data = pd.concat([target_data, peer_data], ignore_index = True,  axis = 0).reset_index(drop = True)

        
        Copy
    

    	
            
#add a column to store the first part of the RIC 
all_data.insert(loc = 1, column = 'RIC_beg', value = [all_data['Instrument'][i].split(".")[0] for i in range(len(all_data))])
 
#run remove duplicate function on newly added column and remove that after the function is executed
all_data = all_data.drop_duplicates(subset = ['RIC_beg'], keep = 'first')
all_data.drop(columns = ['RIC_beg'], inplace = True)

        
        Copy
    

    	
            
#add two dates for 60 and 250 days before M&A announcement which are used to specify
#estimation and observation periods for abnormal return calculation
all_data.insert(loc = 1, column = 'AD-60', value =  all_data['AD'] - datetime.timedelta(60))
all_data.insert(loc = 1, column = 'AD-250', value =  all_data['AD'] - datetime.timedelta(250))
 
#calculate/add several variables that are used in ML models
all_data.insert(loc = len(all_data.columns), column = 'Profit to Capital', value = all_data['Net Income after Minority Interest']/all_data['Total Capital'])
all_data.insert(loc = len(all_data.columns), column = 'Free Cash Flow to Sales', value = all_data['Free Cash Flow']/all_data['Revenue from Business Activities - Total'])
all_data.insert(loc = len(all_data.columns), column = 'Cash to Capital', value = all_data['Cash & Cash Equivalents - Total']/all_data['Total Capital'])
all_data.insert(loc = len(all_data.columns), column = 'EV to EBITDA', value = all_data['Enterprise Value (Daily Time Series)']/all_data['Earnings before Interest Taxes Depreciation & Amortization'])
all_data.insert(loc = len(all_data.columns), column = 'Debt to EV', value = all_data['Debt - Total']/all_data['Enterprise Value (Daily Time Series)'])
all_data.insert(loc = len(all_data.columns), column = 'Sales_growth', value = 
                ((all_data['Revenue from Business Activities - Total']-all_data['Revenue from Business Activities - Total.1'])/all_data['Revenue from Business Activities - Total.1']*100 +
                (all_data['Revenue from Business Activities - Total.1']-all_data['Revenue from Business Activities - Total.2'])/all_data['Revenue from Business Activities - Total.2']*100+
                (all_data['Revenue from Business Activities - Total.2']-all_data['Revenue from Business Activities - Total.3'])/all_data['Revenue from Business Activities - Total.3']*100)/3)
 
#if there is no sales value for some year, sales growth variable is returned as "inf". So we need remove those instances
all_data.drop(all_data.loc[all_data['Sales_growth']==np.inf].index, inplace=True)
#drop NA values and reset index on the final dataset
all_data.dropna(inplace=True)
all_data = all_data.reset_index(drop=True)
all_data.head()

        
        Copy
    

    	
            
def ab_return(RIC, sdate, edate, announce_date, period):
    '''
    Calculate abnormal return of a given security during an observation period based on Event study Methodology(MacKinlay,1997)
    
    Dependencies
    ------------
    Python library 'refinitiv.dataplatform' version 1.0.0a8.post1
    Python library 'numpy' version 1.20.1
    Python library 'pandas' version 1.3.3
    Python library 'Sklearn' version 0.24.1
    
    Parameters
    -----------
        Input:
            RIC (str): Refinitiv Identification Number (RIC) of a stock
            sdate (str): Starting date of the estimation period - in yyyy-mm-dd
            edate (str): End date of the estimation period, which is also starting date of the observation period - in yyyy-mm-dd
            announce_date (str): End date of the observation period which is assumed to be the M&A announcement date or any other specified date
            period (int): Number of trading days in during the observation period. For each date in this period abnormal return is calculated
        Output:
            CAR (int): Cumulative Abnormal Return (CAR) for a given stock
            abnormal_returns (DataFrame): Dataframe containing abnormal returns during the observation period
    '''
    
    #create an empty dataframe to store abnormal returns
    abnormal_returns = pd.DataFrame({'#': np.arange(start = 1, stop = period)})
    
    ## estimate linear regression model parameters based on estimation period
    # get timeseries for the specified RIC and market proxy (S&P 500 in our case) for the both estimation and observation period
    df_all_per = rdp.legacy.get_timeseries([RIC, '.SPX'], 
                      start_date = sdate, 
                      end_date = announce_date,
                      interval='daily',
                      fields = 'CLOSE')
    
    # slice the estimation period
    df_all_per.reset_index(inplace = True)
    df_est_per = df_all_per.loc[(df_all_per['Date'] <= edate)]
    
    # calculate means of percentage change of returns for the stock and market proxy
    df_est_per.insert(loc = len(df_est_per.columns), column = "Return_stock", value = df_est_per[RIC].pct_change()*100)
    df_est_per.insert(loc = len(df_est_per.columns), column = "Return_market", value = df_est_per[".SPX"].pct_change()*100)
    mean_stock = df_est_per["Return_stock"].mean()
    mean_index = df_est_per["Return_market"].mean()
    df_est_per.dropna(inplace = True)
    
    # reshape the dataframe and estimate parameters of linear regression
    y = df_est_per["Return_stock"].to_numpy().reshape(-1,1)
    X = df_est_per["Return_market"].to_numpy().reshape(-1,1)
    model = LinearRegression().fit(X,y)
    Beta = model.coef_[0][0]
    intercept = model.intercept_[0]
    
    # slice the estimation period
    df_obs_per = df_all_per.loc[(df_all_per['Date'] >= edate)]
    
    # calculate percentage change of returns for the stock and market proxy
    df_obs_per.insert(loc = len(df_obs_per.columns), column = "Return_stock", value = df_obs_per[RIC].pct_change()*100)
    df_obs_per.insert(loc = len(df_obs_per.columns), column = "Return_market", value = df_obs_per[".SPX"].pct_change()*100)
    
    df_obs_per.dropna(inplace=True)
    df_obs_per.reset_index(inplace=True)
    
    # calculate and return cumulative abnormal return (CAR) for the observation period
    abnormal_returns.insert(loc = len(abnormal_returns.columns), column = str(RIC)+'_Date', value = df_obs_per["Date"])
    abnormal_returns.dropna(inplace=True)
    abnormal_returns.insert(loc = len(abnormal_returns.columns), column = str(RIC)+'_return', value = df_obs_per["Return_stock"] - (intercept + Beta * df_obs_per["Return_market"]))
    CAR =  abnormal_returns.iloc[:,2].sum()
    return CAR, abnormal_returns

        
        Copy
    

    	
            
RIC = all_data['Instrument'].to_list()
sdate = all_data['AD-250'].dt.strftime('%Y-%m-%d').to_list()
edate = all_data['AD-60'].dt.strftime('%Y-%m-%d').to_list()
announce_date = all_data['AD'].dt.strftime('%Y-%m-%d').to_list()

        
        Copy
    

    	
            
return_list = []
for i in range(len(RIC)):
    try:
        #run abnormal return function for all of the companies
        CAR, abnormal_returns = ab_return(RIC[i], sdate[i], edate[i], announce_date[i], 60)
        #calculate the cumulative abnormal return for the observation period and append the value to the return list
        return_list.append(abnormal_returns[str(RIC[i]) + '_return'].iloc[:len(abnormal_returns)-4].sum())
    except:
        #in case of error append the return list with NAN value
        return_list.append(np.nan)
        continue
 
#insert the return list to our original dataset and remove NA values
all_data.insert(loc = len(all_data.columns), column = 'AR', value = return_list)
all_data.dropna(inplace = True)

        
        Copy
    

    	
            
all_data.to_excel('mergerdata/all_data.xlsx')
 
#read data from a stored excel file
all_data = pd.read_excel('mergerdata/all_data.xlsx').drop(columns = ['Unnamed: 0'])
targets = len(all_data.loc[all_data['Label'] == 1])
non_targets = len(all_data.loc[all_data['Label'] == 0])
print(f'Number of target companies in the dataset: {targets}')
print(f'Number of non-target companies in the dataset: {non_targets}')

        
        Copy
    

    	
            
#extract announcement dates of target companies 
dates = all_data['AD'].loc[all_data['Label'] == 1]
peer_idx = []
 
for date in dates:
    #extract peers as of the announcement day
    peers = all_data.loc[(all_data['Label'] == 0) & (all_data['AD'] == date)]
    #check if target company has at least 7 peers
    if len(peers) > 7:
        #if yes, select the first 7
        peers = peers.iloc[:7]
    #get indexes of selected peers 
    peer_idx.append(peers.index)
#flatten the list of lists into a single list
peers = [item for sublist in peer_idx for item in sublist]
 
#update dataset to make sure only the selected peers are included
all_data = pd.concat([all_data.loc[all_data['Label'] == 1], all_data.loc[peers]], ignore_index = True, axis = 0).reset_index(drop = True)

        
        Copy
    

    	
            
all_data = all_data.drop(columns = ['AD-250', 'AD-60', 'AD-30', 'Net Income after Minority Interest', 'Total Capital','Free Cash Flow', 'Cash & Cash Equivalents - Total',
                                    'Revenue from Business Activities - Total', 'Revenue from Business Activities - Total.1',
                                       'Revenue from Business Activities - Total.1', 'Revenue from Business Activities - Total.2', 'Revenue from Business Activities - Total.3',
                                        "Total Shareholders'" + ' Equity incl Minority Intr & Hybrid Debt', 'Debt - Total',
                                   'Earnings before Interest Taxes Depreciation & Amortization', 'Enterprise Value (Daily Time Series)'])
all_data.head()

        
        Copy
    

    	
            
targets = len(all_data.loc[all_data['Label'] == 1])
non_targets = len(all_data.loc[all_data['Label'] == 0])
print(f'Number of target companies in the dataset: {targets}')
print(f'Number of non-target companies in the dataset: {non_targets}')

        
        Copy
    

    	
            
#create column dates for 30 day and 5 day before the M&A announcement
all_data.insert(loc = 1, column = 'AD-30', value =  all_data['AD'] - datetime.timedelta(30))
all_data.insert(loc = 1, column = 'AD-5', value =  all_data['AD'] - datetime.timedelta(5))

        
        Copy
    

    	
            
#create empty dataframe to store headlines and separate lists to store error values
all_headlines = pd.DataFrame()
err_ric = []
err_sdate = []
err_edate = []
 
for i in range(len(all_data)):
    try:
        ric_headlines = pd.DataFrame()
        #request news headlines for each instrument for the specified period
        df = rdp.get_news_headlines(query = all_data['Instrument'][i] + ' and Language:LEN', date_from = all_data['AD-30'][i].strftime('%Y-%m-%d'), 
                                           date_to = all_data['AD-5'][i].strftime('%Y-%m-%d'), count = 5000)
        #add the headlines along with the instrument and request dates to the ric_headlines dataframe
        if len(df) > 0:
            ric_headlines.insert(loc = 0, column = 'RIC', value = [all_data['Instrument'][i]]*len(df))
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'sdate', value = [all_data['AD-30'][i]]*len(df))
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'edate', value = [all_data['AD-5'][i]]*len(df))
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'Headlines', value = df['text'].values)
        #in case there is no headline, add the instrument and dates by indicating no news for the specified period
        else:
            ric_headlines.insert(loc = 0, column = 'RIC', value = [all_data['Instrument'][i]]*1)
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'sdate', value = all_data['AD-30'][i])
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'edate', value = all_data['AD-5'][i])
            ric_headlines.insert(loc = len(ric_headlines.columns), column = 'Headlines', value = 'no news')       
        all_headlines = pd.concat([all_headlines, ric_headlines], ignore_index = True, axis = 0)
    #store rics and dates resulting error in separate lists
    except:
        err_ric.append(all_data['Instrument'][i])
        err_sdate.append(all_data['AD-30'][i])
        err_edate.append(all_data['AD-5'][i])

        
        Copy
    

    	
            
all_headlines.to_excel('mergerdata/headlinesAll.xlsx')
 
#read headlines dataset from excel and remove duplicated headlines.
headlines = pd.read_excel('mergerdata/headlines.xlsx').drop(columns = ['Unnamed: 0'])
headlines = headlines.drop_duplicates(subset = ['Headlines'], keep = 'first').reset_index(drop = True)
headlines

        
        Copy
    

    	
            
#load FinBert model
model = BertForSequenceClassification.from_pretrained('finbert/pytorch_model.bin',config='finbert/config.json', num_labels=3)
 
#load the tokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

        
        Copy
    

    	
            
# read financial phrasebook dataset into a csv
input_df = pd.read_csv('data/sentiment_data/Sentences_50Agree.txt', sep=".@", header=None, encoding="ISO-8859-1", names=['text','label'])
 
#update the labels
input_df['label'].replace('positive', 0, inplace =True)
input_df['label'].replace('negative', 1, inplace =True)
input_df['label'].replace('neutral', 2, inplace =True)
input_df

        
        Copy
    

    	
            input_df[['text']].to_csv('BERT_RNA_Input.csv', index=False)
        
        Copy
    

    	
            
# read embeddings from .out file exported from the Labs UI 
X = pd.read_json('BERT-RNA/data_phrasebook.out')
y = input_df['label']
X

        
        Copy
    

    	
            
# get average length of sentences for finding the optimal number of PCA components
length = []
for i in range(len(input_df)):
    length.append(len(input_df['text'][i]))
print(f'Average length of our text input is {round(np.mean(length),0)}')

        
        Copy
    

    	
            
# run pca
pca = PCA(n_components = 127)
output_pca = pca.fit_transform(X.to_numpy())

        
        Copy
    

    	
            
#split data into training and testing set
X_train_pca,X_test_pca,y_train_pca,y_test_pca = train_test_split(pd.DataFrame(output_pca), y, test_size = 0.20,random_state=0)
 
#fit logistic regression model on the training set
logistic_regression_pca = LogisticRegression().fit(X_train_pca,y_train_pca)
 
#predict labels on testing set and report classification outputs
y_pred_pca = logistic_regression_pca.predict(X_test_pca)
print(classification_report(y_test_pca, y_pred_pca))

        
        Copy
    

    	
            
# read embeddings from Labs UI exported .out file
sentiment1 = pd.read_json('BERT-RNA/sentiment1.out')
sentiment2 = pd.read_json('BERT-RNA/sentiment2.out')
sentiment3 = pd.read_json('BERT-RNA/sentiment3.out')
sentiment4 = pd.read_json('BERT-RNA/sentiment4.out')
 
#merge sentiment headlines into one dataframe
sentiment = pd.concat([sentiment1, sentiment2, sentiment3, sentiment4], ignore_index = True, axis = 0)
sentiment

        
        Copy
    

    	
            
#apply dimensionality reduction
sentiment_pca = pca.fit_transform(sentiment.to_numpy())
 
#predict labels for news headlines
sentL = logistic_regression_pca.predict(sentiment_pca)
 
#add label list to the headlines dataframe
headlines['sentimentLabs'] = sentL
headlines

        
        Copy
    

    	
            dfs = dict(tuple(headlines.groupby('RIC', sort = False)))
        
        Copy
    

    	
            
#create empty dictionary to store the sentiment values
sentiments = {'RIC':[],'SentOverallFBert':[],'SentOverallLabs':[]}
 
#loop over each RIC
for ric in dfs:
    #append the RIC to the dictionary
    sentiments['RIC'].append(ric)
    
    #initiate overall Sentiment for FinBert
    SentOverallFBert = 0
    #loop over each news headline belonging to the RIC
    for text in dfs[ric]['Headlines']:
        #tokenize the headlines
        inputs = tokenizer(text, return_tensors="pt")
        #get prediction outputs
        outputs = model(**inputs)
        #get the maximum probability class
        sentF = torch.argmax(outputs[0])
        #update SentOverallFBert based on the classification output
        if sentF == 0:
            SentOverallFBert += 1
        elif sentF == 1:
            SentOverallFBert -= 1
    #append FinBert calculated overall sentiment of a company to the dictionary            
    sentiments['SentOverallFBert'].append(SentOverallFBert)
    
    #initiate overall Sentiment for BERT-RNA
    SentOverallLabs = 0
    #update SentOverallLabs based on the classification output
    for sentL in dfs[ric]['sentimentLabs']:
        if sentL == 0:
            SentOverallLabs += 1
        elif sentL == 1:
            SentOverallLabs -= 1 
    #append BERT-RNA calculated overall sentiment of a company to the dictionary            
    sentiments['SentOverallLabs'].append(SentOverallLabs)
    
#convert dictionary to a dataframe
sentiments = pd.DataFrame(sentiments)

        
        Copy
    

    	
            
sentiments.to_excel('mergerdata/sentiment_finbert_Labs.xlsx')
#read sentiment dataset from excel
sentiments = pd.read_excel('mergerdata/sentiment_finbert_Labs.xlsx')

        
        Copy
    

    	
            
#plot values using Plotly scatter plot
fig = px.scatter(sentiments, y = ["SentOverallFBert", "SentOverallLabs"])
#update plot layout
fig.update_layout(height=700, width=1100)
fig.update_yaxes(range=[-150, 550], tick0=0)
 
#move legend to the top left
fig.update_layout(legend=dict(
    yanchor="top",
    y=0.99,
    xanchor="left",
    x=0.01
))
fig.show()

        
        Copy
    

    	
            
#set index columns to implement dataframe joining
all_data.set_index('Instrument', inplace = True)
sentiments.rename(columns = {"RIC": "Instrument"}, inplace=True)
sentiments.set_index('Instrument', inplace = True)
 
#join NLP based and financial variable datasets
all_data = all_data.join(sentiments, on = 'Instrument', how = 'inner').reset_index().drop_duplicates(subset = ['Instrument'], keep = 'first')
all_data.head()

        
        Copy
    

    	
            all_data.corr().unstack().sort_values().drop_duplicates()
        
        Copy
    

    	
            
#drop correlated variables
X = all_data.drop(columns = ['Instrument', 'Label', 'AD', 'Operating Margin - %, TTM','EV to EBITDA', 
                             'Price To Sales Per Share (Daily Time Series Ratio)'])
y = all_data['Label']
 
#create separate datasets for no sentiment and sentiment based data
X_NoSent = X.drop(columns = ['SentOverallLabs', 'SentOverallFBert'])
X_BertRna = X.drop(columns = ['SentOverallFBert'])
X_Finbert = X.drop(columns = ['SentOverallLabs'])

        
        Copy
    

    	
            
#initiate logistic regression model
lr = LogisticRegression(solver = 'liblinear', penalty = 'l2', random_state = 0)
 
#define cross validation parameters
cv = RepeatedStratifiedKFold(n_splits = 20, n_repeats = 5, random_state = 0)
 
#run cross validation on logistic regression model with defined CV parameters
scoresNoSent = cross_validate(lr, X_NoSent, y, scoring=['roc_auc'], cv = cv, n_jobs = -1)
scoresBertRna = cross_validate(lr, X_BertRna, y, scoring=['roc_auc'], cv = cv, n_jobs = -1)
scoresFinbert = cross_validate(lr, X_Finbert, y, scoring=['roc_auc'], cv = cv, n_jobs = -1)
 
#report cross validation outputs on 3 datasets
print('\033[1m' + "Model with no sentiment variable" + '\033[0m')
print("AUC:" + str(round(scoresNoSent['test_roc_auc'].mean(),2)))
 
print('\033[1m' + "With Sentiment from BERT-RNA" + '\033[0m')
print("AUC:" + str(round(scoresBertRna['test_roc_auc'].mean(),2)))
 
print('\033[1m' + "With Sentiment from FinBert" + '\033[0m')
print("AUC:" + str(round(scoresFinbert['test_roc_auc'].mean(),2)))

        
        Copy
    

    	
            
#normalize the data
X_norm = (X_Finbert - np.mean(X_Finbert)) / np.std(X_Finbert, 0)
 
#fit logistic regression model with normalized data
model = lr.fit(X_norm, y)
 
#create a dataframe consisting of the list of coefficients
coefs = pd.DataFrame(index = X_Finbert.columns, data = model.coef_[0], columns = ['Coefficients'])

        
        Copy
    

    	
            
def logit_pvalue(model, x):
    '''
    Calculate z-scores for scikit-learn LogisticRegression.This function uses asymtptics for maximum likelihood estimates.
    
    Parameters:
    ------------
        Input
            model: fitted sklearn.linear_model.LogisticRegression with intercept and large C
            x:     matrix on which the model was fit
        Output:  
            p:     array of p-values
    '''
    p = model.predict_proba(x)
    n = len(p)
    m = len(model.coef_[0]) + 1
    coefs = np.concatenate([model.intercept_, model.coef_[0]])
    x_full = np.matrix(np.insert(np.array(x), 0, 1, axis = 1))
    ans = np.zeros((m, m))
    
    for i in range(n):
        ans = ans + np.dot(np.transpose(x_full[i, :]), x_full[i, :]) * p[i,1] * p[i, 0]
    vcov = np.linalg.inv(np.matrix(ans))
    se = np.sqrt(np.diag(vcov))
    t =  coefs/se  
    p = (1 - norm.cdf(abs(t))) * 2
    
    return p
 
#get p-values
zScores = logit_pvalue(model, X_Finbert)
#convert from scientific numpers
zScores = ['{0:f}'.format(num) for num in zScores][1:]
#append p-values to coefficient's dataframe
coefs.insert(loc = len(coefs.columns), column = 'P-Value', value = zScores)
coefs.T

        
        Copy
    

    	
            
#initiate random forest model with specified parameters
rf = RandomForestClassifier(n_estimators=500, class_weight='balanced_subsample', criterion='entropy', max_depth = 3, n_jobs=-1, random_state = 5)
 
#run cross validation on random forest model with defined CV parameters
scoresNoSentRf = cross_validate(rf, X_NoSent, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
scoresBertRnaRf = cross_validate(rf, X_BertRna, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
scoresFinbertRf = cross_validate(rf, X_Finbert, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
 
#report cross validation outputs on 3 datasets
print('\033[1m' + "Model with no sentiment variable" + '\033[0m')
print("AUC:" + str(round(scoresNoSentRf['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresNoSentRf['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresNoSentRf['test_recall'].mean(),2)))
print("F1:" + str(round(scoresNoSentRf['test_f1'].mean(),2)))
 
print('\033[1m' + "With Sentiment from BERT-RNA" + '\033[0m')
print("AUC:" + str(round(scoresBertRnaRf['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresBertRnaRf['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresBertRnaRf['test_recall'].mean(),2)))
print("F1:" + str(round(scoresBertRnaRf['test_f1'].mean(),2)))
 
print('\033[1m' + "With Sentiment from FinBert" + '\033[0m')
print("AUC:" + str(round(scoresFinbertRf['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresFinbertRf['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresFinbertRf['test_recall'].mean(),2)))
print("F1:" + str(round(scoresFinbertRf['test_f1'].mean(),2)))

        
        Copy
    

    	
            
#calculate feature importance
importanceFinbertRf = [round(imp,2) for imp in rf.fit(X_Finbert, y).feature_importances_]
importanceBertRnaRf = [round(imp,2) for imp in rf.fit(X_BertRna, y).feature_importances_]
 
#create subplots for two models
fig = make_subplots(
    rows=2, cols=1, subplot_titles=[
        'Feature importance for model with FinBert Sentiment',
        'Feature importance for model with BERT-RNA Sentiment'])
 
#add barplot for feature importance from FinBert model
fig.add_trace(go.Bar(y = X_Finbert.columns, x = importanceFinbertRf,
                     orientation='h', text = importanceFinbertRf, showlegend = False), row=1, col=1)
 
#add barplot for feature importance from BERT_RNA model
fig.add_trace(go.Bar(y = X_BertRna.columns, x = importanceBertRnaRf,
                       orientation='h', text = importanceBertRnaRf, showlegend = False), row=2, col=1)
 
#update layout and sort values inside the plot
fig.update_layout(height = 800, width = 1100)
fig.update_yaxes(categoryorder='total ascending')

        
        Copy
    

    	
            
#calculate number of target and non-target companies for assigning weights
target = all_data[all_data['Label'] == 1].shape[0]
nontarget = all_data[all_data['Label'] == 0].shape[0]
 
#initiate XGBoost model with specified parameters
xgBoost = XGBClassifier(scale_pos_weight = nontarget/target, max_depth=3, subsample = 0.5, n_estimators = 500, learning_rate = 0.1, alpha = 30)
 
#run cross validation on XGBoost model with defined CV parameters
scoresNoSentXgb = cross_validate(xgBoost, X_NoSent, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
scoresBertRnaXgb = cross_validate(xgBoost, X_BertRna, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
scoresFinbertXgb = cross_validate(xgBoost, X_Finbert, y, scoring = ['roc_auc', 'accuracy', 'f1', 'precision', 'recall'], cv = cv, n_jobs = -1)
 
#report cross validation outputs on 3 datasets
print('\033[1m' + "Model with no sentiment variable" + '\033[0m')
print("AUC:" + str(round(scoresNoSentXgb['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresNoSentXgb['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresNoSentXgb['test_recall'].mean(),2)))
print("F1:" + str(round(scoresNoSentXgb['test_f1'].mean(),2)))
 
print('\033[1m' + "With Sentiment from BERT-RNA" + '\033[0m')
print("AUC:" + str(round(scoresBertRnaXgb['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresBertRnaXgb['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresBertRnaXgb['test_recall'].mean(),2)))
print("F1:" + str(round(scoresBertRnaXgb['test_f1'].mean(),2)))
 
print('\033[1m' + "With Sentiment from FinBert" + '\033[0m')
print("AUC:" + str(round(scoresFinbertXgb['test_roc_auc'].mean(),2)))
print("Precision:" + str(round(scoresFinbertXgb['test_precision'].mean(),2)))
print("Recall:" + str(round(scoresFinbertXgb['test_recall'].mean(),2)))
print("F1:" + str(round(scoresFinbertXgb['test_f1'].mean(),2)))

        
        Copy
    

    	
            
importanceFinbertXgb = [round(imp,2) for imp in xgBoost.fit(X_Finbert, y).feature_importances_]
importanceBertRnaXgb = [round(imp,2) for imp in xgBoost.fit(X_BertRna, y).feature_importances_]
 
fig = make_subplots(
    rows=2, cols=1, subplot_titles=[
        'Feature importance for model with FinBert Sentiment',
        'Feature importance for model with BERT-RNA Sentiment'])
 
fig.add_trace(go.Bar(y = X_Finbert.columns, x = importanceFinbertXgb,
                     orientation='h', text = importanceFinbertXgb, showlegend = False), row=1, col=1)
 
fig.add_trace(go.Bar(y = X_BertRna.columns, x = importanceBertRnaXgb,
                       orientation='h', text = importanceBertRnaXgb, showlegend = False), row=2, col=1)
 
fig.update_layout(height=800, width=1100)
fig.update_yaxes(categoryorder='total ascending')

        
        Copy