# machine-learning method to predict PCA weights

I have been using certain linear-regression to extract the PCA (top 3) weights relating to a certain data-set. I was wondering, instead of using linear-regression to generate the weights, I can use machine-learning (from python skLearn) to extract the weights.

My block below is how I traditionally extract the weights.

eigVal, eigVec = scipy.linalg.eig(data_.cov())

results_ = []
for cnt in range(len(data_)):
y_ = (data_.iloc[cnt].values)
a1, _, _, _ = np.linalg.lstsq(eigVec[:,0][:,np.newaxis], y_)
a2, _, _, _ = np.linalg.lstsq(eigVec[:,1][:,np.newaxis], y_)
a3, _, _, _ = np.linalg.lstsq(eigVec[:,2][:,np.newaxis], y_)
mv_pca = a1 * eigVec[:,0] + a2 * eigVec[:,1] + a3* eigVec[:,2]
mv_abs = np.sum(abs(y_))
mv_err = np.sum(abs((mv_pca - y_)))
results_.append([data_.index[cnt], a1.real[0], a2.real[0], a3.real[0], mv_abs, mv_err])


this block below is how I am trying to use a few different learning techniques to extract the weights. Two problems I face is that,

• the learning code can only handle integers (hence, the need to digitize the continuous variables).
• I am not sure how to handle what are essentially a vector for each PCA, when machine-learning takes only one (integer) variable for each input state.

This is the block code below

from sklearn.linear_model import SGDClassifier
from sklearn.naive_bayes import GaussianNB # gaussian process machine learning https://scikit-learn.org/stable/modules/naive_bayes.html
from sklearn import tree # Decision Trees (DTs) are a non-parametric supervised learning method used for classification and regression.
from sklearn.ensemble import RandomForestClassifier # random forest tree machine learning
from sklearn.neighbors import KNeighborsClassifier # K nearest neighbour
from sklearn.neighbors import NearestNeighbors # nearest neighbour

from sklearn import datasets

l_time = []

bins1 = np.linspace(np.min(wt_.wt1)*0.95, np.max(wt_.wt1)*1.05, 20)
bins2 = np.linspace(np.min(wt_.wt2)*0.95, np.max(wt_.wt2)*1.05, 20)
bins3 = np.linspace(np.min(wt_.wt3)*0.95, np.max(wt_.wt3)*1.05, 20)
bins_Target = np.linspace(np.min(data_.EUR10Y)*0.95, np.max(data_.EUR10Y)*1.05, 20)
dig_1 = np.digitize(wt_.wt1, bins1)
dig_2 = np.digitize(wt_.wt2, bins2)
dig_3 = np.digitize(wt_.wt3, bins3)
dig_T = np.digitize(data_.EUR10Y, bins_Target)

ml_data = pd.DataFrame([dig_1, dig_2, dig_3]).T.values # generate the input vectors of PCA weights
ml_target = dig_T # set the target to be predicted to the EUR 10y swap rate
fl_data = ml_data[0:1200] # initialize the first 200 data points
fl_target = ml_target[0:1200]
columns = ['wt1', 'wt2', 'wt3']

start_time = time.time()
SKgnb_pred = GaussianNB().fit(ml_data, ml_target).predict(fl_data)
print("gnb --- %s seconds ---" % (time.time() - start_time))
print("Number of mislabeled points out of a total %d points : %d" % (fl_data.shape[0],(fl_target != SKgnb_pred).sum()))
l_time.append(['gnb', 1000 * (time.time() - start_time)])

start_time = time.time()
SKdtr_pred = tree.DecisionTreeClassifier().fit(ml_data, ml_target).predict(fl_data)
print("dtr --- %s seconds ---" % (time.time() - start_time))
print("Number of mislabeled points out of a total %d points : %d" % (fl_data.shape[0],(fl_target != SKdtr_pred).sum()))
l_time.append(['dtr', 1000 * (time.time() - start_time)])

start_time = time.time()
SKftr_pred = RandomForestClassifier(n_estimators= len(ml_data), max_depth= len(columns), random_state= None).fit(ml_data, ml_target).predict(fl_data)
print("ftr --- %s seconds ---" % (time.time() - start_time))
print("Number of mislabeled points out of a total %d points : %d" % (fl_data.shape[0],(fl_target != SKftr_pred).sum()))
l_time.append(['ftr', 1000 * (time.time() - start_time)])

start_time = time.time()
SKknn_pred = KNeighborsClassifier(n_neighbors=1, algorithm='ball_tree', metric = 'euclidean').fit(ml_data, ml_target).predict(fl_data)
print("knn --- %s seconds ---" % (time.time() - start_time))
print("Number of mislabeled points out of a total %d points : %d" % (fl_data.shape[0],(fl_target != SKknn_pred).sum()))
l_time.append(['knn', 1000 * (time.time() - start_time)])

start_time = time.time()
SKsgd_pred = SGDClassifier(loss="hinge", penalty="l2", max_iter= 100).fit(ml_data, ml_target).predict(fl_data) # stochastic gradient descent
print("sgd --- %s seconds ---" % (time.time() - start_time))
print("Number of mislabeled points out of a total %d points : %d" % (fl_data.shape[0],(fl_target != SKsgd_pred).sum()))
l_time.append(['sgd', 1000 * (time.time() - start_time)])

SK_target = pd.DataFrame(fl_target, columns = ['target'])
SK_data = pd.DataFrame(fl_data, columns = columns)

SKgnb_pred = pd.DataFrame(SKgnb_pred, columns = ['pred_gnb'])
SKdtr_pred = pd.DataFrame(SKdtr_pred, columns = ['pred_dtr'])
SKftr_pred = pd.DataFrame(SKftr_pred, columns = ['pred_ftr'])
SKknn_pred = pd.DataFrame(SKknn_pred, columns = ['pred_knn'])
SKsgd_pred = pd.DataFrame(SKsgd_pred, columns = ['pred_sgd'])

SK_all = SK_data.merge(SK_target, left_index = True, right_index = True).merge(SKknn_pred, left_index = True, right_index = True)
SK_all = SK_all.merge(SKdtr_pred, left_index = True, right_index = True).merge(SKftr_pred, left_index = True, right_index = True).merge(SKgnb_pred, left_index = True, right_index = True)
SK_all = SK_all.merge(SKsgd_pred, left_index = True, right_index = True)