ソフトウェアエンジニアの技術ブログ：Software engineer tech blog

Gaussian Kernel and Bandwidth
Mean Integrated Standard Error
E[|Pn-P|^2] = Ef(Pn(x)-P(x))^2*dx

We can technically use MISE or AMISE to select the Optimal Bandwidth

def MahalanobisDist(x, y):
	covariance_xy = np.cov(x,y, rowvar=0)
	inv_covariance_xy = np.linalg.inv(covariance_xy)
	xy_mean = np.mean(x),np.mean(y)
	x_diff = np.array([x_i - xy_mean[0] for x_i in x])
	y_diff = np.array([y_i - xy_mean[1] for y_i in y])
	diff_xy = np.transpose([x_diff, y_diff])
	md = []
	for i in range(len(diff_xy)):
		md.append(np.sqrt(np.dot(np.dot(np.transpose(diff_xy[i]), inv_covariance_xy),diff_xy[i])))
	return md

md = MahalanobisDist(x,xbar)

problem formulation => choice of los/ risk
purpose of the model => problem formulation

Identification
natural sciences, economics, medicine, some engineering

Prediction/Generalize
statistic/ machine learning, comple phenomenon, general applications

M
logistic regression, support vector machine, random forest

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

def init_models(X_train, y_train):
	models = [LogisticRegression(),
			RandomForestClassifier(),
			SVC(probability=True)]

	for model in models:
		model.fit(X_train, y_train)

	return models

models = init_models(X_train, y_train)

Learning Curves
Plot of the model performance:
The Risk or Cost or Score vs.
Size of Training Set and Test Set
Classifiers: Score or l- Score

Non-Parametric Models: KDEs

Derived Feature: x = |f0 – f1|/f0
Definition: the ratio of the submitted charge to the difference between the submitted charge and payment amount by medicare.

x = abs(f0-f1)/f0
n0, bins0, patches0=plt.hist(x,100,normed=0,range=(0,1),histtype='stepfilled')
plt.setp(patches0, 'facecolor','g','alpha', 0.75)

from scipy import stats
from functools import partial
def my_kde_bandwidth(obj, fac=1./5):
	"""We use Scott's Rule, multiplied by a constant factor."""
	return np.power(obj.n, -1./(obj.d+4)) * fac

def getKDE(data, name="", bwfac = 0.2):
	x2 = data
	x_eval = np.linspace(x2.min() - 1, x2.max() + 1, 500)
	kde = stats.gaussian_kde(x2, bw_method=partial(my_kde_bandwidth, fac=bwfac))
	fig1 = plt.figure(figsize=(8.6))
	ax = fig1.add_subplot(111)
	plt.yscale=('log')
	plt.grid(True)
	x2h1, x2h2 = np.histogramix.bins=[0.,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0],normed
	ax.plot(x2, np.zeros(x2.shape), 'b+', ms=12)

Inspecting Distribution of Variables
f0: Average Submitted Charge
f1: Average Payment Amount
f2: Average Allowed Amount

g0 = f_CA.average_submitted_chrg_amt.values
g1 = f_CA.average_Medicare_payment_amt.values
g0 = f_CA.average_Medicare_allowed_amt.values

n0, bins0, patches0=plt.hist(g0,50,normed=0, range=(0,1000),histtype='stepfilled')
n2, bins2, patches2=plt.hist(g2,50,normed=0, range=(0,1000),histtype='stepfilled')
plt.setp(patches0, 'facecolor','g','alpha', 0.75)
plt.setp(patches2, 'facecolor','b','alpha', 0.75)

n0, bins0, patches0=plt.hist(f0,50,normed=1,log=range=(0,1000))

Scale variable we can
f0 – f0.min() / (f0.max() – f0.min())

n0, bins0, patches0=plt.hist((f0-f0.min())/(f0.max()-f0.min()),50,normed=1, log=1,range(-0.2,1.2),histtype='stepfilled')
n1, bins1, patches1=plt.hist((f2-f2.min())/(f2.max()-f2.min()),40,normed=1, log=1,range(-0.2,1,2),histtype='stepfilled')
plt_step(patches0,'facecolor','g','alpha',0.75)
plt_step(patches1,'facecolor','r','alpha',0.75)

The range of scaled variables: [0, 1]

Calculating Correlations
f1, f2: linearly correlated
Converiance: E[(x-μx)(y-μy)]
Pearson’s Correlation Co-efficient: Px,y = cov(x,y)/αxαy= E[(x-μx)(y-μy)]/αxαy

Parametric, Non Parametric, Mathematical
Kenerl K, Density D, Estimates E

CA_data = cwd +"/medicare_data/Medicare_Data_CA_xxx.csv"
f_CA = read_csv(CA_data)

f_CA.describe()
f_CA.head(10)
len(f_IL.columns)
for c in f_IL.columns : print c

IPython notebook

%pylab inline
from IPython.display import HTML
%matplotlib inline

import os
import sys
from StringIO import StringIO
import scipy
import seaborn as sns

from pandas import read_csv
import matplotlib.pyplot as pyplot

cwd = os.getcwd()

IL_data = cwd +"/medicare_data/Medicare_Data_IL_xxx.csv"
f_IL = read_csv(IL_data)

f_IL.describe()
f_IL.head(5)

len(f_IL.columns)
for c in f_IL.columns: print c

print len(f_IL.provider_type.unique())
print len(f_IL.nppes_provider_city.unique())
print len(f_IL.hcpcs_description.unique())

f0 = f_IL.average_submitted_chrg_amt.values
f1 = f_IL.average_Medicare_payment_amt.values
f2 = f_IL.average_Medicare_allowed_amt.values