Pandas Vectorized Methods

>>> from pandas import Series, DataFrame
>>> d = {'one': Series([1,2,3], index=['a','b','c']),
... 'two': Series([1,2,3,4], index=['a','b','c','d'])}
>>> df = DataFrame(d)
>>> df
   one  two
a  1.0    1
b  2.0    2
c  3.0    3
d  NaN    4
>>> import numpy
>>> df.apply(numpy.mean)
one    2.0
two    2.5
dtype: float64
>>> df['one'].map(lambda x: x>= 1)
a     True
b     True
c     True
d    False
Name: one, dtype: bool
>>> df.applymap(lambda x: x>= 1)
     one   two
a   True  True
b   True  True
c   True  True
d  False  True
from pandas import DataFrame, Series
import numpy
def avg_bronze_medal_count():

    countries = ['Russian Fed.', 'Norway', 'Canada', 'United States',
                 'Netherlands', 'Germany', 'Switzerland', 'Belarus',
                 'Austria', 'France', 'Poland', 'China', 'Korea', 
                 'Sweden', 'Czech Republic', 'Slovenia', 'Japan',
                 'Finland', 'Great Britain', 'Ukraine', 'Slovakia',
                 'Italy', 'Latvia', 'Australia', 'Croatia', 'Kazakhstan']

    gold = [13, 11, 10, 9, 8, 8, 6, 5, 4, 4, 4, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
    silver = [11, 5, 10, 7, 7, 6, 3, 0, 8, 4, 1, 4, 3, 7, 4, 2, 4, 3, 1, 0, 0, 2, 2, 2, 1, 0]
    bronze = [9, 10, 5, 12, 9, 5, 2, 1, 5, 7, 1, 2, 2, 6, 2, 4, 3, 1, 2, 1, 0, 6, 2, 1, 0, 1]
    
    olympic_medal_counts = {'country_name':Series(countries),
                            'gold': Series(gold),
                            'silver': Series(silver),
                            'bronze': Series(bronze)}
    olympic_medal_counts_df = DataFrame(olympic_medal_counts)
    bronze_at_least_one_gold = olympic_medal_counts_df['bronze'][olympic_medal_counts_df['gold'] >= 1]
    avg_bronze_at_least_one_gold = numpy.mean(bronze_at_least_one_gold)

    print(avg_bronze_at_least_one_gold)

Create DataFrame

from pandas import DataFrame, series

def create_dataframe():

	countries = ['Russian Fed.', 'Norway', 'Canada', 'United States',
                 'Netherlands', 'Germany', 'Switzerland', 'Belarus',
                 'Austria', 'France', 'Poland', 'China', 'Korea', 
                 'Sweden', 'Czech Republic', 'Slovenia', 'Japan',
                 'Finland', 'Great Britain', 'Ukraine', 'Slovakia',
                 'Italy', 'Latvia', 'Australia', 'Croatia', 'Kazakhstan']

    gold = [13, 11, 10, 9, 8, 8, 6, 5, 4, 4, 4, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]
    silver = [11, 5, 10, 7, 7, 6, 3, 0, 8, 4, 1, 4, 3, 7, 4, 2, 4, 3, 1, 0, 0, 2, 2, 2, 1, 0]
    bronze = [9, 10, 5, 12, 9, 5, 2, 1, 5, 7, 1, 2, 2, 6, 2, 4, 3, 1, 2, 1, 0, 6, 2, 1, 0, 1]

    olympic_medal_counts = {'country_name': Series(countries), 'gold':Series(gold), \
    'silver': Series(silver), 'bronze': Series(bronze)}

    olympic_medal_counts_df = DataFrame(olympic_medal_counts)

    print(olympic_medal_counts_df)

concept of Series in Pandas

*This playground is inspired by Greg Reda’s post on Intro to Pandas Data Structures:
http://www.gregreda.com/2013/10/26/intro-to-pandas-data-structures/

import pandas as pd

if False:
	series = pd.Series(['Dave', 'Cheng-Han', 'Uxxxx', 42, -1789710578])
	print series

if False:
	series = pd.Series(['Dave', 'Cheng-Han', 359, 9001],
			index=['Instructor', 'Curriculum Manager', 'Course Number', 'Power Level'])
	print series['Instructor']
	print ""
	print series[['Instructor', 'Curriculum Manager', 'Course Number']]

if False:
	cuteness = pd.Series([1, 2, 3, 4, 5], index=['Cockroach', 'Fish', 'Mini Pig', 'Puppy', 'Kitten'])

	print cuteness > 3
	print ""
	print cuteness[cuteness > 3]

*This playground is inspired by Greg Reda’s post on Intro to Pandas Data Structures:
http://www.gregreda.com/2013/10/26/intro-to-pandas-data-structures/

import numpy as np
import pandas as pd

if False:
	data = {'year': [2010, 2011, 2012, 2011, 2012, 2010, 2011, 2012],
		'team':['Bears','Bears','Bears','Packers','Lions','Lions','Lions'],
		'wins': [11, 8, 10, 15, 11, 6, 10, 4],
		'losses':[5, 8, 6, 1, 5, 10, 6, 12]}
	football = pd.DataFrame(data)
	print football

if False:
	data = {'year':[2010, 2011, 2012, 2011, 2012, 2010, 2011, 2012],
			'team':['Bears','Bears','Bears','Packers','Packers','Lions',
			'Lions','Lions'],
			'wins':[11, 8, 10, 15, 11, 6, 10, 4],
			'losses':[5, 8, 6, 1, 5, 10, 6, 12]}
	football = pd.DataFrame(data)
	print football.dtypes
	print ""
	print football.describe()
	print ""
	print football.head()
	print ""
	print football.tail()

Pandas

Dataframes: string, int, float, boolean

install pandas

[vagrant@localhost ~]$ pip install pyparsing
Collecting pyparsing
  Downloading pyparsing-2.2.0-py2.py3-none-any.whl (56kB)
    100% |████████████████████████████████| 61kB 328kB/s
Installing collected packages: pyparsing
Successfully installed pyparsing-2.2.0
[vagrant@localhost ~]$ pip install pandas
Collecting pandas
  Downloading pandas-0.20.3-cp35-cp35m-manylinux1_x86_64.whl (24.0MB)
    100% |████████████████████████████████| 24.0MB 49kB/s
Collecting python-dateutil>=2 (from pandas)
  Downloading python_dateutil-2.6.1-py2.py3-none-any.whl (194kB)
    100% |████████████████████████████████| 194kB 785kB/s
Requirement already satisfied: numpy>=1.7.0 in ./.pyenv/versions/3.5.2/lib/python3.5/site-packages (from pandas)
Collecting pytz>=2011k (from pandas)
  Downloading pytz-2017.2-py2.py3-none-any.whl (484kB)
    100% |████████████████████████████████| 491kB 373kB/s
Collecting six>=1.5 (from python-dateutil>=2->pandas)
  Downloading six-1.10.0-py2.py3-none-any.whl
Installing collected packages: six, python-dateutil, pytz, pandas
Successfully installed pandas-0.20.3 python-dateutil-2.6.1 pytz-2017.2 six-1.10.0
>>> from pandas import Series, DataFrame
>>> d = {'name': Series(['Braund', 'Cummings', 'Heikkinen', 'Allen'], index=['a', 'b', 'c', 'd']),
... 'age': Series([22,38,26,35], index=['a','b','c','d']),
... 'fare': Series([7.25, 71.83, 8.05], index=['a','b','d']),
... 'survived?': Series([False, True, True, False], index=['a','b','c','d'])}
>>> df = DataFrame(d)
>>> print(df)
   age   fare       name  survived?
a   22   7.25     Braund      False
b   38  71.83   Cummings       True
c   26    NaN  Heikkinen       True
d   35   8.05      Allen      False

Using numpy

>>> numbers = [1,2,3,4,5]
>>> numpy.mean(numbers)
Traceback (most recent call last):
  File "", line 1, in 
NameError: name 'numpy' is not defined
>>> import numpy
>>> numbers = [1,2,3,4,5]
>>> numpy.mean(numbers)
3.0
>>> numpy.median(numbers)
3.0
>>> numpy.std(numbers)
1.4142135623730951

install numpy

[vagrant@localhost ~]$ python -V
Python 3.5.2
[vagrant@localhost ~]$ pip list
pip (8.1.1)
setuptools (20.10.1)
You are using pip version 8.1.1, however version 9.0.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
[vagrant@localhost ~]$ pip install --upgrade pip
Collecting pip
  Downloading pip-9.0.1-py2.py3-none-any.whl (1.3MB)
    100% |████████████████████████████████| 1.3MB 478kB/s
Installing collected packages: pip
  Found existing installation: pip 8.1.1
    Uninstalling pip-8.1.1:
      Successfully uninstalled pip-8.1.1
Successfully installed pip-9.0.1
[vagrant@localhost ~]$ pip install numpy
Collecting numpy
  Downloading numpy-1.13.1-cp35-cp35m-manylinux1_x86_64.whl (16.9MB)
    100% |████████████████████████████████| 16.9MB 54kB/s
Installing collected packages: numpy
Successfully installed numpy-1.13.1

Data Scientist

Hacking skills, Math & statistics knowledge, Substantive Expertise
Machine Learning, Data Science, Traditional Research

Raw data – processing – data set – statistical models / analysis – machine learning predictions – data driven products, reports visualization blogs

‘substantive expertise’
– knows which question to ask
– can interpret the data well
– understands structure of the data
– data scientist often work in team

Data science can solve problems you’d expect…
– netflix, social media, web apps, (okcupid, uber, etc)

bioin formatics, urban planning, a straphysics, public health, public health, sports

Tools to use
– Numpy
– multidimensional arrays + matrices
– mathematical functions
– Pandas
– handle data in a way suited to analysis
– similar to R
Both common among data scientists

MISE

Gaussian Kernel and Bandwidth
Mean Integrated Standard Error
E[|Pn-P|^2] = Ef(Pn(x)-P(x))^2*dx

We can technically use MISE or AMISE to select the Optimal Bandwidth

def MahalanobisDist(x, y):
	covariance_xy = np.cov(x,y, rowvar=0)
	inv_covariance_xy = np.linalg.inv(covariance_xy)
	xy_mean = np.mean(x),np.mean(y)
	x_diff = np.array([x_i - xy_mean[0] for x_i in x])
	y_diff = np.array([y_i - xy_mean[1] for y_i in y])
	diff_xy = np.transpose([x_diff, y_diff])
	md = []
	for i in range(len(diff_xy)):
		md.append(np.sqrt(np.dot(np.dot(np.transpose(diff_xy[i]), inv_covariance_xy),diff_xy[i])))
	return md

md = MahalanobisDist(x,xbar)

problem formulation => choice of los/ risk
purpose of the model => problem formulation

Identification
natural sciences, economics, medicine, some engineering

Prediction/Generalize
statistic/ machine learning, comple phenomenon, general applications

M
logistic regression, support vector machine, random forest

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

def init_models(X_train, y_train):
	models = [LogisticRegression(),
			RandomForestClassifier(),
			SVC(probability=True)]

	for model in models:
		model.fit(X_train, y_train)

	return models

models = init_models(X_train, y_train)

Learning Curves
Plot of the model performance:
The Risk or Cost or Score vs.
Size of Training Set and Test Set
Classifiers: Score or l- Score

Kerndel Density Estimates

Non-Parametric Models: KDEs

Derived Feature: x = |f0 – f1|/f0
Definition: the ratio of the submitted charge to the difference between the submitted charge and payment amount by medicare.

x = abs(f0-f1)/f0
n0, bins0, patches0=plt.hist(x,100,normed=0,range=(0,1),histtype='stepfilled')
plt.setp(patches0, 'facecolor','g','alpha', 0.75)

from scipy import stats
from functools import partial
def my_kde_bandwidth(obj, fac=1./5):
	"""We use Scott's Rule, multiplied by a constant factor."""
	return np.power(obj.n, -1./(obj.d+4)) * fac

def getKDE(data, name="", bwfac = 0.2):
	x2 = data
	x_eval = np.linspace(x2.min() - 1, x2.max() + 1, 500)
	kde = stats.gaussian_kde(x2, bw_method=partial(my_kde_bandwidth, fac=bwfac))
	fig1 = plt.figure(figsize=(8.6))
	ax = fig1.add_subplot(111)
	plt.yscale=('log')
	plt.grid(True)
	x2h1, x2h2 = np.histogramix.bins=[0.,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0],normed
	ax.plot(x2, np.zeros(x2.shape), 'b+', ms=12)