数据准备技巧及其对机器学习的重要性

# 读取数据
df_car = pd.read_csv('CarPrice_Assignment.csv')
df_car.head(10)  # 显示前十行数据
df_car.drop(['car_ID'], axis=1, inplace=True)  # 重复信息，丢弃car_ID列 
# 检查列是否包含空数值
df_car.isnull().sum()
# 丢弃重复行
df_car.drop_duplicates(inplace=True)
# 根据其类型分配列名
cat_variable = []
num_variable = []
for col in df_car.columns:
    if df_car[col].dtype == 'object':
        cat_variable.append(col)
    elif df_car[col].dtype == 'float' or df_car[col].dtype == 'int64':
        num_variable.append(col)
        
# CarName列中类别数量 
df_car['CarName'].value_counts()
# 检测数字变量中的离群点
df_car[num_variable].describe()

# 分离汽车及其公司名称，从而进行特征分割
# 检查Company_name列是否有重复数据 
df_car['Company_name'] = df_car['CarName'].apply(lambda x:x.split(' ')[0])
df_car['Company_name'].value_counts()
df_car.drop(labels='CarName',axis=1,inplace=True)
# 创建字典合并相似公司名称
company_name_map_dict = {
    'maxda' : 'mazda',
    'vw' : 'volkswagen',
    'toyouta' : 'toyota',
    'Nissan' : 'nissan',
    'vokswagen' : 'volkswagen',
    'porcshce' : 'porsche'}
df_car['Company_name'] = df_car['Company_name'].replace(company_name_map_dict)
df_car['Company_name'].value_counts()
# 绘图显示price与Company_name的关系
plt.figure(figsize=(20, 8))
sns.boxplot(x = 'Company_name', y = 'price', data = df_car)
# 绘图显示数字列是否有线性相关
plt.figure(figsize=(30, 25))
for i, col in enumerate(num_variable):
    plt.subplot(5,3,i+1)
    sns.regplot(col, 'price', df_car, fit_reg=False)

    # 将类别列转换为虚拟变量，方便后续ML模型处理
	df_car['fueltype'] = df_car['fueltype'].map({'gas':0, 'diesel':1})
	df_car['aspiration'] = df_car['aspiration'].map({'std':0, 'turbo':1})
	df_car['doornumber'] = df_car['doornumber'].map({'four':0, 'two':1})
	df_car['enginelocation'] = df_car['enginelocation'].map({'front':0, 'rear':1})
	
	# 为拥有超过两种类别的列创建虚拟变量
	for col in ['carbody', 'drivewheel', 'enginetype', 'cylindernumber', 'fuelsystem', 'Company_name']:
	    df_car = pd.get_dummies(df_car, columns=[col], drop_first=True)
	   
	# 显示将所有类型数据列转换为0-1后的新数据 
	df_car.head()
	
	# 将数据集分为训练与测试两部分 
	df_train, df_test = train_test_split(df_car, test_size = 0.15, random_state=44)
	# 将所有数字类别列转化为0或1 
	from sklearn.preprocessing import MinMaxScaler
	scaler = MinMaxScaler()
	df_train[list(df_train.columns)] = scaler.fit_transform(df_train)
	# 取price列为因变量‘y’，其余数据为自变量‘x’ 
	y_train = df_train.pop('price')
	X_train = df_train

* 特征选择：用于对数据集中现有特征的重要性进行排序，并剔除不重要的特征。
* 特征提取：用于将数据投影到一个较低维度的空间，但依然保留原始数据最重要的特征

def corr_heatmap(df, v):
	    correlations = df[v].corr()
	    # Create color map ranging between two colors
	    cmap = sns.diverging_palette(220, 10, as_cmap=True)
	    fig, ax = plt.subplots(figsize=(20,20))
	    sns.heatmap(correlations, cmap=cmap, vmax=1.0, center=0, fmt='.2f',
	                square=True, linewidths=.5, annot=True, cbar_kws={"shrink": .75})
	    plt.show()
	    
	# 可视化相关矩阵以排除因变量 
	corr_heatmap(df_car, num_variable[:-1])
	
	# 丢弃高度相关的属性
	df_car = df_car.drop(['citympg', 'curbweight', 'wheelbase', 'enginesize', 'carwidth'], axis=1)
	# Importing RFE and LinearRegression
	from sklearn.feature_selection import RFE
	from sklearn.linear_model import LinearRegression
	
	# Running RFE with the output number of the variable equal to 10
	lm = LinearRegression()
	lm.fit(X_train, y_train)               # Model_1
	rfe = RFE(lm, 10)                      # running RFE
	rfe = rfe.fit(X_train, y_train)
	# Displaying the list of column with there rfe rank
	list(zip(X_train.columns,rfe.support_,rfe.ranking_))
	# taking a subset of column, that rfe has suggested important
	col = X_train.columns[rfe.support_]
	
	# Creating X_test dataframe with RFE selected variables
	X_rfe = X_train[col]
	lm.fit(X_rfe, y_train)              # Model_2