唐宇迪老师推荐算法实战课程的源码复现，原理还未仔细解释 不足之处望多多指正。

导入所需工具包

import warnings
warnings.filterwarnings("ignore")
import pandas as pd
import numpy as np
import recmetrics
import matplotlib.pyplot as plt
from surprise import Reader,SVD,Dataset
from surprise.model_selection import train_test_split

加载评分数据，（这里只过滤筛选一部分数据用于实验）

ratings=pd.read_csv("ratings.csv")
ratings=ratings.query('rating>=3')
ratings.reset_index(drop=True,inplace=True)

ratings.head()

数据过滤：过滤出评分超过1000部电影的用户，方便起见避免出现稀疏矩阵的情况

n=1000
users=ratings.userId.value_counts()
users=users[users>n].index.tolist()

ratings=ratings.query('userId in @users')
print(ratings.shape)
ratings.head()

(1317902, 4)

从movies.csv中拿出对应的数据的（电影的title、genres）

movies=pd.read_csv('movies.csv')
print(movies.shape)
movies.head()

(27278, 3)

# 得到上表ratings表里对应的电影特征数据
rated_movies=ratings.movieId.tolist()
movies=movies.query('movieId in @rated_movies')
movies.set_index('movieId',inplace=True,drop=True)

制作数据集里的题材特征（将两张表实现关联）

movies=movies.genres.str.split("|",expand=True)
movies.reset_index(inplace=True)
movies.head()

Long tail绘图

• 使用工具包中的recmetics函数
• 其中percentage是红色虚线的百分比

import matplotlib.pyplot as plt
fig=plt.figure(figsize=(15,7))
recmetrics.long_tail_plot(df= ratings,
                         item_id_column='movieId',
                         interaction_type='movie ratings',
                         percentage=0.5,
                         x_labels=False)

利用surprise包构建推荐系统

# 获取surprise所需要格式的数据
reader=Reader(rating_scale=(0,5))
data=Dataset.load_from_df(ratings[['userId','movieId','rating']],reader)
# 切分数据集
trainset,testset=train_test_split(data,test_size=0.25)

#SVD训练数据
algo=SVD()
algo.fit(trainset)

<surprise.prediction_algorithms.matrix_factorization.SVD at 0x1af42397080>

# 测试
test=algo.test(testset)
test=pd.DataFrame(test)
test.drop("details",inplace=True,axis=1)
test.columns=['userId','movieId','actual','predictions']
test.head()

# MSE和RMSE评估指标
print(recmetrics.mse(test.actual,test.predictions))
print(recmetrics.rmse(test.actual,test.predictions))

0.26556372554189756
0.5153287548176383

#模型创建
cf_model=test.pivot_table(index='userId',columns='movieId',values='predictions').fillna(0)
cf_model.head()

5 rows × 16632 columns

# 推荐系统设计
def get_users_predictions(user_id,n,model):
    recmmended_items=pd.DataFrame(model.loc[user_id])
    recmmended_items.columns=['predicted_rating']
    recmmended_items=recmmended_items.sort_values('predicted_rating',ascending=False)
    recmmended_items=recmmended_items.head(n)
    return recmmended_items.index.tolist()

# 使用举例
get_users_predictions(156,10,cf_model)

[2028, 2762, 1198, 1704, 1242, 593, 1210, 919, 2268, 1136]

# 批量的测试结果
test = test.copy().groupby('userId')['movieId'].agg({'actual':(lambda x:list(set(x)))})
cf_recs=[]=[]
for user in test.index:
    cf_predictions=get_users_predictions(user,10,cf_model)
    cf_recs.append(cf_predictions)
    
test['cf_predictions']=cf_recs
test.head()

实验对比

#排行榜
popularity_recs = ratings.movieId.value_counts().head(10).index.tolist()

pop_recs=[]
for user in test.index:
    pop_predictions=popularity_recs
    pop_recs.append(pop_predictions)
    
test['pop_predictions']=pop_recs
test.head()

# 随机选择
ran_recs=[]
for user in test.index:
    random_predictions = ratings.movieId.sample(10).values.tolist()
    ran_recs.append(random_predictions)
test['random_predictions']=ran_recs
test.head()

topK 求精度与召回率Precision与Recall

覆盖率

def prediction_coverage(predicted,catalog):
    predicted_flattened = [p for sublist in predicted for p in sublist]
    unique_predictions = len(set(predicted_flattened))
    prediction_coverage = round(unique_predictions/(len(catalog)*1.0)*100,2)
    return prediction_coverage

catalog = ratings.movieId.unique().tolist()
random_coverage = prediction_coverage(ran_recs,catalog)
pop_coverage = prediction_coverage(pop_recs,catalog)
cf_coverage = prediction_coverage(cf_recs, catalog)

# 覆盖率
coverage_scores=[random_coverage,pop_coverage,cf_coverage]
model_names= ['Random Recommender','Popular Recommender','Collaborative Fillter']

fig=plt.figure(figsize=(7,5))
recmetrics.coverage_plot(coverage_scores,model_names)

问题（review）

*  以数据为基础的论文查找技巧；
*  SVD为算法基础的推荐基础的算法原理；（之前接触SVD主要是矩阵计算是将其用于对图像的压缩处理，此处的SVD的推荐排序原理不是很清楚，需要查资料学习）；
*  原来（大三阶段）看过的研究文献数据集大多来之kaggle实验过程与唐课程的区别在哪？