# -*- coding: utf-8 -*-
“””
Created on Wed Jul 4 18:40:55 2018
@author: zhen
“””
import pandas as pd
import numpy as np
# 创建空的df,保存测试数据
test_df = pd.DataFrame({‘K1’:[‘C1′,’C1′,’C2′,’C3′,’C4′,’C2′,’C1′],’K2’:[‘A’,’A’,’B’,’C’,’D’,np.NaN,np.NaN]})
# 按K1列进行分组,组内进行unique操作(去除重复元素,返回元组或列表)
test_df_unique = pd.DataFrame(test_df.groupby([‘K1’])[‘K2’].agg(‘unique’))
# 自定义函数判断元组中是否含有nan
def has_nan(list):
flag = False
for x in list:
if x is np.NaN:
flag = True
break
return flag
# 自定义函数判断元组中是否不含有nan
def no_nan(list):
flag = True
for x in list:
if x is np.NaN:
flag = False
break
return flag
# 获取k2列含有nan的数据
test_df_unique_has_nan = test_df_unique[test_df_unique[‘K2’].apply(has_nan)]
# 获取k2列不含有nan的数据
test_df_unique_no_nan = test_df_unique[test_df_unique[‘K2’].apply(no_nan)]
# 管理测试数据,获取源数据
test_df_get = test_df[test_df[‘K1’].isin(test_df_unique_has_nan.index.tolist())]
test_df_alone = test_df[test_df[‘K1’].isin(test_df_unique_no_nan.index.tolist())]
# 去除含nan的重复数据
test_df_get_nonan = test_df_get[~test_df_get[‘K2’].isna()]
# 组合数据
result = test_df_get_nonan.append(test_df_alone)
# 去重,得到最终结果
result_save = result.drop_duplicates(subset=[‘K1′,’K2′],keep=’last’)
# 结果落地
result_save.to_excel(‘C:/Users/zhen/Desktop/数据清洗之去重.xlsx’)
测试数据:
结果:
原创文章,作者:ItWorker,如若转载,请注明出处:https://blog.ytso.com/industrynews/13000.html