1.替换
思路：
- 把双引号【"】替换为横线【-】
-
把双竖线【||】替换为单个双引号【"】
image.png
#替换
import re
from tqdm import tqdm
#清洗数据
def file_sub(old_file,new_file):
file_data = [] # 初始化
with open(old_file, "r", encoding="GB18030") as f:
print('开始替换...')
rows = f.readlines()
for line in tqdm(rows):
a = re.sub('\x00','',re.sub('\s','',line))
a = re.sub('"','-',a)
a = re.sub('\|\|','"',a)
file_data.append(a)
with open(new_file, "w", encoding="GB18030") as f: # 写入替换好的文本
print('写入替换文本...')
for line in tqdm(file_data):
f.write(line + '\n')
print('批量替换完成')
def main():
file_sub('work.txt','work_new.csv')
2.查看是否有串行
image.png
无报错
#2.发现串行
#发现错误行
import pandas as pd
file ='work_new.csv'
dat = pd.read_table(file, sep=',',encoding = "GB18030",
dtype=str,low_memory=False,
header=None
)
#无错误行,导出文件
dat.to_csv('work_clean.csv',sep='|',index=False,quoting=1,encoding = 'GB18030')
#读取错误行
dat = pd.read_table(file, sep=',',encoding = "GB18030",
dtype=str,skiprows=2704912, nrows=1 ,low_memory=False,
header=None
)
