大数据场景下，如何快速将Linux 大文件处理小

!/bin/bash
count=0
cat oriTest.txt | while read data
do
count=$(( $count+1 ))
echo $count
dir=echo "$data" | awk -F "\t" '{print $5}'
if [ -e $dir ];then
echo "$data" >> exist.txt
else
echo "$data" >> noexist.txt
fi
done

!/bin/bash
source ~/.bashrc

readdata(){
cat $1 | while read data
do
dir=echo "$data" | awk -F "\t" '{print $5}'
if [ -e $dir ];then
echo "$data" >> "exist_$1.txt"
else
echo "$data" >> "noexist_$1.txt"
fi
done
}

for i in ${files[@]};do
echo $i
readdata $i &
done