为你呈上热腾腾的shell多线程并发处理任务脚本

使用shell多线程并发完成任务

batchTask.sh

#!/usr/bin/bash###################### Autor: Troll## Date:  2020-06-15## Desc:  本地文件批量put性能测试(多线程)##################### 并发进程数thread_num=30startTime=$(date +%H:%M:%S)echo \"${startTime} task start...\"# mkfifotempfifo=\"upload_hdfs\"mkfifo ${tempfifo}# 使文件描述符为非阻塞式exec 6<>${tempfifo}rm -f ${tempfifo}# 为文件描述符创建占位信息for ((i=1;i<=${thread_num};i++))do{echo}done >&6# 数据目录localDirPath=\'../data\'# hdfs目录hdfsDirPath=\'/tmp/perfomance/data/\'# 删除hdfs目录$(hdfs dfs -rmr ${hdfsDirPath})# 创建hdfs目录$(hdfs dfs -mkdir -p ${hdfsDirPath})# echo \"hdfs dfs -mkdir -p ${hdfsDirPath}\"# 目录赋权$(hdfs dfs -chmod -R 777 ${hdfsDirPath})# echo \"hdfs dfs -chmod -R 777 ${hdfsDirPath}\"# 文件列表dataFiles=$(ls \"${localDirPath}/pending/\")# echo \"dataFiles:${dataFiles}\"# echo -e \"$dataFiles\"for line in ${dataFiles}doread -u6{# 上传hdfs集群$(hdfs dfs -put ${localDirPath}/pending/${line} ${hdfsDirPath})# echo \"hdfs dfs -put ${line} ${hdfsDirPath}\"# 移动目录$(mv ${localDirPath}/pending/${line}  ${localDirPath}/finish/ )# echo \"mv ${line}  ${localDirPath}/finish/\"# 打印echo \"${line} 处理成功！\" >&6# 等待1ssleep 1} &done# 等待子程序运行完成wait# 数据复位$(mv ${localDirPath}/finish/* ${localDirPath}/pending/ )# 关闭fd6管道exec 6>&-endTime=$(date +%H:%M:%S)echo \"${endTime} task run finish!\"