数据文件推送sftp文件服务器脚本

脚本：

1. 从impala中抽取数据生成文件(.csv/.dat)，并生成.chk检查文件
2. 将生成的本地文件put到sftp文件服务器

#!/bin/bash
path="$(cd "$(dirname "$0")"; pwd)"
cd $path

##配置文件名
conf=${1}
# 跑批时间
trade=${2}

current_date=`date "+%Y%m%d"`
current_time=`date "+%H%M%S"`

mkdir -p /data/TextData_Transfer/log/${current_date}
mkdir -p /data/TextData_Transfer/everydayFile/${current_date}

TSK_ZIP_PATH="/data/TextData_Transfer/everydayFile/${current_date}"
log_file="/data/TextData_Transfer/log/${current_date}/${file_name}_${current_time}.log"


cat ${conf} | while read line
do            
            ##获取配置文件信息
		    sql=`echo ${line}|awk -F "@@" '{print $2}'|sed "s/##trade##/${trade}/g"`
		    file_name=`echo ${line}|awk -F "@@" '{print $4}'`
			file_name_temp=${file_name}_temp
			# 通用类型
			file_type=`echo ${line}|awk -F "@@" '{print $6}'`
			file_delimiter=`echo ${line}|awk -F "@@" '{print $8}'`
			
			# 获取SFTP相关信息
			SFTP_USER=`echo ${line}|awk -F "@@" '{print $10}'`
			SFTP_PASS=`echo ${line}|awk -F "@@" '{print $12}'`
			SFTP_HOST=`echo ${line}|awk -F "@@" '{print $14}'`
			SFTP_PORT=`echo ${line}|awk -F "@@" '{print $16}'`
            sftp_path=`echo ${line}|awk -F "@@" '{print $18}' |sed "s/##trade##/${trade}/g"`		

			echo "data write to ...file  ${current_time}" >> ${log_file}
			##生成文件
			impala-shell -i data02.bigdata.com -d pdm_data -l --auth_creds_ok_in_clear -u hive --ldap_password_cmd="echo -n Ab123456" -q "${sql}" -B --output_delimiter="${file_delimiter}" --print_header -o ${TSK_ZIP_PATH}/${file_name}.${file_type} 1>>${log_file} 2>&1
#            echo "impala output file ... end">> ${log_file}
#            sed -i "s/\t/!|/g" ${TSK_ZIP_PATH}/${trade}_${file_name_temp}.${file_type}
#            echo "change delimiter ... end" >> ${log_file}

            if [ $? -gt 0 ]
            then
                echo "Error : touch data ... FAIL" >> ${log_file}
                echo "Error : touch data ... FAIL"
            exit 31
            else
                echo "touch data ... end - Succeed" >> ${log_file}
                echo "touch data ... end - Succeed"
            fi
			
			##生成 check file 
			# 获取文件条数
            check1=`cat ${TSK_ZIP_PATH}/${file_name}.${file_type}|wc -l`
			# 获取文件大小
            check2=`ls -l ${TSK_ZIP_PATH}/${file_name}.${file_type}|awk -F " " '{print $5}'`
			# 获取文件生成时间
			check3=`ls -l --time-style '+%Y-%m-%d %H:%M:%S' ${TSK_ZIP_PATH}/${file_name}.${file_type}|awk -F " " '{print $6" "$7}'`
			echo "${file_name}.${file_type} ${check1} ${check2} ${check3}" >> ${log_file}	
			# 将上述信息输出至chk文件中
            echo "${file_name}.${file_type},${check1},${check2},${check3}" > ${TSK_ZIP_PATH}/${file_name}.chk          
            cur_time_start=`date "+%Y-%m-%d %H:%M:%S"`
            echo "${trade}_${file_name} sftp ... start ${cur_time_start}" >> ${log_file}

            ##sftp start
			# 将生成文件put到sftp文件服务器
            lftp -u ${SFTP_USER},${SFTP_PASS} sftp://${SFTP_HOST}:${SFTP_PORT} << EOF
            mkdir -p  ${sftp_path}
            cd '${sftp_path}'
            lcd ${TSK_ZIP_PATH}
            put ${file_name}.${file_type}
			put ${file_name}.chk
            bye
EOF
			# 判断是否上传成功
            if [ $? -gt 0 ]
            then
                echo "Error : ${trade}_${file_name} sftp ... FAIL" >> ${log_file}
            exit 32
            else
                echo "${trade}_${file_name} sftp ... end - Succeed" >> ${log_file}
            fi  
done

配置文件：

@@select asset_acct,trade_dt,prod_no,trade_mrkt,ser_num,order_status_cd from app_data.app_int_d_nwf_order_evt_z where biz_date=##trade##@@ @@INT_D_NWF_ORDER_EVT@@ @@csv@@ @@|@@ @@sftpdata@@ @@sftpdata123@@ @@192.168.215.144@@ @@22@@ @@data/##trade##@@