过滤Sqoop脚本内容中库表并生成定表刷新元数据脚本

#!/bin/bash
#description:for special table invalid metadata;

function grep_databseAndTableName(){
for file in $(ls *.sh)
	do
	grep -o 'database\s\+\w\+\s\+\-*\w*\-\+\w\+\s\+\w\+' $file>>metadata_$file
	done
}
grep_databseAndTableName
echo "done grep database and tableName"

function create_invalidMetaDataSh(){
for metadataFile in $(ls metadata*.sh)
	do
		cat $metadataFile |while read line
		do
			database=`echo "$line" | awk -F " " '{print $2}' | sort | uniq`
			tableName=`echo "$line" | awk -F " " '{print $NF}' | sort | uniq`
			echo "impala-shell -q 'invalidate metadata $database.$tableName'">> invalidate_matadata_$metadataFile
		done

	done
}
create_invalidMetaDataSh
echo "done print invalidate shell"

sqoop脚本举例

sqoop import 
--connect jdbc:mysql://xxx/hb_paycenter_db?tinyInt1isBit=false --username xxx 
--password xxx 
--table nuggets_refund --hive-import --hive-overwrite --hive-database hb_paycenter_db 
--hive-table  nuggets_refund --fields-terminated-by '\001' --null-string '\\N' 
--null-non-string '\\N' 
--delete-target-dir 
--columns 'refund_id,contract_index,product_loan_index,period,refund_type,pending_clear_amt,status,create_time,update_time,valid,rpy_pre_dt,rpy_pre_jrn,rpy_ord_no,rpy_ord_dt,batch_no,crd_no_last,bank_code,ac_pay_typ,status_msg,bus_rsp_cd,with_drw_typ,clear_amt,is_notify,remark,dis_fee,dis_to_account,serial_no,telephone,id_num,real_name' 
--m 2;

过滤正则表达式

正则表达式：database\s+\w+\s+-\w-+\w+\s+\w+

过滤命令

grep -o ‘database\s+\w+\s+-\w-+\w+\s+\w+’ import_hotdata_ods.sh>>test

过滤结果

database xxx --hive-table xxx
database xxx --hive-table xxx
database xxx --hive-table urge_shop_tpl_log

最终结果

impala-shell -q ‘invalidate metadata xxx.xxx’
impala-shell -q ‘invalidate metadata xxx.xxx’
impala-shell -q ‘invalidate metadata xxx.xxx’
impala-shell -q ‘invalidate metadata xxx.debit_ceb_contract_info’
impala-shell -q ‘invalidate metadata xxx.xxx’