THRESHOLD=10000 # 文件数阈值
LOG_FILE=”/bigdata/logs/small_file_governance_$(date +%Y%m%d).log”

echo “$(date): Starting small file governance” >> ${LOG_FILE}

# 遍历需要治理的目录
for dir in $(hdfs dfs -ls -R /bigdata/warehouse/fgedu | grep “^d” | awk ‘{print $NF}’); do
FILE_COUNT=$(hdfs dfs -count ${dir} | awk ‘{print $2}’)

if [ ${FILE_COUNT} -gt ${THRESHOLD} ]; then
echo “$(date): Processing ${dir} with ${FILE_COUNT} files” >> ${LOG_FILE}

# 判断是否为历史数据
if [[ “${dir}” == *”/archive/”* ]] || [[ “${dir}” == *”/history/”* ]]; then
# 历史数据归档
ARCHIVE_NAME=$(echo ${dir} | tr ‘/’ ‘_’).har
hadoop archive -archiveName ${ARCHIVE_NAME} -p $(dirname ${dir}) $(basename ${dir}) /bigdata/archive/fgedu/
hdfs dfs -rm -r -skipTrash ${dir}
else
# 热数据合并
spark-submit –master yarn –deploy-mode client \
–class org.apache.spark.sql.execution.datasources.FileMerger \
–conf spark.sql.shuffle.partitions=10 \
/bigdata/app/spark/tools/file-merger.jar ${dir}
fi
fi
done

echo “$(date): Small file governance completed” >> ${LOG_FILE}