# 修改JVM内存参数
JAVA_OPTS=”-Xms2G -Xmx4G -XX:MaxPermSize=512m -XX:+UseConcMarkSweepGC
-XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/azkaban/logs”

# 编辑Web服务器内存配置
# vi /data/azkaban/azkaban-web-server-3.91.0/bin/start-web.sh

# 修改JVM内存参数
JAVA_OPTS=”-Xms2G -Xmx4G -XX:MaxPermSize=512m -XX:+UseConcMarkSweepGC
-XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/azkaban/logs”

# 验证配置
# grep JAVA_OPTS /data/azkaban/azkaban-exec-server-3.91.0/bin/start-exec.sh
JAVA_OPTS=”-Xms2G -Xmx4G -XX:MaxPermSize=512m -XX:+UseConcMarkSweepGC
-XX:+CMSClassUnloadingEnabled -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/azkaban/logs”

# 优化执行器参数
executor.maxThreads=100
executor.flow.threads=50
executor.queue.size=2000
executor.process.threads=50
executor.flow.threadpool.core.size=20
executor.flow.threadpool.max.size=100
executor.flow.threadpool.keepalive.seconds=60
executor.cleanup.retry.interval.millis=30000
executor.cleanup.retry.count=5
executor.flow.lru.size=1000

# 优化数据库连接池
mysql.numconnections=200
mysql.pool.maxActive=200
mysql.pool.maxIdle=50
mysql.pool.minIdle=20
mysql.pool.maxWait=30000
mysql.pool.validationQuery=SELECT 1
mysql.pool.testOnBorrow=true
mysql.pool.testWhileIdle=true
mysql.pool.timeBetweenEvictionRunsMillis=30000

# 编辑Web服务器配置
# vi /data/azkaban/azkaban-web-server-3.91.0/conf/azkaban.properties

# 优化数据库连接池
mysql.numconnections=200
mysql.pool.maxActive=200
mysql.pool.maxIdle=50
mysql.pool.minIdle=20
mysql.pool.maxWait=30000
mysql.pool.validationQuery=SELECT 1
mysql.pool.testOnBorrow=true
mysql.pool.testWhileIdle=true
mysql.pool.timeBetweenEvictionRunsMillis=30000

# 设置日志级别
log4j.rootLogger=INFO, file, console

# 文件输出配置
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=/data/azkaban/logs/azkaban-executor.log
log4j.appender.file.MaxFileSize=100MB
log4j.appender.file.MaxBackupIndex=10
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p
%c{1}:%L – %m%n

# 控制台输出配置
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p
%c{1}:%L – %m%n

# 编辑Web服务器日志配置
# vi /data/azkaban/azkaban-web-server-3.91.0/conf/log4j.properties

# 设置日志级别
log4j.rootLogger=INFO, file, console

# 文件输出配置
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=/data/azkaban/logs/azkaban-webserver.log
log4j.appender.file.MaxFileSize=100MB
log4j.appender.file.MaxBackupIndex=10
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p
%c{1}:%L – %m%n

# 控制台输出配置
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p
%c{1}:%L – %m%n

# 激活执行服务器
# curl -G “http://localhost:12321/executor?action=activate”
{“status”:”success”}

# 启动Web服务器
# cd /data/azkaban/azkaban-web-server-3.91.0
# ./bin/start-web.sh

# 访问Web控制台
# 打开浏览器，访问 https://azkaban01.fgedu.net.cn:8443

# 验证Web控制台访问
# curl -k -I https://localhost:8443
HTTP/1.1 200 OK
Date: Fri, 05 Apr 2024 10:00:00 GMT
Content-Type: text/html;charset=UTF-8
Content-Length: 12345
Server: Jetty(9.4.43.v20210629)

Web控制台登录：
– 用户名：admin
– 密码：admin

<azkaban-users>
<user groups=”admin” password=”admin123″ roles=”admin” username=”admin”/>
<user groups=”users” password=”user123″ roles=”reader” username=”user”/>
<user groups=”developers” password=”dev123″ roles=”reader,writer”
username=”dev”/>
<role name=”admin” permissions=”ADMIN”/>
<role name=”reader” permissions=”READ”/>
<role name=”writer” permissions=”WRITE”/>
<role name=”metrics” permissions=”METRICS”/>
</azkaban-users>

# 配置HTTPS
# vi /data/azkaban/azkaban-web-server-3.91.0/conf/azkaban.properties

web.useHttps=true
web.ssl.port=8443
web.keystore.file=/data/azkaban/azkaban-web-server-3.91.0/keystore
web.keystore.password=azkaban

# 创建flow20.xml
# vi /data/azkaban/projects/wordcount/flow20.xml

<?xml version=”1.0″ encoding=”UTF-8″?>
<azkaban-flow xmlns=”http://azkaban.github.io/azkaban-flow/2.0″>
<name>WordCount Flow</name>
<nodes>
<node name=”prepare-input” type=”command”>
<config>
<property name=”command”>hdfs dfs -mkdir -p
/user/azkaban/wordcount/input</property>
</config>
<edges>
<edge to=”copy-input” />
</edges>
</node>
<node name=”copy-input” type=”command”>
<config>
<property name=”command”>hdfs dfs -copyFromLocal -f
/data/azkaban/projects/wordcount/input.txt
/user/azkaban/wordcount/input/</property>
</config>
<edges>
<edge to=”wordcount” />
</edges>
</node>
<node name=”wordcount” type=”command”>
<config>
<property name=”command”>hadoop jar
/data/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar
wordcount /user/azkaban/wordcount/input
/user/azkaban/wordcount/output</property>
</config>
<edges>
<edge to=”check-output” />
</edges>
</node>
<node name=”check-output” type=”command”>
<config>
<property name=”command”>hdfs dfs -cat
/user/azkaban/wordcount/output/part-r-00000</property>
</config>
</node>
</nodes>
</azkaban-flow>

# 创建输入文件
# echo “hello world hello azkaban” > /data/azkaban/projects/wordcount/input.txt

# 创建项目属性文件
# vi /data/azkaban/projects/wordcount/project.properties

name=WordCount
version=1.0
description=WordCount Example

# 打包项目
# cd /data/azkaban/projects/wordcount
# zip -r wordcount.zip .

# 验证打包
# ls -la wordcount.zip
-rw-r–r– 1 root root 1024 Apr 5 10:00 wordcount.zip

# 执行工作流
# 点击项目名称”WordCount”
# 点击工作流名称”WordCount Flow”
# 点击”Execute Flow”
# 点击”Submit”

# 监控工作流执行
# 点击”Executions” -> 查看执行状态
# 点击执行ID -> 查看详细日志

# 输出案例如下：
24/04/05 10:00:00 INFO [JobRunnerThread] Starting job prepare-input
24/04/05 10:00:01 INFO [JobRunnerThread] Job prepare-input succeeded
24/04/05 10:00:01 INFO [JobRunnerThread] Starting job copy-input
24/04/05 10:00:02 INFO [JobRunnerThread] Job copy-input succeeded
24/04/05 10:00:02 INFO [JobRunnerThread] Starting job wordcount
24/04/05 10:00:30 INFO [JobRunnerThread] Job wordcount succeeded
24/04/05 10:00:30 INFO [JobRunnerThread] Starting job check-output
24/04/05 10:00:31 INFO [JobRunnerThread] Job check-output succeeded
24/04/05 10:00:31 INFO [FlowRunner] Flow WordCount Flow completed successfully

# 验证输出结果
# hdfs dfs -cat /user/azkaban/wordcount/output/part-r-00000
hello 2
azkaban 1
world 1

<?xml version=”1.0″ encoding=”UTF-8″?>
<azkaban-flow xmlns=”http://azkaban.github.io/azkaban-flow/2.0″>
<name>WordCount Flow</name>
<config>
<property name=”schedule”>0 0 * * *</property>
</config>
<nodes>

</nodes>
</azkaban-flow>

# 重新打包项目
# cd /data/azkaban/projects/wordcount
# zip -r wordcount.zip .

# 上传到Azkaban
# 打开Web控制台 -> 项目WordCount -> Upload -> 选择wordcount.zip -> Upload

# 设置定时调度
# 点击工作流名称”WordCount Flow”
# 点击”Schedule”
# 设置调度时间：每天 00:00
# 点击”Schedule”

# 查看调度计划
# 点击”Schedules” -> 查看调度计划
# 点击调度ID -> 查看详细信息

# 优化执行器参数
executor.maxThreads=200
executor.flow.threads=100
executor.queue.size=5000
executor.process.threads=100
executor.flow.threadpool.core.size=50
executor.flow.threadpool.max.size=200
executor.flow.threadpool.keepalive.seconds=120
executor.cleanup.retry.interval.millis=60000
executor.cleanup.retry.count=3
executor.flow.lru.size=2000

[mysqld]
binlog_format = ROW
innodb_buffer_pool_size = 2G
innodb_log_file_size = 512M
innodb_flush_log_at_trx_commit = 2
innodb_max_dirty_pages_pct = 75
innodb_file_per_table = 1
max_connections = 500
query_cache_size = 128M
query_cache_type = 1

# 重启MySQL服务
# systemctl restart mysqld

# 优化Azkaban数据库表
# mysql -u azkaban -pazkaban123 -e “USE azkaban; OPTIMIZE TABLE execution_flows;
OPTIMIZE TABLE execution_jobs; OPTIMIZE TABLE project_flows;”

# 查看数据库表大小
# mysql -u azkaban -pazkaban123 -e “USE azkaban; SELECT table_name,
round(((data_length + index_length) / 1024 / 1024), 2) as ‘Size (MB)’ FROM
information_schema.tables WHERE table_schema = ‘azkaban’ ORDER BY (data_length +
index_length) DESC LIMIT 10;”

# 根据服务器内存调整JVM参数
export JAVA_OPTS=”-Xms4G -Xmx8G -XX:MaxPermSize=1024m -XX:+UseG1GC
-XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/azkaban/logs”

# 编辑Web服务器内存配置
# vi /data/azkaban/azkaban-web-server-3.91.0/bin/start-web.sh

# 根据服务器内存调整JVM参数
export JAVA_OPTS=”-Xms4G -Xmx8G -XX:MaxPermSize=1024m -XX:+UseG1GC
-XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/data/azkaban/logs”

# 重启Azkaban服务
# cd /data/azkaban/azkaban-exec-server-3.91.0
# ./bin/shutdown-exec.sh
# ./bin/start-exec.sh

# 激活执行服务器
# curl -G “http://localhost:12321/executor?action=activate”

# 重启Web服务器
# cd /data/azkaban/azkaban-web-server-3.91.0
# ./bin/shutdown-web.sh
# ./bin/start-web.sh

# 清理历史执行数据
# mysql -u azkaban -pazkaban123 -e “USE azkaban; DELETE FROM execution_flows
WHERE end_time < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 30 DAY));" # 查看系统状态 # curl -G "http://localhost:12321/executor?action=getStats" {"stats":{"runningFlows":0,"queuedFlows":0,"completedFlows":1000,"failedFlows":100}}