# 创建调度池配置
$ cat > /bigdata/app/spark/conf/fairscheduler.xml << 'EOF'
FAIR
3
10 FAIR
1
5 FIFO
2
8
EOF

# 配置公平调度
$ cat >> /bigdata/app/spark/conf/spark-defaults.conf << 'EOF' spark.scheduler.mode=FAIR spark.scheduler.allocation.file=/bigdata/app/spark/conf/fairscheduler.xml EOF # 提交任务到指定调度池 scala> spark.sparkContext.setLocalProperty(“spark.scheduler.pool”, “fgedu_production”)

# YARN多租户队列配置
# capacity-scheduler.xml yarn.scheduler.capacity.root.fgedu_prod.capacity
50 yarn.scheduler.capacity.root.fgedu_dev.capacity
30 yarn.scheduler.capacity.root.fgedu_batch.capacity
20

# 提交到指定队列
$ spark-submit –queue fgedu_prod …
$ spark-submit –queue fgedu_dev …
$ spark-submit –queue fgedu_batch …

{
“url”: “spark://192.168.1.60:7077”,
“workers”: [
{“host”: “192.168.1.61”, “cores”: 16, “memory”: 98304, “state”: “ALIVE”},
{“host”: “192.168.1.62”, “cores”: 16, “memory”: 98304, “state”: “ALIVE”},
{“host”: “192.168.1.63”, “cores”: 16, “memory”: 98304, “state”: “ALIVE”},
{“host”: “192.168.1.64”, “cores”: 16, “memory”: 98304, “state”: “ALIVE”},
{“host”: “192.168.1.65”, “cores”: 16, “memory”: 98304, “state”: “ALIVE”}
],
“cores”: 80,
“memory”: 491520,
“activeApps”: [
{“id”: “app-20260408120000-0001”, “name”: “fgedu-app-1”, “cores”: 20, “memoryPerExecutor”: 8192}
]
}

# 查看运行中的应用
$ /bigdata/app/spark/bin/spark-submit –status app-20260408120000-0001

Application report for app-20260408120000-0001 (state: RUNNING)
Name: fgedu-app-1
Cores: 20
Memory per Executor: 8192 MB
Submit Date: Mon Apr 08 12:00:00 CST 2026
State: RUNNING
Duration: 30 minutes

# 杀死应用
$ /bigdata/app/spark/bin/spark-submit –kill app-20260408120000-0001

Killing application app-20260408120000-0001

# YARN应用管理
$ yarn application -list
$ yarn application -kill application_1680940800000_0001
$ yarn application -status application_1680940800000_0001

# 查看队列资源使用
$ yarn queue -status production

Queue Information
Queue Name: production
State: RUNNING
Capacity: 60.0%
Current Capacity: 45.0%
Maximum Capacity: 100.0%
Default Node Label Expression:
Num Active Applications: 5
Num Pending Applications: 2

# 排查步骤
# 1. 查看集群资源
$ curl http://192.168.1.60:8080/json

# 2. 查看队列资源
$ yarn queue -status production

# 3. 查看应用状态
$ yarn application -list

# 解决方案
# 1. 减少资源请求
–executor-memory 4g
–num-executors 5

# 2. 使用动态资源分配
spark.dynamicAllocation.enabled=true

# 3. 调整队列容量
# 修改capacity-scheduler.xml

# 4. 杀死低优先级应用
$ yarn application -kill application_xxx

# 排查步骤
# 1. 查看调度池配置
$ cat /bigdata/app/spark/conf/fairscheduler.xml

# 2. 查看应用资源使用
$ curl http://192.168.1.60:8080/json

# 解决方案
# 1. 调整调度池权重 3
10

# 2. 限制最大资源
spark.executor.memory 8g
spark.executor.instances 10

# 3. 使用Fair调度
spark.scheduler.mode=FAIR

# 4. 配置资源限制
spark.cores.max=20
spark.executor.memory=8g

2. 使用动态资源分配
– 根据负载自动调整
– 提高资源利用率

3. 配置公平调度
– 多用户公平共享
– 设置调度池权重

4. 监控资源使用
– 定期检查资源使用
– 及时调整配置