欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

spark高可用,yarn

程序员文章站 2024-02-23 09:56:22
...

1.配置spark-env.sh

# 配置大哥;在二哥上面,MASTER_PORT=指的是自己
SPARK_MASTER_HOST=hadoop102
# 设置zookeepr,不能换行
SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=hadoop101:2181,hadoop102:2181,hadoop103:2181 -Dspark.deploy.zookeeper.dir=/spark"
# 告诉Spark,hadoop放到哪里面了,前题是每一个Spark,的服务器上都装有hadoop
HADOOP_CONF_DIR=/data/hadoop/hadoop-3.2.1/etc/hadoop/

配置二哥MASTER_PORT=指的是自己
SPARK_MASTER_HOST=hadoop101

2.配置slaves

#配置的小弟
hadoop103
hadoop104

3.启动

  1. 启动zookeeper
bin/zkServer.sh
  1. 启动hadoop
sbin/start-all.sh
  1. 启动spark
sbin/start-all.sh

停止spark

sbin/stop-all.sh

注意查看spark的web端的时候,由于启动了zookeeper,spark的端口与zookeeper的端口冲突,spark会把端口改成8081,在logs日志中会显示

spark高可用,yarn

# 查看网络的状态;是zookeeper占用了8080端口
netstat -anp|grep 8080

spark高可用,yarnspark高可用,yarn

启动二哥在hadoop101上启动

sbin/start-master.sh

spark高可用,yarn
4.运行示例

# --master:把任务提交给大哥;
# --executor-memory:指定executor的内存;
# --executor-cores:指定executor的cpu,核心
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
100000
# --master:把任务提交给大哥;
# --executor-memory:指定executor的内存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000
# --driver-cores:适应于集群模式;2核
# --driver-memory:适应于集群模式;500m;
bin/spark-submit \
--master spark://hadoop102:7077 \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--driver-cores 2 \
--driver-memory 500m \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
100
# --master:把任务提交给大哥;
# --executor-memory:指定executor的内存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master yarn \
--name myPi \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000

集群提交

# --master:把任务提交给大哥;
# --executor-memory:指定executor的内存;(都是workder的,executor)
# --executor-cores:指定executor的cpu,核心(都是workder的,executor)
# --deploy-mode:部署模式
bin/spark-submit \
--master yarn \
--name myPi \
--deploy-mode cluster \
--executor-memory 500m \
--total-executor-cores 2 \
--class org.apache.spark.examples.SparkPi \
examples/jars/spark-examples_2.11-2.4.4.jar \
10000