海豚调度器初次使用 .......

一，部署部分省略，官网有详细的教程，顺着步骤做就好了。

二，先说说调用spark运行wordcount案例流程

1，编写代码

package com.sjb.example


import org.apache.log4j.Logger
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession


 //  com.sjb.example.WordCount
object WordCount {
   var LOGGER:Logger = Logger.getLogger(WordCount.getClass)
  def main(args: Array[String]): Unit = {
//    val wordFile = "file:\\C:\\Users\\Administrator\\Desktop\\test.txt"
//    val wordFile = "file:/wyyt/software/flink-1.11.2/test.txt"
//    val wordFile = "/spark/test/data/test.txt"
    val wordFile = "/dolphinscheduler/test/resources/spark_jar/word.txt"
    System.setProperty("HADOOP_USER_NAME", "hive")
//    System.setProperty("HADOOP_USER_NAME", "test")
//    System.setProperty("HADOOP_USER_NAME", "dolphinscheduler")

//    val wordFile = args(0)
    println("接受参数信息："+wordFile)
    LOGGER.error("$$$$$$接受参数信息："+wordFile)
//    val conf = new SparkConf().setAppName("wordcount").setMaster("local[*]")
    val conf = new SparkConf().setAppName("wordcount")
    conf.set("dfs.client.use.datanode.hostname", "true")
    val sc = new SparkContext(conf)
    val input = sc.textFile(wordFile)
    val lines: RDD[String] = input.flatMap(line => line.split(" "))
    val count: RDD[(String, Int)] = lines.map(word => (word, 1)).reduceByKey { case (x, y) => x + y }

    count.collect().foreach(println)
    sc.stop()
  }


}

2，上传jar包