打开spark-shell命令以3号机为master: bin/spark-shell --master spark://linux-centos7-03:7077 Scala读取csv文件 var df=spark.read.format("csv").option("sep",",").option("inferSchema","true").option("header","true").load("hdfs://linux-centos7-03:8020/10061789243186.csv") 1. 展示读取...
.config("spark.sql.shuffle.partitions", "2") .getOrCreate() } /** * 读取CSV格式文本文件数据,封装到DataFrame数据集 */ def readCsvFile(spark: SparkSession, path: String, verbose: Boolean = true): DataFrame = { val dataframe: DataFrame = spark.read // 设置分隔符为逗号 .option("sep",...