api代码示例入下
object DataFrameCase { def main(args: Array[String]): Unit = { val sparksession = SparkSession.builder().appName("DataFrameCase").master("local[2]").getorCreate() val path = "G:\\student.data" val sparkrdd = sparksession.sparkContext.textFile(path) //隐式转化 import sparksession.implicits._ val studentDF = sparkrdd.map(_.split("\\|")).map(line => Student(line(0).toInt,line(1),line(2),line(3))).toDF() val studentDF2 = sparkrdd.map(_.split("\\|")).map(line => Student(line(0).toInt,line(1),line(2),line(3))).toDF() studentDF.show() studentDF.take(10) studentDF.filter("name='' OR name = 'NULL'").show() //取前30行,不截取字段值 studentDF.show(30,false) studentDF.where("name='' OR name = 'NULL'").show() studentDF.filter("SUBSTRING(name,0,1)='t'").show() studentDF.sort("name").show() //可以传入列,或直接传入字段 studentDF.sort(studentDF("name").desc,studentDF("id")).show() studentDF.select(studentDF.col("name").as("studentname")).show() //join studentDF.join(studentDF2,studentDF.col("id") === studentDF2.col("id")).show() } case class Student(id:Int,name:String,phone:String,email:String) }
1|burke|18722323|sdfsdf.com 2|turke|18722323|sdfsdf.com 3|turke|18722323|sdfsdf.com 4|curke|18722323|sdfsdf.com 5|burke|18722323|sdfsdf.com 6|burke|18722323|sdfsdf.com 7|burke|18722323|sdfsdf.com 8|murke|18722323|sdfsdf.com 9|burke|18722323|sdfsdf.com 10||1872563|sdfsdf.com 11||2232323|dsfsdf.com 12|NULL|937489|dsfsdew.com
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。