Shulou(Shulou.com)06/01 Report--

This article mainly explains how to load and export data to mysql with spark sql data. Interested friends may wish to have a look. The method introduced in this paper is simple, fast and practical. Let's let Xiaobian take you to learn "how to load and export data to mysql with spark sql"!

package hgs.spark.sqlimport org.apache.spark. SparkConfimport org.apache.spark.SparkConfimport org. apache. spark.sql.SQLContextimport org. apache.spark. sql. SQLImplicitsimport org. apache. spark. sql.types.StructTypeimport org. apache. spark.sql. types. Structdimport org.apache. spark. sql. Fiels.StringTypeimport org. apache. spark. sql. types. Integrimport org. apache. spark.sql.Rowimport java.util. TestType1 {def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("sqltest1").setMaster("local") val context = new SparkContext(conf) val sqlContext = new SQLContext(context) val rdd = context.textFile("d:\\person",1) val rdd2 = rdd.map(x=>{val t = x.split(" ");person(t(0).toInt,t(1),t(2).toInt)}) val rdd3 = rdd.map(x=>{val t = x.split(" ");person(t(0).toInt,t(1),t(2).toInt)}) //The first way to create a dataframe is to import an implicit conversion import sqlContext.implicits._ val persondf = rdd2.toDF() val persondf3 = rdd3.toDF() //This method was deprecated in 2.1.0 //persondf.registerTempTable("person") //Use this function instead persondf.createOrReplaceTempView("person") persondf3.createOrReplaceTempView("person2") val result = sqlContext.sql("select a.id ,b.name from person a ,person2 b where a.id = b.id") //Print query results result.show() //or save results to file //result.write.json("d://personselect") val props = new Properties(); props.put("user", "root") props.put("password", "123456") result.write.mode("append").jdbc("jdbc:mysql://192.168.6.133:3306/hgs", "person", props) //result.write.mode(saveMode) context.stop() }}case class person(id:Int,name:String,age:Int)object sqlTest3{ def main(args: Array[String]): Unit = { val conf = new SparkConf().setAppName("sqltest3").setMaster("local") val context = new SparkContext(conf) val sqlContext = new SQLContext(context) val persontable = sqlContext.read.format("jdbc").option("url", "jdbc:mysql://192.168.6.133:3306/hgs") .option("dbtable", "person") .option("user", "root") .option("password", "123456").load().createOrReplaceTempView("person") sqlContext.sql("select id,concat(name,\"xxx\") from person").show() context.stop() }} At this point, I believe that everyone has a deeper understanding of "how to load and export data to mysql from spark sql and mysql", so let's actually operate it! Here is the website, more related content can enter the relevant channels for inquiry, pay attention to us, continue to learn!

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.