Shulou(Shulou.com)06/01 Report--

The editor will share with you how to use spark and hbase. I hope you will get something after reading this article. Let's discuss it together.

Package hgs.spark.hbaseimport org.apache.spark.SparkConfimport org.apache.spark.SparkContextimport org.apache.hadoop.conf.Configurationimport org.apache.hadoop.hbase.HBaseConfigurationimport org.apache.spark.rdd.NewHadoopRDDimport org.apache.hadoop.hbase.mapreduce.TableInputFormatobject HbaseTest {def main (args: Array [String]): Unit = {val conf = new SparkConf conf.setMaster ("local") .setAppName ("local") val context = new SparkContext (conf) val hadoopconf = new HBaseConfiguration hadoopconf.set ("hbase.zookeeper.quorum" "bigdata01:2181,bigdata02:2181,bigdata03:2181") hadoopconf.set ("hbase.zookeeper.property.clientPort", "2181") val tableName = "test1" hadoopconf.set (TableInputFormat.INPUT_TABLE, tableName) hadoopconf.set (TableInputFormat.SCAN_ROW_START, "h") hadoopconf.set (TableInputFormat.SCAN_ROW_STOP, "x") hadoopconf.set (TableInputFormat.SCAN_COLUMN_FAMILY, "cf1") hadoopconf.set (TableInputFormat.SCAN_COLUMNS "cf1:col1,cf1:col2") / * val startrow = "h" val stoprow = "w" val scan = new Scan scan.setStartRow (startrow.getBytes) scan.setStartRow (stoprow.getBytes) val proto = ProtobufUtil.toScan (scan) val scanToString = Base64.encodeBytes (proto.toByteArray ()) println (scanToString) hadoopconf.set (TableInputFormat.SCAN, scanToString) * / val hbaseRdd = context.newAPIHadoopRDD (hadoopconf, classOf [TableInputFormat] ClassOf [org.apache.hadoop.hbase.io.ImmutableBytesWritable], classOf [org.apache.hadoop.hbase.client.Result]) hbaseRdd.foreach (x = > {val vale = x._2.getValue ("cf1" .getBytes, "col1" .getBytes) val val2 = x._2.getValue ("cf1" .getBytes, "col2" .getBytes) println (new String (vale) New String (val2)}) context.stop ()}} package hgs.spark.hbaseimport org.apache.spark.SparkConfimport org.apache.spark.SparkContextimport org.apache.hadoop.hbase.HBaseConfigurationimport org.apache.hadoop.hbase.mapred.TableOutputFormatimport org.apache.hadoop.mapred.JobConfimport org.apache.hadoop.hbase.client.Putimport org.apache.hadoop.hbase.io.ImmutableBytesWritableobject SparkToHbase {def main (args: Array [String]): Unit = {val conf = new SparkConf Conf.setMaster ("local") .setAppName ("local") val context = new SparkContext (conf) val rdd = context.parallelize (("aaaaaaa") "aaaaaaa"), ("bbbbb", "bbbbb"), 2) val hadoopconf = new HBaseConfiguration hadoopconf.set ("hbase.zookeeper.quorum", "bigdata01:2181,bigdata02:2181,bigdata03:2181") hadoopconf.set ("hbase.zookeeper.property.clientPort", "2181") hadoopconf.set (TableOutputFormat.OUTPUT_TABLE, "test1") / / hadoopconf.set (TableOutputFormat., "test1") val jobconf = new JobConf (hadoopconf This.getClass) jobconf.set (TableOutputFormat.OUTPUT_TABLE, "test1") jobconf.setOutputFormat (val exterrdd [TableOutputFormat]) val exterrdd = rdd.map (x = > {val put = new Put (x._1.getBytes) put.add ("cf1" .getBytes, "col1" .getBytes, x._2.getBytes) (new ImmutableBytesWritable Put)}) exterrdd.saveAsHadoopDataset (jobconf) context.stop ()}} finished reading this article I believe you have a certain understanding of "how to use spark and hbase". If you want to know more about it, you are welcome to follow the industry information channel. Thank you for reading!

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.