Shulou(Shulou.com)05/31 Report--

This article mainly introduces how to achieve GenericWritable in hadoop, has a certain reference value, interested friends can refer to, I hope you can learn a lot after reading this article, the following let the editor take you to understand it.

Package com.test;import java.io.IOException;import java.util.Iterator;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.GenericWritable;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.Writable;import org.apache.hadoop.mapreduce.Job Import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner / * Business scenario: * contains two files. Words in the two files are separated in different ways, but count the number of common words appearing in the two files * * File Source 1, comma delimited text1.txt * hello,what * you, * File Source 2 Tabs separate text2.txt * girl boy * father mother * / public class WordCountGenericWritable extends Configured implements Tool {public static class Map1 extends Mapper {public void map (LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line = value.toString () StringTokenizer st = new StringTokenizer (line, ","); while (st.hasMoreElements ()) {context.write (new Text (st.nextElement (). ToString ()), new MyGenericWritable (new LongWritable (1));} public static class Map2 extends Mapper {public void map (Text key, Text value, Context context) throws IOException, InterruptedException {context.write (key, new MyGenericWritable (new Text ("1") Context.write (value, new MyGenericWritable (new Text ("1"));}} public static class Reduce extends Reducer {public void reduce (Text key, Iterable values, Context context) throws IOException, InterruptedException {int count = 0; Iterator it = values.iterator (); while (it.hasNext ()) {MyGenericWritable myGw = it.next (); Writable value = myGw.get () If (value instanceof LongWritable) {count = count + Long.valueOf (LongWritable) value). Get ()). IntValue ();} if (value instanceof Text) {count = count + Long.valueOf (Text) value). ToString ()). IntValue ();} context.write (key, new IntWritable (count));} public int run (String [] args) throws IOException, InterruptedException, ClassNotFoundException {Configuration conf = this.getConf () Job job = new Job (conf); job.setJobName (WordCountGenericWritable.class.getSimpleName ()); job.setJarByClass (WordCountGenericWritable.class); MultipleInputs.addInputPath (job, new Path ("hdfs://grid131:9000/text1.txt"), TextInputFormat.class, Map1.class); MultipleInputs.addInputPath (job, new Path ("hdfs://grid131:9000/text2.txt"), KeyValueTextInputFormat.class, Map2.class); FileOutputFormat.setOutputPath (job, new Path (args [1])); job.setReducerClass (Reduce.class) Job.setOutputFormatClass (TextOutputFormat.class); / / when the output type of map is inconsistent with that of reduce, you need to set the map output type job.setMapOutputKeyClass (Text.class); job.setMapOutputValueClass (MyGenericWritable.class); job.setOutputKeyClass (Text.class); job.setOutputValueClass (IntWritable.class); job.waitForCompletion (true); return job.isSuccessful ()? 0:1 } public static void main (String [] args) throws Exception {int exit = ToolRunner.run (new WordCount (), args); System.exit (exit);}} class MyGenericWritable extends GenericWritable {public MyGenericWritable () {} public MyGenericWritable (LongWritable longWritable) {super.set (longWritable);} public MyGenericWritable (Text text) {super.set (text);} @ Override protected Class

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.