How to implement KeyValueTextInputFormat in hadoop
This article mainly shows you "how to achieve KeyValueTextInputFormat in hadoop", the content is easy to understand, clear, hope to help you solve doubts, the following let the editor lead you to study and learn "how to achieve KeyValueTextInputFormat in hadoop" this article.
Package com.test;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader Import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner / * hello jim * hello tim * * final output * hello 1 * jim 1 * hello 1 * tim 1 * / public class WordCountKeyValue extends Configured implements Tool {public static class Map extends Mapper {/ * * key hello * value jim * / public void map (Text key, Text value, Context context) throws IOException, InterruptedException {context.write (key, new IntWritable (1)); context.write (value, new IntWritable (1)) }} public int run (String [] args) throws IOException, InterruptedException, ClassNotFoundException {Configuration conf = this.getConf (); / / specify the KeyValueTextInputFormat separator. The default separator is\ t / / conf.set ("mapreduce.input.keyvaluelinerecordreader.key.value.separator", "\ t"); conf.set (KeyValueLineRecordReader.KEY_VALUE_SEPERATOR, "\ t"); Job job = new Job (conf); job.setJobName (WordCountKeyValue.class.getSimpleName ()); job.setJarByClass (WordCountKeyValue.class) FileInputFormat.addInputPath (job, new Path (args [0])); FileOutputFormat.setOutputPath (job, new Path (args [1])); job.setNumReduceTasks (0); job.setMapperClass (Map.class); job.setInputFormatClass (KeyValueTextInputFormat.class); job.setOutputFormatClass (TextOutputFormat.class); job.setMapOutputKeyClass (Text.class); job.setMapOutputValueClass (IntWritable.class); job.waitForCompletion (true); return job.isSuccessful ()? 0:1 } public static void main (String [] args) throws Exception {int exit = ToolRunner.run (new WordCount (), args); System.exit (exit);}} these are all the contents of the article "how to implement KeyValueTextInputFormat in hadoop". Thank you for reading! I believe we all have a certain understanding, hope to share the content to help you, if you want to learn more knowledge, welcome to follow the industry information channel!
