Shulou(Shulou.com)06/02 Report--

Wordcount workflow

Input- > split Split- > Mapping map- > dispatching Shuffle- > shrinking reduce- > output

Hadoop jar / usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar wordcount 10803060234.txt / output

Package wordcount

Import java.io.IOException

Import org.apache.hadoop.conf.Configuration

Import org.apache.hadoop.fs.Path

Import org.apache.hadoop.io.IntWritable

Import org.apache.hadoop.io.LongWritable

Import org.apache.hadoop.io.Text

Import org.apache.hadoop.mapreduce.Job

Import org.apache.hadoop.mapreduce.lib.input.FileInputFormat

Import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat

Public class Test {

Public Test () {/ / TODO Auto-generated constructor stub} public static void main (String [] args) throws Exception {/ / TODO Auto-generated method stub Configuration conf = new Configuration (); conf.set ("fs.defaultFS", "hdfs://172.26.19.40:9000"); conf.set ("mapreduce.job.jar", "target/wc.jar"); conf.set ("mapreduce.framework.name", "yarn") Conf.set ("yarn.resourcemanager.hostname", "hmaster"); conf.set ("mapreduce.app-submission.cross-platform", "true"); Job job = Job.getInstance (conf); job.setMapperClass (WordMapper.class); job.setReducerClass (WordReducer.class); job.setMapOutputKeyClass (Text.class); job.setMapOutputValueClass (IntWritable.class); job.setOutputKeyClass (Text.class); job.setOutputValueClass (LongWritable.class) FileInputFormat.setInputPaths (job, "); FileOutputFormat.setOutputPath (job, new Path (")); job.waitForCompletion (true);}

}

Package wordcount

Import java.io.IOException

Import org.apache.hadoop.io.IntWritable

Import org.apache.hadoop.io.LongWritable

Import org.apache.hadoop.io.Text

Import org.apache.hadoop.mapreduce.Mapper

Public class WordMapper extends Mapper {

@ Overrideprotected void map (LongWritable key, Text value, Mapper.Context context) throws IOException, InterruptedException {String lineValue = value.toString (); String [] words = lineValue.split (""); IntWritable cIntWritable = new IntWritable (1); for (String word: words) {context.write (new Text (word), cIntWritable);}}

}

Package wordcount

Import java.io.IOException

Import org.apache.hadoop.io.IntWritable

Import org.apache.hadoop.io.LongWritable

Import org.apache.hadoop.io.Text

Import org.apache.hadoop.mapreduce.Reducer

Public class WordReducer extends Reducer {

@ Overrideprotected void reduce (Text key, Iterable values, Reducer.Context context) throws IOException, InterruptedException {Long tmpCount = 0L; for (IntWritable value: values) {tmpCount = tmpCount + value.get ();} context.write (key, new LongWritable (tmpCount));}

}

4.0.0

Com.skcc

Wordcount

0.0.1-SNAPSHOT

Wordcount

Count the word

UTF-8 2.7.3 junit junit 4.12 org.apache.hadoop hadoop-client ${hadoop.version} org.apache.hadoop hadoop-common ${hadoop.version} org.apache.hadoop hadoop-hdfs ${hadoop.version}

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.