Shulou(Shulou.com)06/01 Report--

Editor to share with you how to customize hadoop GroupComparator to achieve the maximum, I believe most people do not know much about it, so share this article for your reference, I hope you can learn a lot after reading this article, let's go to know it!

Import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.io.WritableComparator;import org.apache.hadoop.mapreduce.Job Import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat / * * for undercut test data, the first field is the order number, the second field is id, and the third field is price The data is separated by the tab key * it is required to find out the record with the same order number and the largest price: O0000123 1234O00002 1243435O00003 126334O00004 1278976O00003 126635O00002 12923O00001 131131111O00002 13266O00003 13313488O00005 135900 the results are as follows: O00001231234.0O00002 1243435.0O00003 1278976.0O00005 135900 * * / public class GroupComparatorMian {static final Log LOG = LogFactory.getLog (GroupComparatorMian.class) Public static void main (String [] args) throws IOException, ClassNotFoundException, InterruptedException {Configuration conf = new Configuration (); Job job = Job.getInstance (conf); job.setJarByClass (GroupComparatorMian.class); job.setMapperClass (GroupComparatorMapper.class); job.setReducerClass (GroupComparatorReducer.class); job.setOutputKeyClass (OrderBean.class) Job.setOutputValueClass (NullWritable.class); job.setGroupingComparatorClass (CustGroupComparator.class); String jobName = "'Customize groupcomparator test'"; job.setJobName (jobName); FileInputFormat.addInputPath (job, new Path (args [0])); FileOutputFormat.setOutputPath (job, new Path (args [1])) Boolean bb = job.waitForCompletion (true); if (bb) {LOG.info ("Job" + jobName + "is done.");} else {LOG.info ("Job" + jobName + "is going wrong,now exit.") System.exit (0);} class CustGroupComparator extends WritableComparator {public CustGroupComparator () {super (OrderBean.class,true);} @ Override public int compare (WritableComparable a, WritableComparable b) {OrderBean oa = (OrderBean) a; OrderBean ob = (OrderBean) b Return oa.getOrder_id () .compareTo (ob.getOrder_id ());}} class OrderBean implements WritableComparable {private String order_id; private String id; private double prise Public OrderBean () {} public OrderBean (String order_id,String id,double prise) {this.order_id = order_id; this.id = id; this.prise = prise;} public String getOrder_id () {return order_id } public void setOrder_id (String order_id) {this.order_id = order_id;} public String getId () {return id;} public void setId (String id) {this.id = id;} public double getPrise () {return prise } public void setPrise (double prise) {this.prise = prise;} @ Override public void write (DataOutput out) throws IOException {out.writeUTF (order_id); out.writeUTF (id); out.writeDouble (prise) } @ Override public void readFields (DataInput in) throws IOException {this.order_id = in.readUTF (); this.id = in.readUTF (); this.prise = in.readDouble ();} @ Override public int compareTo (OrderBean o) {int cnt = this.order_id.compareTo (o.getOrder_id ()) If (cnt==0) {cnt= (int) (- (this.prise- o.getPrise ();} return cnt;} @ Override public String toString () {return order_id + "\ t" + id + "\ t" + prise }} class GroupComparatorMapper extends Mapper {NullWritable nul = NullWritable.get (); OrderBean ob = new OrderBean (); @ Override protected void map (LongWritable key, Text value, Mapper.Context context) throws IOException, InterruptedException {String [] split = value.toString () .split ("\ t") Ob.setOrder_id (split [0]); ob.setId (split [1]); ob.setPrise (Double.parseDouble (split [2])); context.write (ob, nul);} class GroupComparatorReducer extends Reducer {NullWritable nul = NullWritable.get () @ Override protected void reduce (OrderBean bean, Iterable iter, Reducer.Context context) throws IOException, InterruptedException {context.write (bean, nul);}} these are all the contents of the article "how to customize the GroupComparator implementation for hadoop to maximize the value". Thank you for reading! I believe we all have a certain understanding, hope to share the content to help you, if you want to learn more knowledge, welcome to follow the industry information channel!

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.