Hadoop 类Grep源代码注释

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.tdxx.hadoop.example;

import java.util.Random;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapred.lib.*;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


/**
 * 从input中提取与表达式相符的单词并计算词频
 * 
 * 继承自配置基类Configured,并扩展接口Tool
 * Configured类中有一个变量conf用于存储配置文件
 * Tool中只有一个方法需要实现
 * int run(String [] args)用于运行输入参数
 * hadoop jar Grep.jar /user/hadoop/20130704/grep.txt /user/hadoop/output/ 'aaa.*'
 * hadoop jar Grep.jar /user/hadoop/20130704/grep.txt /user/hadoop/output/ '[a-z.]+'
 */
public class Grep extends Configured implements Tool {
	// singleton
	private Grep() {
	} 

	public int run(String[] args) throws Exception {
		if (args.length < 3) {
			System.out.println("Grep <inDir> <outDir> <regex> [<group>]");
			ToolRunner.printGenericCommandUsage(System.out);
			return -1;
		}

		Path tempDir = new Path("grep-temp-"
				+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));

		//创建job
		JobConf grepJob = new JobConf(getConf(), Grep.class);

		try {
			//job命名
			grepJob.setJobName("grep-search");

			//设置job的输入路径
			FileInputFormat.setInputPaths(grepJob, args[0]);
			
			//设置Mapper类
			grepJob.setMapperClass(RegexMapper.class);
			
			grepJob.set("mapred.mapper.regex", args[2]);
			if (args.length == 4)
				grepJob.set("mapred.mapper.regex.group", args[3]);

			//设置Combiner类
			grepJob.setCombinerClass(LongSumReducer.class);
			
			//设置Reducer类
			grepJob.setReducerClass(LongSumReducer.class);

			//设置输出路径
			FileOutputFormat.setOutputPath(grepJob, tempDir);
			
			//设置输出格式
			grepJob.setOutputFormat(SequenceFileOutputFormat.class);
			
			//设置输出键的类
			grepJob.setOutputKeyClass(Text.class);
			
			//设置输出值的类
			grepJob.setOutputValueClass(LongWritable.class);

			//运行
			JobClient.runJob(grepJob);

			JobConf sortJob = new JobConf(getConf(), Grep.class);
			sortJob.setJobName("grep-sort");

			FileInputFormat.setInputPaths(sortJob, tempDir);
			sortJob.setInputFormat(SequenceFileInputFormat.class);

			sortJob.setMapperClass(InverseMapper.class);

			sortJob.setNumReduceTasks(1); // write a single file
			FileOutputFormat.setOutputPath(sortJob, new Path(args[1]));
			sortJob.setOutputKeyComparatorClass // sort by decreasing freq
					(LongWritable.DecreasingComparator.class);

			JobClient.runJob(sortJob);
		} finally {
			FileSystem.get(grepJob).delete(tempDir, true);
		}
		return 0;
	}

	public static void main(String[] args) throws Exception {
		int res = ToolRunner.run(new Configuration(), new Grep(), args);
		System.exit(res);
	}

}

补充：软件开发 , Java ,

上一个：内部排序之交换排序
下一个：总结下java经常犯的错误