day5_Java开发Map/Reduce
本地运行
- job_local
package com.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Test {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setMapperClass(WordMapper.class);
job.setReducerClass(WordReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
//数据文件在本地
FileInputFormat.setInputPaths(job, "/Users/lujiafeng/GitRepositories/bigdata/hadoop/test/test.txt");
//输出文件在本地
FileOutputFormat.setOutputPath(job, new Path("/Users/lujiafeng/GitRepositories/bigdata/hadoop/test/out/"));
//true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
job.waitForCompletion(true);
}
}
把hdfs中的文件拉到本地来运行
- 需要启动hdfs
- 注意：这里是把hdfs文件拉到本地来运行，如果观察输出的话会观察到jobID带有local字样
- 同时这样运行的时候是不需要yarn的。
package com.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Test {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setMapperClass(WordMapper.class);
job.setReducerClass(WordReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
FileInputFormat.setInputPaths(job, "hdfs://master:9000/input/");
FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/output2/"));
//true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
job.waitForCompletion(true);
}
}
在远程服务器执行
- 临时设置
System.setProperty("HADOOP_USER_NAME", "root"); - 把Mapper和Reducer打包成jar包上传到hdfs
- 使用maven打包成jar文件
package com.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
/**
* Created by lujiafeng on 2018/10/7.
*/
public class Test {
public static void main(String[] args) throws Exception {
//System.out.println(System.getProperty("HADOOP_USER_NAME"));
System.setProperty("HADOOP_USER_NAME", "root");
//System.out.println(System.getProperty("HADOOP_USER_NAME"));
Configuration conf = new Configuration();
//添加如下内容后下面的文件输入输出位置可以使用相对路径
conf.set("fs.defaultFS", "hdfs://master:9000/");
//注意jar文件所在目录
conf.set("mapreduce.job.jar", "target/hadoopTest-1.0-SNAPSHOT.jar");
conf.set("mapreduce.framework.name", "yarn");
conf.set("yarn.resourcemanager.hostname", "master");
conf.set("mapreduce.app-submission.cross-platform", "true");
Job job = Job.getInstance(conf);
job.setMapperClass(WordMapper.class);
job.setReducerClass(WordReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
FileInputFormat.setInputPaths(job, "hdfs://master:9000/input/");
FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9000/output7"));
//true表示将运行进度等信息及时输出给用户，false的话只是等待作业结束
job.waitForCompletion(true);
}
}
