windows系统IEDA构建maven工程编写HDFS或Mapreduce代码，打包jar到linux提交

目录1. 创建maven工程2.编写pom.xml文件3.编写mapreduce代码4.打包项目为jar文件5.导入linux6.执行代码7. hdfs代码编写与提交1. 创建maven工程设定nameArchetype选择第一个 maven-archetype-archetypejdk: 如果有1.8 就选择1.8 目前我hadoop所在linux用的是1.8。没有选择也关系点击 create 按钮即可2.编写pom.xml文件打开pom.xml文档。project/project内添加 下面属性和依赖由于我linux的jdk是1.8windows的jdk是17所以修改了maven.compiler.source1.8/maven.compiler.source maven.compiler.target1.8/maven.compiler.targethadoop.version 根据你自己实际的hadoop集群版本号填写即可。具体如下properties maven.compiler.source1.8/maven.compiler.source maven.compiler.target1.8/maven.compiler.target project.build.sourceEncodingUTF-8/project.build.sourceEncoding hadoop.version3.2.4/hadoop.version /properties dependencies dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-common/artifactId version${hadoop.version}/version /dependency dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs/artifactId version${hadoop.version}/version /dependency dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId version${hadoop.version}/version /dependency dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-core/artifactId version${hadoop.version}/version /dependency /dependencies更新maven库右上角m按钮点击更新第一次需要比较长时间下载hadoop依赖。3.编写mapreduce代码首先构建maven source directories双击下面的java即可此时在main文件夹下会新建出一个java文件夹基于java文件夹构建我们的package或者直接写.java文件。MyMapper类如下import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import java.io.IOException; import java.util.StringTokenizer; public class MyMapper extends MapperObject, Text, Text, IntWritable { private final static IntWritable one new IntWritable(1); private Text word new Text(); public void map(Object key, Text value, Context context) throws IOException, InterruptedException { //每次处理一行一个mapper里的value为一行,key为该行在文件中的偏移量 StringTokenizer iter new StringTokenizer(value.toString()); while (iter.hasMoreTokens()) { word.set(iter.nextToken()); // 向context中写入word, 1 context.write(word, one); System.out.println(word); } } }MyReducer类如下import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; import java.io.IOException; public class MyReducer extends ReducerText, IntWritable, Text, IntWritable { private IntWritable result new IntWritable(); public void reduce(Text key, IterableIntWritable values, Context context) throws IOException, InterruptedException{ int sum 0; for (IntWritable val : values) { sum val.get(); } result.set(sum); context.write(key, result); } }MyDriver类import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class Driver { public static void main(String[] args) throws Exception{ Configuration conf new Configuration(); Job job Job.getInstance(conf, word_count); job.setJarByClass(Driver.class); job.setMapperClass(MyMapper.class); //此处的Combine操作意为即第每个mapper工作完了先局部reduce一下最后再全局reduce job.setCombinerClass(MyReducer.class); job.setReducerClass(MyReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); //第0个参数是输入目录第1个参数是输出目录 //先判断output path是否存在如果存在则删除 Path path new Path(args[1]);// FileSystem fileSystem path.getFileSystem(conf); if (fileSystem.exists(path)) { fileSystem.delete(path, true); } //设置输入目录和输出目录 FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true)?0:1); } }4.打包项目为jar文件点击 File 下的 Project Structure按下面红色框从左到右选择。选择 main class然后一路点击 ok 按钮即可。重新Build Artifas–jar包此时工程目录多出out目录已经生成jar文件了5.导入linux使用工具或者直接拖动方式若支持的话将上图中的HDFS_Mapreduce.jar文件导入linux系统。6.执行代码/input/word.txt 为hdfs上的输入路径可以为具体文件或者目录可以先创建input文件夹 hadoop fs -mkdir /input然后上传本地某个文件如word.txt 到hdfs的/input中 hadoop fs -put word.txt /input这样 hdfs的 /input 目录下就有需要的word.txt 文件了。/output0330为程序的hdfs输出路径执行程序前确保该路径不存在也可以写完整路径hdfs://主节点的IP或别名:9000/input/word.txt查看输出的结果7. hdfs代码编写与提交前面完成mapreduce词频统计的编写如果操作的是hdfs过程类似仅仅是代码不一样和执行jar文件给定参数不一样而已如简单列举hadoop目录下的文件的代码conf.set语句可以不给我里面的主节点用别名master代替端口号是9000根据实际情况修改。import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; public class GetData { public static void main(String[] args) throws IOException { Configuration conf new Configuration(); conf.set(fs.defaultFS, hdfs://master:9000); conf.set(fs.hdfs.impl, org.apache.hadoop.hdfs.DistributedFileSystem); FileSystem fs FileSystem.get(conf); FileStatus[] files fs.listStatus(new Path(/)); for(FileStatus fileStatus : files) { System.out.println(fileStatus.getPath().toString()); } fs.close(); } }执行程序程序不需要传入参数直接执行即可。hadoop jar hdfs.jar