pom.xml 内容：
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.mobvista</groupId> <artifactId>mr-orc-test</artifactId> <version>1.0-SNAPSHOT</version> <name>mr-orc-test</name> <!-- FIXME change it to the project's website --> <url>http://www.example.com</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> </properties> <dependencies> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>3.8.1</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.2</version> <exclusions> <exclusion> <artifactId>avro</artifactId> <groupId>org.apache.avro</groupId> </exclusion> </exclusions> <!-- <scope>provided</scope> --> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.2</version> <!-- <scope>provided</scope> --> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.7.2</version> <!-- <scope>provided</scope> --> </dependency> <dependency> <groupId>com.twitter</groupId> <artifactId>parquet-hadoop</artifactId> <version>1.6.0</version> <scope>provided</scope> </dependency> <dependency> <groupId>com.twitter</groupId> <artifactId>parquet-avro</artifactId> <version>1.6.0</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro-mapred</artifactId> <version>1.7.6</version> <exclusions> <exclusion> <artifactId>avro</artifactId> <groupId>org.apache.avro</groupId> </exclusion> </exclusions> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> <version>0.13.1</version> <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.orc</groupId> <artifactId>orc-mapreduce</artifactId> <version>1.5.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.7.1</version> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> <version>2.6</version> <configuration> <descriptorRefs> <descriptorRef>jar-with-dependencies</descriptorRef> </descriptorRefs> </configuration> <executions> <execution> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> </executions> </plugin> </plugins> </build> </project>
代码逻辑：
package com.mobvista; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.orc.TypeDescription; import org.apache.orc.mapred.OrcStruct; public class ORCSample2 { public static class ORCMapper extends Mapper<NullWritable, OrcStruct, Text, Text> { public void map(NullWritable key, OrcStruct value, Context output) throws IOException, InterruptedException { output.write((Text) value.getFieldValue(1), (Text) value.getFieldValue(2)); } } public static class ORCReducer extends Reducer<Text, Text, NullWritable, OrcStruct> { private TypeDescription schema = TypeDescription .fromString("struct<name:string,mobile:string>"); private OrcStruct pair = (OrcStruct) OrcStruct.createValue(schema); private final NullWritable nw = NullWritable.get(); public void reduce(Text key, Iterable<Text> values, Context output) throws IOException, InterruptedException { for (Text val : values) { pair.setFieldValue(0, key); pair.setFieldValue(1, val); output.write(nw, pair); } } } public static void main(String args[]) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: wordcount <in> [<in>...] <out>"); System.exit(2); } conf.set("orc.mapred.output.schema","struct<name:string,mobile:string>"); Job job = Job.getInstance(conf, "ORC Test"); System.out.println("----libjars:" + conf.get("tmpjars")); System.out.println("mr contatinerclasspath: " + conf.get("mapreduce.application.classpath")); // job.addFileToClassPath(null); // DistributedCache.addCacheFile(new URI("s3://mob-emr-test/dataplatform/DataWareHouse/mobvista_dwh/tables/perm/orc_classpath/orc-mapreduce-1.5.0.jar"), conf); System.out.println("===mapreduce.job.cache.files===:" + conf.get("mapreduce.job.cache.files")); System.out.println("=====mapreduce.job.cache.files:======:" + conf.get("mapreduce.job.cache.files")); System.out.println("=====mapreduce.job.classpath.files:======:" + conf.get("mapreduce.job.classpath.files")); System.out.println("=====mapreduce.job.inputformat.class===:" + conf.get("mapreduce.job.inputformat.class")); System.out.println("=====mapreduce.job.outputformat.class===:" + conf.get("mapreduce.job.outputformat.class")); System.out.println("====mapreduce.job.output.value.class====:" + conf.get("mapreduce.job.output.value.class")); job.setJarByClass(ORCSample.class); job.setMapperClass(ORCMapper.class); job.setReducerClass(ORCReducer.class); /* job.setInputFormatClass(org.apache.orc.mapreduce.OrcInputFormat.class); job.setOutputFormatClass(org.apache.orc.mapreduce.OrcOutputFormat.class);*/ job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(NullWritable.class); // job.setOutputValueClass(OrcStruct.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } }提交命令：
hadoop jar mr-orc-test-1.0-SNAPSHOT.jar com.mobvista.ORCSample2 \ ** 主类**
-D mapreduce.job.output.value.class=org.apache.orc.mapred.OrcStruct \ ** 输出类型为ORC文件时需指定**
-D mapreduce.job.inputformat.class=org.apache.orc.mapreduce.OrcInputFormat \ ** 输入文件的解析类**
-D mapreduce.job.outputformat.class=org.apache.orc.mapreduce.OrcOutputFormat \ ** 输出文件的解析类**
-libjars /data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-core-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-mapreduce-1.5.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/aircompressor-0.10.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/hive-storage-api-2.6.0.jar,/data/azkaban-hadoop/command-home/hadoop/hadoop-lib/orc-shims-1.5.0.jar \ ** 线下azkaban环境引用jars**
s3://mob-emr-test/dataplatform/DataWareHouse/mobvista_dwh/tables/perm/t_test_orc/7aa5ab82-7dd1-42cc-aa1d-3bd6beb5c6a2-000000 s3://mob-emr-test/dataplatform/DataWareHouse/mobvista_dwh/tables/perm/t_test_orc_new ** 输入 和 输出**
注意点：切忌代码里面 不要出现下面代码 (要通过上面-D来指定参数)
job.setInputFormatClass(org.apache.orc.mapreduce.OrcInputFormat.class);
job.setOutputFormatClass(org.apache.orc.mapreduce.OrcOutputFormat.class);
job.setOutputValueClass(org.apache.orc.mapred.OrcStruct.class);
版权声明：本文为andyliuzhii原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接和本声明。