09-Hadoop编译

[root@hadoop001 sourcecode]# pwd

/opt/sourcecode

[root@hadoop001 sourcecode]# ll

total 33756

-rw-r--r--. 1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# tar -xzvf hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# ll

total 33760

drwxr-xr-x. 17 root root    4096 Jun  2 14:13 hadoop-2.8.1-src

-rw-r--r--.  1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz

[root@hadoop001 sourcecode]# cd hadoop-2.8.1-src

[root@hadoop001 hadoop-2.8.1-src]# ll

total 224

-rw-rw-r--.  1 root root 15623 May 24 07:14 BUILDING.txt

drwxr-xr-x.  4 root root  4096 Aug 20 12:44 dev-support

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-assemblies

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-build-tools

drwxrwxr-x.  2 root root  4096 Aug 20 12:44 hadoop-client

drwxr-xr-x. 10 root root  4096 Aug 20 12:44 hadoop-common-project

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-dist

drwxr-xr-x.  8 root root  4096 Aug 20 12:44 hadoop-hdfs-project

drwxr-xr-x.  9 root root  4096 Aug 20 12:44 hadoop-mapreduce-project

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-maven-plugins

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-minicluster

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-project

drwxr-xr-x.  2 root root  4096 Aug 20 12:44 hadoop-project-dist

drwxr-xr-x. 18 root root  4096 Aug 20 12:44 hadoop-tools

drwxr-xr-x.  3 root root  4096 Aug 20 12:44 hadoop-yarn-project

-rw-rw-r--.  1 root root 99253 May 24 07:14 LICENSE.txt

-rw-rw-r--.  1 root root 15915 May 24 07:14 NOTICE.txt

drwxrwxr-x.  2 root root  4096 Jun  2 14:24 patchprocess

-rw-rw-r--.  1 root root 20477 May 29 06:36 pom.xml

-rw-r--r--.  1 root root  1366 May 20 13:30 README.txt

-rwxrwxr-x.  1 root root  1841 May 24 07:14 start-build-env.sh

[root@hadoop001 hadoop-2.8.1-src]#

[root@hadoop001 hadoop-2.8.1-src]# cat BUILDING.txt

Build instructions for Hadoop

Requirements:

* Unix System

* JDK 1.7+

* Maven 3.0 or later

* Findbugs 1.3.9 (if running findbugs)

* ProtocolBuffer 2.5.0

* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac

* Zlib devel (if compiling native code)

* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)

* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)

* Internet connection for first build (to fetch all Maven and Hadoop dependencies)

2.JAVA安装

[root@hadoop001 ~]# rz #上传jdk-8u45-linux-x64.gz

[root@hadoop001 ~]# mkdir -p /usr/java

[root@hadoop001 ~]# mv jdk-8u45-linux-x64.gz /usr/java

[root@hadoop001 ~]# cd /usr/java

[root@hadoop001 ~]# tar -xzvf jdk-8u45-linux-x64.gz

#修改用户和用户组

[root@hadoop002 java]# ll

total 169388

drwxr-xr-x. 8 uucp  143      4096 Apr 11  2015 jdk1.8.0_45

-rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz

[root@hadoop002 java]# chown -R root:root jdk1.8.0_45

[root@hadoop002 java]# ll

total 169388

drwxr-xr-x. 8 root root      4096 Apr 11  2015 jdk1.8.0_45

-rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz

[root@hadoop002 java]#

[root@hadoop001 java]# vi /etc/profile

export JAVA_HOME=/usr/java/jdk1.8.0_45

export PATH=$JAVA_HOME/bin:$PATH

[root@hadoop001 java]# source /etc/profile

[root@hadoop001 java]# which java

/usr/java/jdk1.8.0_45/bin/java

[root@hadoop001 java]# java -version

java version "1.8.0_45"

Java(TM) SE Runtime Environment (build 1.8.0_45-b14)

Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)

[root@hadoop001 java]#

3.Maven安装

[root@hadoop001 ~]# mkdir -p /opt/sourcecode /opt/software

[root@hadoop001 ~]# cd /opt/software/

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring apache-maven-3.3.9-bin.zip...

  100%    8415 KB    8415 KB/sec    00:00:01      0 Errors 

[root@hadoop001 software]#

[root@hadoop001 software]# ll

total 8432

-rw-r--r--. 1 root root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip

[root@hadoop001 software]# unzip apache-maven-3.3.9-bin.zip

[root@hadoop001 java]# vi /etc/profile

export MAVEN_HOME=/opt/software/apache-maven-3.3.9

export MAVEN_OPTS="-Xms256m -Xmx512m"

export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

[root@hadoop001 ~]# mvn -version

Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-11T00:41:47+08:00)

Maven home: /opt/software/apache-maven-3.3.9

Java version: 1.8.0_45, vendor: Oracle Corporation

Java home: /usr/java/jdk1.8.0_45/jre

Default locale: en_US, platform encoding: UTF-8

OS name: "linux", version: "2.6.32-431.el6.x86_64", arch: "amd64", family: "unix"

[root@hadoop001 ~]#

4.protobuf安装

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring protobuf-2.5.0.tar.gz...

  100%    2345 KB    2345 KB/sec    00:00:01      0 Errors 

[root@hadoop001 software]# tar -xzvf protobuf-2.5.0.tar.gz

[root@hadoop001 software]# ll

total 10792

drwxr-xr-x.  6 root  root    4096 Nov 10  2015 apache-maven-3.3.9

-rw-r--r--.  1 root  root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip

drwxr-xr-x. 10 109965 5000    4096 Feb 27  2013 protobuf-2.5.0

-rw-r--r--.  1 root  root 2401901 Aug 20 13:03 protobuf-2.5.0.tar.gz

[root@hadoop001 software]# cd protobuf-2.5.0

[root@hadoop001 protobuf-2.5.0]# yum install -y gcc gcc-c++ make cmake  #已经安装cmake

[root@hadoop001 protobuf-2.5.0]# ./configure --prefix=/usr/local/protobuf

[root@hadoop001 protobuf-2.5.0]# make && make install

[root@hadoop001 java]# vi /etc/profile

export PROTOC_HOME=/usr/local/protobuf

export PATH=$PROTOC_HOME/bin:$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

[root@hadoop001 protobuf-2.5.0]# source /etc/profile

[root@hadoop001 protobuf-2.5.0]# protoc --version

libprotoc 2.5.0

[root@hadoop001 protobuf-2.5.0]#

5.Findbugs安装

[root@hadoop001 software]# rz

rz waiting to receive.

Starting zmodem transfer.  Press Ctrl+C to cancel.

Transferring findbugs-1.3.9.zip...

  100%    7369 KB    7369 KB/sec    00:00:01      0 Errors 

[root@hadoop001 software]# unzip findbugs-1.3.9.zip

[root@hadoop002 software]# vi /etc/profile

export FINDBUGS_HOME=/opt/software/findbugs-1.3.9

export PATH=$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH

[root@hadoop001 software]#

[root@hadoop001 software]# source /etc/profile

[root@hadoop001 software]# findbugs -version

1.3.9

[root@hadoop001 software]#

6.其他依赖

yum install -y openssl openssl-devel svn ncurses-devel zlib-devel libtool

yum install -y snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop autoconf automake

7.编译

[root@hadoop001 sourcecode]# cd hadoop-2.8.1-src

[root@hadoop001 hadoop-2.8.1-src]# mvn clean package -Pdist,native -DskipTests -Dtar

[INFO] --- maven-site-plugin:3.5:attach-descriptor (attach-descriptor) @ hadoop-dist ---

[INFO] ------------------------------------------------------------------------

[INFO] Reactor Summary:

[INFO]

[INFO] Apache Hadoop Main ................................. SUCCESS [  7.770 s]

[INFO] Apache Hadoop Build Tools .......................... SUCCESS [  4.990 s]

[INFO] Apache Hadoop Project POM .......................... SUCCESS [  4.920 s]

[INFO] Apache Hadoop Annotations .......................... SUCCESS [ 12.936 s]

[INFO] Apache Hadoop Assemblies ........................... SUCCESS [  0.809 s]

[INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [  5.013 s]

[INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [ 17.617 s]

[INFO] Apache Hadoop MiniKDC .............................. SUCCESS [ 23.551 s]

[INFO] Apache Hadoop Auth ................................. SUCCESS [ 26.352 s]

[INFO] Apache Hadoop Auth Examples ........................ SUCCESS [ 10.305 s]

[INFO] Apache Hadoop Common ............................... SUCCESS [05:58 min]

[INFO] Apache Hadoop NFS .................................. SUCCESS [ 19.105 s]

[INFO] Apache Hadoop KMS .................................. SUCCESS [ 27.479 s]

[INFO] Apache Hadoop Common Project ....................... SUCCESS [  0.174 s]

[INFO] Apache Hadoop HDFS Client .......................... SUCCESS [01:32 min]

[INFO] Apache Hadoop HDFS ................................. SUCCESS [06:42 min]

[INFO] Apache Hadoop HDFS Native Client ................... SUCCESS [ 20.370 s]

[INFO] Apache Hadoop HttpFS ............................... SUCCESS [ 40.720 s]

[INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [ 21.298 s]

[INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [ 11.818 s]

[INFO] Apache Hadoop HDFS Project ......................... SUCCESS [  0.148 s]

[INFO] Apache Hadoop YARN ................................. SUCCESS [  0.192 s]

[INFO] Apache Hadoop YARN API ............................. SUCCESS [ 41.517 s]

[INFO] Apache Hadoop YARN Common .......................... SUCCESS [01:19 min]

[INFO] Apache Hadoop YARN Server .......................... SUCCESS [  0.192 s]

[INFO] Apache Hadoop YARN Server Common ................... SUCCESS [ 19.421 s]

[INFO] Apache Hadoop YARN NodeManager ..................... SUCCESS [ 42.398 s]

[INFO] Apache Hadoop YARN Web Proxy ....................... SUCCESS [  8.925 s]

[INFO] Apache Hadoop YARN ApplicationHistoryService ....... SUCCESS [ 16.120 s]

[INFO] Apache Hadoop YARN ResourceManager ................. SUCCESS [ 57.415 s]

[INFO] Apache Hadoop YARN Server Tests .................... SUCCESS [  3.869 s]

[INFO] Apache Hadoop YARN Client .......................... SUCCESS [ 14.325 s]

[INFO] Apache Hadoop YARN SharedCacheManager .............. SUCCESS [ 11.814 s]

[INFO] Apache Hadoop YARN Timeline Plugin Storage ......... SUCCESS [ 10.027 s]

[INFO] Apache Hadoop YARN Applications .................... SUCCESS [  0.276 s]

[INFO] Apache Hadoop YARN DistributedShell ................ SUCCESS [  8.333 s]

[INFO] Apache Hadoop YARN Unmanaged Am Launcher ........... SUCCESS [  5.473 s]

[INFO] Apache Hadoop YARN Site ............................ SUCCESS [  0.160 s]

[INFO] Apache Hadoop YARN Registry ........................ SUCCESS [ 13.204 s]

[INFO] Apache Hadoop YARN Project ......................... SUCCESS [  8.106 s]

[INFO] Apache Hadoop MapReduce Client ..................... SUCCESS [  0.514 s]

[INFO] Apache Hadoop MapReduce Core ....................... SUCCESS [01:09 min]

[INFO] Apache Hadoop MapReduce Common ..................... SUCCESS [ 40.479 s]

[INFO] Apache Hadoop MapReduce Shuffle .................... SUCCESS [ 10.304 s]

[INFO] Apache Hadoop MapReduce App ........................ SUCCESS [ 27.335 s]

[INFO] Apache Hadoop MapReduce HistoryServer .............. SUCCESS [ 19.910 s]

[INFO] Apache Hadoop MapReduce JobClient .................. SUCCESS [ 16.657 s]

[INFO] Apache Hadoop MapReduce HistoryServer Plugins ...... SUCCESS [  4.591 s]

[INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [ 12.346 s]

[INFO] Apache Hadoop MapReduce ............................ SUCCESS [  5.966 s]

[INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [  7.940 s]

[INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [ 15.245 s]

[INFO] Apache Hadoop Archives ............................. SUCCESS [  5.380 s]

[INFO] Apache Hadoop Archive Logs ......................... SUCCESS [  5.812 s]

[INFO] Apache Hadoop Rumen ................................ SUCCESS [ 11.785 s]

[INFO] Apache Hadoop Gridmix .............................. SUCCESS [  9.890 s]

[INFO] Apache Hadoop Data Join ............................ SUCCESS [  5.784 s]

[INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [  3.254 s]

[INFO] Apache Hadoop Extras ............................... SUCCESS [  5.495 s]

[INFO] Apache Hadoop Pipes ................................ SUCCESS [ 10.630 s]

[INFO] Apache Hadoop OpenStack support .................... SUCCESS [ 11.234 s]

[INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [ 14.060 s]

[INFO] Apache Hadoop Azure support ........................ SUCCESS [ 10.535 s]

[INFO] Apache Hadoop Client ............................... SUCCESS [ 13.519 s]

[INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [  2.164 s]

[INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [ 10.405 s]

[INFO] Apache Hadoop Tools Dist ........................... SUCCESS [ 11.514 s]

[INFO] Apache Hadoop Azure Data Lake support .............. SUCCESS [  9.201 s]

[INFO] Apache Hadoop Tools ................................ SUCCESS [  0.129 s]

[INFO] Apache Hadoop Distribution ......................... SUCCESS [01:07 min]

[INFO] ------------------------------------------------------------------------

[INFO] BUILD SUCCESS

[INFO] ------------------------------------------------------------------------

[INFO] Total time: 31:41 min

[INFO] Finished at: 2017-12-10T11:55:28+08:00

[INFO] Final Memory: 166M/494M

[INFO] ------------------------------------------------------------------------

[root@rzdatahadoop001 hadoop-2.8.1-src]#

download需要3.5h

不想download就使用.m2.tar.gz -->解压-->root家目录

8.编译好的tar包

/opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target/hadoop-2.8.1.tar.gz

提醒：

1、有时候编译过程中会出现下载某个包的时间太久，这是由于连接网站的过程中会出现假死，

此时按ctrl+c，重新运行编译命令。

2、如果出现缺少了某个文件的情况，则要先清理maven(使用命令 mvn clean) 再重新编译。

©著作权归作者所有,转载或内容合作请联系作者
  • 人面猴
    序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
    沈念sama阅读 203,547评论 6赞 477
  • 死咒
    序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
    沈念sama阅读 85,399评论 2赞 381
  • 救了他两次的神仙让他今天三更去死
    文/潘晓璐 我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
    开封第一讲书人阅读 150,428评论 0赞 337
  • 道士缉凶录：失踪的卖姜人
    文/不坏的土叔 我叫张陵，是天一观的道长。 经常有香客问我，道长，这世上最难降的妖魔是什么？ 我笑而不...
    开封第一讲书人阅读 54,599评论 1赞 274
  • 港岛之恋（遗憾婚礼）
    正文 为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
    茶点故事阅读 63,612评论 5赞 365
  • 恶毒庶女顶嫁案：这布局不是一般人想出来的
    文/花漫 我一把揭开白布。 她就那样静静地躺着，像睡着了一般。 火红的嫁衣衬着肌肤如雪。 梳的纹丝不乱的头发上，一...
    开封第一讲书人阅读 48,577评论 1赞 281
  • 城市分裂传说
    那天，我揣着相机与录音，去河边找鬼。 笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。 我是一名探鬼主播，决...
    沈念sama阅读 37,941评论 3赞 395
  • 双鸳鸯连环套：你想象不到人心有多黑
    文/苍兰香墨 我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
    开封第一讲书人阅读 36,603评论 0赞 258
  • 万荣杀人案实录
    序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
    沈念sama阅读 40,852评论 1赞 297
  • 护林员之死
    正文 独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋 以下内容为张勋视角 年9月15日...
    茶点故事阅读 35,605评论 2赞 321
  • 白月光启示录
    正文 我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。 大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
    茶点故事阅读 37,693评论 1赞 329
  • 活死人
    序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
    沈念sama阅读 33,375评论 4赞 318
  • 日本核电站爆炸内幕
    正文 年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
    茶点故事阅读 38,955评论 3赞 307
  • 男人毒药：我在死后第九天来索命
    文/蒙蒙 一、第九天 我趴在偏房一处隐蔽的房顶上张望。 院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
    开封第一讲书人阅读 29,936评论 0赞 19
  • 一桩弑父案，背后竟有这般阴谋
    文/苍兰香墨 我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。 一阵脚步声响...
    开封第一讲书人阅读 31,172评论 1赞 259
  • 情欲美人皮
    我被黑心中介骗来泰国打工， 没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。 一个月前我还...
    沈念sama阅读 43,970评论 2赞 349
  • 代替公主和亲
    正文 我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。 传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
    茶点故事阅读 42,414评论 2赞 342

推荐阅读更多精彩内容