Shulou(Shulou.com)06/03 Report--

01 hadoop pseudo-distributed model

Configure the environment

Node1: 192.168.1.131 CentOS Linux release 7.2

1. Install hadoop

# installation

[root@node1 ~] # vim / etc/profile.d/java.sh

Export JAVA_HOME=/usr

[root@node1 ~] # yum install java-1.7.0-openjdk-devel.x86_64

[root@node1 ~] # mkdir / bdapps

[root@node1] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/

[root@node1 ~] # cd / bdapps/

[root@node1 bdapps] # ll

Total 0

Lrwxrwxrwx 1 root root 13 Jan 7 09:24 hadoop- > hadoop-2.6.2/

Drwxr-xr-x 9 10011 10011 139 Oct 22 2015 hadoop-2.6.2

[root@node1 hadoop] # vim / etc/profile.d/hadoop.sh

Export HADOOP_PREFIX=/bdapps/hadoop

Export PATH=$PATH:$ {HADOOP_PREFIX} / bin:$ {HADOOP_PREFIX} / sbin

Export HADOOP_YARN_HOME=$ {HADOOP_PREFIX}

Export HADOOP_MAPPERD_HOME=$ {HADOOP_PREFIX}

Export HADOOP_COMMON_HOME=$ {HADOOP_PREFIX}

Export HADOOP_HDFS_HOME=$ {HADOOP_PREFIX}

[root@node1 hadoop] #. / etc/profile.d/hadoop.sh

[root@node1 hadoop] # pwd

/ bdapps/hadoop

[root@node1 hadoop] # ls

Bin etc include lib libexec LICENSE.txt NOTICE.txt README.txt sbin share

[root@node1 hadoop] # ls bin/

Container-executor hdfs mapred.cmd yarn

Hadoop hdfs.cmd rcc yarn.cmd

Hadoop.cmd mapred test-container-executor

[root@node1 hadoop] # ls include/

Hdfs.h Pipes.hh SerialUtils.hh StringUtils.hh TemplateFactory.hh

[root@node1 hadoop] # ls lib

Native

[root@node1 hadoop] # ls libexec/

Hadoop-config.cmd hdfs-config.sh mapred-config.cmd yarn-config.sh

Hadoop-config.sh httpfs-config.sh mapred-config.sh

Hdfs-config.cmd kms-config.sh yarn-config.cmd

[root@node1 hadoop] # ls sbin/

Distribute-exclude.sh start-all.cmd stop-balancer.sh

Hadoop-daemon.sh start-all.sh stop-dfs.cmd

Hadoop-daemons.sh start-balancer.sh stop-dfs.sh

Hdfs-config.cmd start-dfs.cmd stop-secure-dns.sh

Hdfs-config.sh start-dfs.sh stop-yarn.cmd

Httpfs.sh start-secure-dns.sh stop-yarn.sh

Kms.sh start-yarn.cmd yarn-daemon.sh

Mr-jobhistory-daemon.sh start-yarn.sh yarn-daemons.sh

Refresh-namenodes.sh stop-all.cmd

Slaves.sh stop-all.sh

[root@node1 hadoop] # ls etc/hadoop/

Capacity-scheduler.xml httpfs-env.sh mapred-env.sh

Configuration.xsl httpfs-log4j.properties mapred-queues.xml.template

Container-executor.cfg httpfs-signature.secret mapred-site.xml.template

Core-site.xml httpfs-site.xml slaves

Hadoop-env.cmd kms-acls.xml ssl-client.xml.example

Hadoop-env.sh kms-env.sh ssl-server.xml.example

Hadoop-metrics2.properties kms-log4j.properties yarn-env.cmd

Hadoop-metrics.properties kms-site.xml yarn-env.sh

Hadoop-policy.xml log4j.properties yarn-site.xml

Hdfs-site.xml mapred-env.cmd

# create users and related directories running Hadoop processes

[root@node1 hadoop] # groupadd hadoop

[root@node1 hadoop] # useradd-g hadoop yarn

[root@node1 hadoop] # useradd-g hadoop hdfs

[root@node1 hadoop] # useradd-g hadoop mapred

# create data and log directories

[root@node1 hadoop] # mkdir-p / data/hadoop/hdfs/ {nn,snn,dn}

[root@node1 hadoop] # chown-R hdfs:hadoop / data/hadoop/hdfs/

[root@node1 hadoop] # ll / data/hadoop/hdfs/

Total 0

Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 dn

Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 nn

Drwxr-xr-x 2 hdfs hadoop 6 Jan 7 09:48 snn

[root@node1 hadoop] # mkdir logs

[root@node1 hadoop] # chmod Grouw logs

[root@node1 hadoop] # chown-R yarn:hadoop. / *

[root@node1 hadoop] # ll

Total 36

Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 bin

Drwxr-xr-x 3 yarn hadoop 19 Oct 22 2015 etc

Drwxr-xr-x 2 yarn hadoop 101 Oct 22 2015 include

Drwxr-xr-x 3 yarn hadoop 19 Oct 22 2015 lib

Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 libexec

-rw-r--r-- 1 yarn hadoop 15429 Oct 22 2015 LICENSE.txt

Drwxrwxr-x 2 yarn hadoop 6 Jan 7 09:51 logs

-rw-r--r-- 1 yarn hadoop 101 Oct 22 2015 NOTICE.txt

-rw-r--r-- 1 yarn hadoop 1366 Oct 22 2015 README.txt

Drwxr-xr-x 2 yarn hadoop 4096 Oct 22 2015 sbin

Drwxr-xr-x 4 yarn hadoop 29 Oct 22 2015 share

# configure hadoop

[root@node1 hadoop] # cd etc/hadoop/

[root@node1 hadoop] # vim core-site.xml

Add at the end:

Fs.defaultFS

Hdfs://localhost:8020

True

[root@node1 hadoop] # vim hdfs-site.xml

Add at the end:

Dfs.replication

one

Dfs.namenode.name.dir

File:///data/hadoop/hdfs/nn

Dfs.datanode.data.dir

File:///data/hadoop/hdfs/dn

Dfs.checkpoint.dir

File:///data/hadoop/hdfs/snn

Fs.checkpoint.edits.dir

File:///data/hadoop/hdfs/snn

[root@node1 hadoop] # cp mapred-site.xml.template mapred-site.xml

[root@node1 hadoop] # vim mapred-site.xml

Add at the end:

Mapreduce.framework.name

Yarn

[root@node1 hadoop] # vim yarn-site.xml

Yarn.resourcemanager.address

Localhost:8032

Yarn.resourcemanager.schedul.address

Localhost:8030

Yarn.resourcemanager.resource-tracker.address

Localhost:8031

Yarn.resourcemanager.admin.address

Localhost:8033

Yarn.resourcemanager.webapp.address

Localhost:8088

Yarn.nodemanager.aux-services

Mapreduce_shuffle

Yarn.nodemanager.auxservices.mapreduce_shuffle.class

Org.apache.hadoop.mapred.ShuffleHandler

Yarn.resourcemanager.scheduler.class

Org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler

# formatting HDFS

[root@node1 hadoop] # su-hdfs

[hdfs@node1 ~] $hdfs namenode-format

[hdfs@node1 ~] $ls / data/hadoop/hdfs/nn/current/

Fsp_w_picpath_0000000000000000000 fsp_w_picpath_0000000000000000000.md5 seen_txid VERSION

# start Hadoop

[hdfs@node1 ~] $hadoop-daemon.sh start namenode

Starting namenode, logging to / bdapps/hadoop/logs/hadoop-hdfs-namenode-node1.out

[hdfs@node1 ~] $ls / bdapps/hadoop/logs/

Hadoop-hdfs-namenode-node1.log SecurityAuth-hdfs.audit

Hadoop-hdfs-namenode-node1.out

[hdfs@node1 ~] $jps

41937 NameNode

42025 Jps

[hdfs@node1 ~] $hadoop-daemon.sh start secondarynamenode

Starting secondarynamenode, logging to / bdapps/hadoop/logs/hadoop-hdfs-secondarynamenode-node1.out

[hdfs@node1 ~] $jps

41937 NameNode

42090 SecondaryNameNode

42133 Jps

[hdfs@node1 ~] $hadoop-daemon.sh start datanode

Starting datanode, logging to / bdapps/hadoop/logs/hadoop-hdfs-datanode-node1.out

[hdfs@node1 ~] $jps

41937 NameNode

42242 Jps

42166 DataNode

42090 SecondaryNameNode

[hdfs@node1 ~] $hdfs dfs-mkdir / test

[hdfs@node1 ~] $hdfs dfs-ls /

Found 1 items

Drwxr-xr-x-hdfs supergroup 0 2017-01-07 11:08 / test

[hdfs@node1 ~] $hdfs dfs-put / etc/fstab / test/fstab

[hdfs@node1 ~] $hdfs dfs-ls / test

Found 1 items

-rw-r--r-- 1 hdfs supergroup 465 2017-01-07 11:11 / test/fstab

[hdfs@node1 ~] $hdfs dfs-lsr /

Lsr: DEPRECATED: Please use'ls-R 'instead.

Drwxr-xr-x-hdfs supergroup 0 2017-01-07 11:11 / test

-rw-r--r-- 1 hdfs supergroup 465 2017-01-07 11:11 / test/fstab

[hdfs@node1] $ls / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/

Blk_1073741825 blk_1073741825_1001.meta

[hdfs@node1] $file / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825

/ data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825: ASCII text

[hdfs@node1] $cat / data/hadoop/hdfs/dn/current/BP-1017498243-192.168.1.131-1483757917078/current/finalized/subdir0/subdir0/blk_1073741825

#

# / etc/fstab

# Created by anaconda on Fri Sep 2 00:27:27 2016

#

# Accessible filesystems, by reference, are maintained under'/ dev/disk'

# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info

#

/ dev/mapper/centos-root / xfs defaults 0 0

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0

/ dev/mapper/centos-swap swap swap defaults 0 0

[hdfs@node1 ~] $hdfs dfs-cat / test/fstab

#

# / etc/fstab

# Created by anaconda on Fri Sep 2 00:27:27 2016

#

# Accessible filesystems, by reference, are maintained under'/ dev/disk'

# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info

#

/ dev/mapper/centos-root / xfs defaults 0 0

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0

/ dev/mapper/centos-swap swap swap defaults 0 0

[hdfs@node1 ~] $exit

[root@node1 hadoop] # su-yarn

[yarn@node1 ~] $yarn-daemon.sh start resourcemanager

Starting resourcemanager, logging to / bdapps/hadoop/logs/yarn-yarn-resourcemanager-node1.out

[yarn@node1 ~] $jps

42932 ResourceManager

43161 Jps

[yarn@node1 ~] $yarn-daemon.sh start nodemanager

Starting nodemanager, logging to / bdapps/hadoop/logs/yarn-yarn-nodemanager-node1.out

[root@node1 hadoop] # pwd

/ bdapps/hadoop

[root@node1 hadoop] # cd share/hadoop/mapreduce/

[root@node1 mapreduce] # ls

Hadoop-mapreduce-client-app-2.6.2.jar

Hadoop-mapreduce-client-common-2.6.2.jar

Hadoop-mapreduce-client-core-2.6.2.jar

Hadoop-mapreduce-client-hs-2.6.2.jar

Hadoop-mapreduce-client-hs-plugins-2.6.2.jar

Hadoop-mapreduce-client-jobclient-2.6.2.jar

Hadoop-mapreduce-client-jobclient-2.6.2-tests.jar

Hadoop-mapreduce-client-shuffle-2.6.2.jar

Hadoop-mapreduce-examples-2.6.2.jar

Lib

Lib-examples

Sources

[root@node1 mapreduce] # su-hdfs

Last login: Sat Jan 7 10:40:42 CST 2017 on pts/0

[hdfs@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar

An example program must be given as the first argument.

Valid program names are:

Aggregatewordcount: An Aggregate based map/reduce program that counts the words in the input files.

Aggregatewordhist: An Aggregate based map/reduce program that computes the histogram of the words in the input files.

Bbp: A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact digits of Pi.

Dbcount: An example job that count the pageview counts from a database.

Distbbp: A map/reduce program that uses a BBP-type formula to compute exact bits of Pi.

Grep: A map/reduce program that counts the matches of a regex in the input.

Join: A job that effects a join over sorted, equally partitioned datasets

Multifilewc: A job that counts words from several files.

Pentomino: A map/reduce tile laying program to find solutions to pentomino problems.

Pi: A map/reduce program that estimates Pi using a quasi-Monte Carlo method.

Randomtextwriter: A map/reduce program that writes 10GB of random textual data per node.

Randomwriter: A map/reduce program that writes 10GB of random data per node.

Secondarysort: An example defining a secondarysort to the reduce.

Sort: A map/reduce program that sorts the data written by the random writer.

Sudoku: A sudoku solver.

Teragen: Generate data for the terasort

Terasort: Run the terasort

Teravalidate: Checking results of terasort

Wordcount: A map/reduce program that counts the words in the input files.

Wordmean: A map/reduce program that counts the average length of the words in the input files.

Wordmedian: A map/reduce program that counts the median length of the words in the input files.

Wordstandarddeviation: A map/reduce program that counts the standarddeviation of the length of the words in the input files.

[hdfs@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar wordcount / test/fstab / test/fstab.out

17-01-07 11:43:42 INFO client.RMProxy: Connecting to ResourceManager at localhost/127.0.0.1:8032

17-01-07 11:43:43 INFO input.FileInputFormat: Total input paths to process: 1

17-01-07 11:43:43 INFO mapreduce.JobSubmitter: number of splits:1

17-01-07 11:43:44 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1483759378332_0001

17-01-07 11:43:45 INFO impl.YarnClientImpl: Submitted application application_1483759378332_0001

11:43:45 on 17-01-07 INFO mapreduce.Job: The url to track the job: http://node1:8088/proxy/application_1483759378332_0001/

17-01-07 11:43:45 INFO mapreduce.Job: Running job: job_1483759378332_0001

17-01-07 11:43:55 INFO mapreduce.Job: Job job_1483759378332_0001 running in uber mode: false

17-01-07 11:43:55 INFO mapreduce.Job: map 0 reduce 0

17-01-07 11:44:03 INFO mapreduce.Job: map 100% reduce 0

17-01-07 11:44:10 INFO mapreduce.Job: map 100 reduce 100%

17-01-07 11:44:11 INFO mapreduce.Job: Job job_1483759378332_0001 completed successfully

17-01-07 11:44:11 INFO mapreduce.Job: Counters: 49

File System Counters

FILE: Number of bytes read=554

FILE: Number of bytes written=214131

FILE: Number of read operations=0

FILE: Number of large read operations=0

FILE: Number of write operations=0

HDFS: Number of bytes read=562

HDFS: Number of bytes written=396

HDFS: Number of read operations=6

HDFS: Number of large read operations=0

HDFS: Number of write operations=2

Job Counters

Launched map tasks=1

Launched reduce tasks=1

Data-local map tasks=1

Total time spent by all maps in occupied slots (ms) = 5413

Total time spent by all reduces in occupied slots (ms) = 4521

Total time spent by all map tasks (ms) = 5413

Total time spent by all reduce tasks (ms) = 4521

Total vcore-seconds taken by all map tasks=5413

Total vcore-seconds taken by all reduce tasks=4521

Total megabyte-seconds taken by all map tasks=5542912

Total megabyte-seconds taken by all reduce tasks=4629504

Map-Reduce Framework

Map input records=11

Map output records=54

Map output bytes=588

Map output materialized bytes=554

Input split bytes=97

Combine input records=54

Combine output records=38

Reduce input groups=38

Reduce shuffle bytes=554

Reduce input records=38

Reduce output records=38

Spilled Records=76

Shuffled Maps = 1

Failed Shuffles=0

Merged Map outputs=1

GC time elapsed (ms) = 258

CPU time spent (ms) = 3020

Physical memory (bytes) snapshot=429760512

Virtual memory (bytes) snapshot=4256686080

Total committed heap usage (bytes) = 301465600

Shuffle Errors

BAD_ID=0

CONNECTION=0

IO_ERROR=0

WRONG_LENGTH=0

WRONG_MAP=0

WRONG_REDUCE=0

File Input Format Counters

Bytes Read=465

File Output Format Counters

Bytes Written=396

[hdfs@node1 ~] $hdfs dfs-ls / test/fstab.out

Found 2 items

-rw-r--r-- 1 hdfs supergroup 0 2017-01-07 11:44 / test/fstab.out/_SUCCESS

-rw-r--r-- 1 hdfs supergroup 396 2017-01-07 11:44 / test/fstab.out/part-r-00000

[hdfs@node1 ~] $hdfs dfs-cat / test/fstab.out/part-r-00000

# 7

'/ dev/disk' 1

/ 1

/ boot 1

/ dev/mapper/centos-root 1

/ dev/mapper/centos-swap 1

/ etc/fstab 1

0 6

00:27:27 1

2 1

2016 1

Accessible 1

Created 1

Fri 1

See 1

Sep 1

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 1

Anaconda 1

And/or 1

Are 1

Blkid (8) 1

By 2

Defaults 3

Filesystems, 1

Findfs (8), 1

For 1

Fstab (5), 1

Info 1

Maintained 1

Man 1

More 1

Mount (8) 1

On 1

Pages 1

Reference, 1

Swap 2

Under 1

Xfs 2

02 Hadoop 2 distributed cluster

Configure the environment

Node1: 192.168.1.130 CentOS Linux release 7.2

Node2: 192.168.1.131 CentOS Linux release 7.2

Node3: 192.168.1.132 CentOS Linux release 7.2

Node4: 192.168.1.133 CentOS Linux release 7.2

[root@node1 ~] # vim / etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4

:: 1 localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.1.130 node1 master

192.168.1.131 node2

192.168.1.132 node3

192.168.1.133 node4

[root@node1 ~] # scp / etc/hosts node2:/etc/

[root@node1 ~] # scp / etc/hosts node3:/etc/

[root@node1 ~] # scp / etc/hosts node4:/etc/

[root@node1 ~] # vim / etc/profile.d/java.sh

Export JAVA_HOME=/usr

[root@node1 ~] # yum makecache

[root@node1 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y

[root@node1 ~] # java-version

Openjdk version "1.8.0mm 101"

OpenJDK Runtime Environment (build 1.8.0_101-b13)

OpenJDK 64-Bit Server VM (build 25.101-b13, mixed mode)

[root@node1 ~] # scp / etc/profile.d/java.sh node2:/etc/profile.d/

[root@node1 ~] # scp / etc/profile.d/java.sh node3:/etc/profile.d/

[root@node1 ~] # scp / etc/profile.d/java.sh node4:/etc/profile.d/

[root@node1 ~] # groupadd hadoop

[root@node1 ~] # useradd-g hadoop hadoop

[root@node1 ~] # echo hadoop:mageedu | chpasswd

[root@node2 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y

[root@node2 ~] # useradd hadoop

[root@node2 ~] # echo hadoop:mageedu | chpasswd

[root@node3 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y

[root@node3 ~] # useradd hadoop

[root@node3 ~] # echo hadoop:mageedu | chpasswd

[root@node4 ~] # yum install java-1.7.0-openjdk-devel.x86_64-y

[root@node4 ~] # useradd hadoop

[root@node4 ~] # echo hadoop:mageedu | chpasswd

[hadoop@node1 ~] $ssh-keygen-t rsa-P''

Generating public/private rsa key pair.

Enter file in which to save the key (/ home/hadoop/.ssh/id_rsa):

Created directory'/ home/hadoop/.ssh'.

Your identification has been saved in / home/hadoop/.ssh/id_rsa.

Your public key has been saved in / home/hadoop/.ssh/id_rsa.pub.

The key fingerprint is:

2b:fc:62:6d:4e:ff:0d:01:6a:9e:4b:9f:55:a7:af:53 hadoop@node1

The key's randomart p_w_picpath is:

+-[RSA 2048]-+

| | |

| | |

|. | |

|. . | |

| S. . . |

|. O o o oE |

| o.* o. | |

| | oroomo + o o.. | | |

|. + +. Oo. |

+-+

[hadoop@node1] $for i in 2 34 X do ssh-copy-id-I. ssh / id_rsa.pub hadoop@node$ {I}; done

[hadoop@node1 ~] $exit

Logout

[root@node1] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}

[root@node1] # chown-R hadoop.hadoop / data/hadoop/hdfs/

[root@node1] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/

[root@node1 ~] # cd / bdapps/

[root@node1 bdapps] # ls

Hadoop-2.6.2

[root@node1 bdapps] # ln-s hadoop-2.6.2/ hadoop

[root@node1 bdapps] # cd hadoop

[root@node1 hadoop] # mkdir logs

[root@node1 hadoop] # chmod Grouw logs

[root@node1 hadoop] # chown-R hadoop.hadoop. / *

[root@node1 hadoop] # ll

Total 36

Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 bin

Drwxr-xr-x 3 hadoop hadoop 19 Oct 22 2015 etc

Drwxr-xr-x 2 hadoop hadoop 101 Oct 22 2015 include

Drwxr-xr-x 3 hadoop hadoop 19 Oct 22 2015 lib

Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 libexec

-rw-r--r-- 1 hadoop hadoop 15429 Oct 22 2015 LICENSE.txt

Drwxrwxr-x 2 hadoop hadoop 6 Jan 7 14:21 logs

-rw-r--r-- 1 hadoop hadoop 101 Oct 22 2015 NOTICE.txt

-rw-r--r-- 1 hadoop hadoop 1366 Oct 22 2015 README.txt

Drwxr-xr-x 2 hadoop hadoop 4096 Oct 22 2015 sbin

Drwxr-xr-x 4 hadoop hadoop 29 Oct 22 2015 share

[root@node1 hadoop] # cd etc/hadoop/

[root@node1 hadoop] # vim core-site.xml

Last line add

Fs.defaultFS

Hdfs://master:8020

True

[root@node1 hadoop] # vim yarn-site.xml

Add at the end

Yarn.resourcemanager.address

Master:8032

Yarn.resourcemanager.schedul.address

Master:8030

Yarn.resourcemanager.resource-tracker.address

Master:8031

Yarn.resourcemanager.admin.address

Master:8033

Yarn.resourcemanager.webapp.address

Master:8088

Yarn.nodemanager.aux-services

Mapreduce_shuffle

Yarn.nodemanager.auxservices.mapreduce_shuffle.class

Org.apache.hadoop.mapred.ShuffleHandler

Yarn.resourcemanager.scheduler.class

Org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler

[root@node1 hadoop] # vim hdfs-site.xml

Add at the end

Dfs.replication

two

Dfs.namenode.name.dir

File:///data/hadoop/hdfs/nn

Dfs.datanode.data.dir

File:///data/hadoop/hdfs/dn

Dfs.checkpoint.dir

File:///data/hadoop/hdfs/snn

Fs.checkpoint.edits.dir

File:///data/hadoop/hdfs/snn

[root@node1 hadoop] # vim slaves

Node2

Node3

Node4

[root@node2] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}

[root@node2] # chown-R hadoop.hadoop / data/hadoop/hdfs/

[root@node3] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}

[root@node3] # chown-R hadoop.hadoop / data/hadoop/hdfs/

[root@node4] # mkdir-p / bdapps / data/hadoop/hdfs/ {nn,snn,dn}

[root@node4] # chown-R hadoop.hadoop / data/hadoop/hdfs/

[root@node2] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/

[root@node2 ~] # cd / bdapps/

[root@node2 bdapps] # ln-s hadoop-2.6.2/ hadoop

[root@node2 bdapps] # cd hadoop

[root@node2 hadoop] # mkdir logs

[root@node2 hadoop] # chmod Grouw logs/

[root@node2 hadoop] # chown hadoop.hadoop-R. / *

[root@node3] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/

[root@node3 ~] # cd / bdapps/

[root@node3 bdapps] # ln-s hadoop-2.6.2/ hadoop

[root@node3 bdapps] # cd hadoop

[root@node3 hadoop] # mkdir logs

[root@node3 hadoop] # chmod Grouw logs

[root@node3 hadoop] # chown hadoop.hadoop-R. / *

[root@node4] # tar xf hadoop-2.6.2.tar.gz-C / bdapps/

[root@node4 ~] # cd / bdapps/

[root@node4 bdapps] # ln-s hadoop-2.6.2/ hadoop

[root@node4 bdapps] # cd hadoop

[root@node4 hadoop] # mkdir logs

[root@node4 hadoop] # chmod Grouw logs/

[root@node4 hadoop] # chown hadoop.hadoop-R. / *

[root@node1 ~] # su-hadoop

[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node2:/bdapps/hadoop/etc/hadoop/

[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node3:/bdapps/hadoop/etc/hadoop/

[hadoop@node1 ~] $scp / bdapps/hadoop/etc/hadoop/* node4:/bdapps/hadoop/etc/hadoop/

[hadoop@node1 ~] $exit p/etc/hadoop/

Logout

[root@node1 ~] # vim / etc/profile.d/hadoop.sh

Export HADOOP_PREFIX=/bdapps/hadoop

Export PATH=$PATH:$ {HADOOP_PREFIX} / bin:$ {HADOOP_PREFIX} / sbin

Export HADOOP_COMMON_HOME=$ {HADOOP_PREFIX}

Export HADOOP_YARN_HOME=$ {HADOOP_PREFIX}

Export HADOOP_HDFS_HOME=$ {HADOOP_PREFIX}

Export HADOOP_MAPRED_HOME=$ {HADOOP_PREFIX}

[root@node1 ~] # scp / etc/profile.d/hadoop.sh node2:/etc/profile.d/

[root@node1 ~] # scp / etc/profile.d/hadoop.sh node3:/etc/profile.d/

[root@node1 ~] # scp / etc/profile.d/hadoop.sh node4:/etc/profile.d/

# formatting HDFS

[root@node1 ~] # su-hadoop

[hadoop@node1 ~] $hdfs namenode-format

[hadoop@node1 ~] $ls / data/hadoop/hdfs/nn/

Current

# start the service

[hadoop@node1 ~] $start-dfs.sh

[hadoop@node1 ~] $jps

35957 NameNode

36153 SecondaryNameNode

36292 Jps

# stop the service

[hadoop@node1 ~] $stop-dfs.sh

[root@node2 hadoop] # su-hadoop

[hadoop@node2 ~] $jps

35527 DataNode

35639 Jps

[root@node3 hadoop] # su-hadoop

[hadoop@node3 ~] $jps

35113 DataNode

35241 Jps

[root@node4 hadoop] # su-hadoop

[hadoop@node4 ~] $jps

35113 DataNode

35242 Jps

# create a directory

[hadoop@node1 ~] $hdfs dfs-mkdir / test

# upload a file to hadoop

[hadoop@node1 ~] $hdfs dfs-put / etc/fstab / test/fstab

# View uploaded files

[hadoop@node1 ~] $hdfs dfs-lsr / test

Lsr: DEPRECATED: Please use'ls-R 'instead.

-rw-r--r-- 2 hadoop supergroup 465 2017-01-07 15:16 / test/fstab

# View the contents of the uploaded file

[hadoop@node1 ~] $hdfs dfs-cat / test/fstab

#

# / etc/fstab

# Created by anaconda on Fri Sep 2 00:27:27 2016

#

# Accessible filesystems, by reference, are maintained under'/ dev/disk'

# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info

#

/ dev/mapper/centos-root / xfs defaults 0 0

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0

/ dev/mapper/centos-swap swap swap defaults 0 0

# Storage location of uploaded files in hadoop

[hadoop@node2] $cat / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/subdir0/subdir0/blk_1073741825

#

# / etc/fstab

# Created by anaconda on Fri Sep 2 00:27:27 2016

#

# Accessible filesystems, by reference, are maintained under'/ dev/disk'

# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info

#

/ dev/mapper/centos-root / xfs defaults 0 0

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0

/ dev/mapper/centos-swap swap swap defaults 0 0

[hadoop@node3] $cat / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/subdir0/subdir0/blk_1073741825

#

# / etc/fstab

# Created by anaconda on Fri Sep 2 00:27:27 2016

#

# Accessible filesystems, by reference, are maintained under'/ dev/disk'

# See man pages fstab (5), findfs (8), mount (8) and/or blkid (8) for more info

#

/ dev/mapper/centos-root / xfs defaults 0 0

UUID=a2f7c1a7-a991-470a-b5f5-1c4ed4e098b6 / boot xfs defaults 0 0

/ dev/mapper/centos-swap swap swap defaults 0 0

[hadoop@node4] $ls / data/hadoop/hdfs/dn/current/BP-1026073846-192.168.1.130-1483772708588/current/finalized/

[hadoop@node1 ~] $start-yarn.sh

Starting yarn daemons

Starting resourcemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-resourcemanager-node1.out

Node4: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node4.out

Node3: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node3.out

Node2: starting nodemanager, logging to / bdapps/hadoop/logs/yarn-hadoop-nodemanager-node2.out

[hadoop@node1 ~] $jps

35957 NameNode

36153 SecondaryNameNode

36980 Jps

36715 ResourceManager

[hadoop@node2 ~] $jps

35996 Jps

35527 DataNode

35880 NodeManager

[hadoop@node3 ~] $jps

35113 DataNode

35474 NodeManager

35591 Jps

[hadoop@node4 ~] $jps

35477 NodeManager

35113 DataNode

35600 Jps

[root@node1 ~] # ss-tnl

State Recv-Q Send-Q Local Address:Port Peer Address:Port

LISTEN 0 128 *: 50090 *: *

LISTEN 0 128 192.168.1.130 8020 *: *

LISTEN 0 5 192.168.122.1 LISTEN 53 *: *

LISTEN 0 128 *: 50070 *: *

LISTEN 0 128 *: 22 *: *

LISTEN 0 128 127.0.0.1 631 *: *

LISTEN 0 100 127.0.0.1 purl 25 *: *

LISTEN 0 128:: 22: *

LISTEN 0 128:: 1 LISTEN:: *

LISTEN 0 128:: ffff:192.168.1.130:8088: *

LISTEN 0 100:: 1:25: *

LISTEN 0 128:: 8030: *

LISTEN 0 128:: ffff:192.168.1.130:8031: *

LISTEN 0 128:: ffff:192.168.1.130:8032: *

LISTEN 0 128:: ffff:192.168.1.130:8033: *

[hadoop@node1 ~] $hdfs dfs-put / etc/rc.d/init.d/functions / test

[hadoop@node1 ~] $hdfs dfs-ls / test

Found 2 items

-rw-r--r-- 2 hadoop supergroup 465 2017-01-07 15:16 / test/fstab

-rw-r--r-- 2 hadoop supergroup 13948 2017-01-07 15:35 / test/functions

[hadoop@node1 ~] $hdfs dfs-put hadoop-2.6.2.tar.gz / test/

# Statistical text file

[hadoop@node1 ~] $yarn jar / bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.2.jar wordcount / test/fstab / test/functions / test/wc

[hadoop@node1 ~] $hdfs-ls / test/wc

[hadoop@node1 ~] $hdfs dfs-cat / test/wc/part-r-00000

Hadoop application

Http://192.168.1.130:8088

Manage hadoop

Http://192.168.1.130:50070

2. YARN cluster management command

[hadoop@node1 ~] $yarn application-list

17-01-07 17:11:25 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032

Total number of applications (application-types: [] and states: [SUBMITTED, ACCEPTED, RUNNING]): 0

Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL

[hadoop@node1 ~] $yarn application-list-appStates=all

17-01-07 17:12:13 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032

Total number of applications (application-types: [] and states: [NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED]): 0

Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL

[hadoop@node1 ~] $yarn application-status

[hadoop@node1 ~] $yarn node-list

17-01-07 17:16:47 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032

Total Nodes:3

Node-Id Node-State Node-Http-Address Number-of-Running-Containers

Node2:40860 RUNNING node2:8042 0

Node3:47737 RUNNING node3:8042 0

Node4:45637 RUNNING node4:8042 0

[hadoop@node1 ~] $yarn node-status node3:47737

17-01-07 17:20:41 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8032

Node Report:

Node-Id: node3:47737

Rack: / default-rack

Node-State: RUNNING

Node-Http-Address: node3:8042

Last-Health-Update: Sat 07/Jan/17 05:18:54:160CST

Health-Report:

Containers: 0

Memory-Used: 0MB

Memory-Capacity: 8192MB

CPU-Used: 0 vcores

CPU-Capacity: 8 vcores

Node-Labels:

[hadoop@node1 ~] $yarn logs-applicationId

[hadoop@node1 ~] $yarn rmadmin-refreshNodes

17-01-07 17:27:20 INFO client.RMProxy: Connecting to ResourceManager at master/192.168.1.130:8033

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.