Shulou(Shulou.com)06/03 Report--

#! / bin/bash

#

# Install Hadoop 2 using pdsh/pdcp where possible.

#

# Command can be interactive or file-based. This script sets up

# a Hadoop 2 cluster with basic configuration. Modify data, log, and pid

# directories as desired. Further configure your cluster with. / conf-hadoop2.sh

#

#

# Basic environment variables. Edit as necessary

HADOOP_VERSION=2.0.5-alpha

HADOOP_HOME= "/ opt/hadoop-$ {HADOOP_VERSION}"

NN_DATA_DIR=/var/data/hadoop/hdfs/nn

SNN_DATA_DIR=/var/data/hadoop/hdfs/snn

DN_DATA_DIR=/var/data/hadoop/hdfs/dn

YARN_LOG_DIR=/var/log/hadoop/yarn

HADOOP_LOG_DIR=/var/log/hadoop/hdfs

HADOOP_MAPRED_LOG_DIR=/var/log/hadoop/mapred

YARN_PID_DIR=/var/run/hadoop/yarn

HADOOP_PID_DIR=/var/run/hadoop/hdfs

HADOOP_MAPRED_PID_DIR=/var/run/hadoop/mapred

HTTP_STATIC_USER=hdfs

YARN_PROXY_PORT=8081

Source hadoop-xml-conf.sh

CMD_OPTIONS=$ (getopt-n "$0"-o hif-- long "help,interactive,file"-- "$@")

# Take care of bad options in the command

If [$?-ne 0]

Then

Exit 1

Fi

Eval set-"$CMD_OPTIONS"

All_hosts= "all_hosts"

Nn_host= "nn_host"

Snn_host= "snn_host"

Dn_hosts= "dn_hosts"

Rm_host= "rm_host"

Nm_hosts= "nm_hosts"

Mr_history_host= "mr_history_host"

Yarn_proxy_host= "yarn_proxy_host"

Install ()

{

Echo "Copying Hadoop $HADOOP_VERSION to all hosts..."

Pdcp-w ^ all_hosts hadoop- "$HADOOP_VERSION" .tar.gz / opt

Echo "Copying JDK 1.6.031 to all hosts..."

Pdcp-w ^ all_hosts jdk-6u31-linux-x64-rpm.bin / opt

Echo "Installing JDK 1.6.031 on all hosts..."

Pdsh-w ^ all_hosts chmod adepx / opt/jdk-6u31-linux-x64-rpm.bin

Pdsh-w ^ all_hosts / opt/jdk-6u31-linux-x64-rpm.bin-noregister 1 > &-2 > &-

Echo "Setting JAVA_HOME and HADOOP_HOME environment variables on all hosts..."

Pdsh-w ^ all_hosts' echo export JAVA_HOME=/usr/java/jdk1.6.0_31 > / etc/profile.d/java.sh'

Pdsh-w ^ all_hosts "source / etc/profile.d/java.sh"

Pdsh-w ^ all_hosts "echo export HADOOP_HOME=$HADOOP_HOME > / etc/profile.d/hadoop.sh"

Pdsh-w ^ all_hosts' echo export HADOOP_PREFIX=$HADOOP_HOME > > / etc/profile.d/hadoop.sh'

Pdsh-w ^ all_hosts "source / etc/profile.d/hadoop.sh"

Echo "Extracting Hadoop $HADOOP_VERSION distribution on all hosts..."

Pdsh-w ^ all_hosts tar-zxf / opt/hadoop- "$HADOOP_VERSION" .tar.gz-C / opt

Echo "Creating system accounts and groups on all hosts..."

Pdsh-w ^ all_hosts groupadd hadoop

Pdsh-w ^ all_hosts useradd-g hadoop yarn

Pdsh-w ^ all_hosts useradd-g hadoop hdfs

Pdsh-w ^ all_hosts useradd-g hadoop mapred

Echo "Creating HDFS data directories on NameNode host, Secondary NameNode host, and DataNode hosts..."

Pdsh-w ^ nn_host "mkdir-p $NN_DATA_DIR & & chown hdfs:hadoop $NN_DATA_DIR"

Pdsh-w ^ snn_host "mkdir-p $SNN_DATA_DIR & & chown hdfs:hadoop $SNN_DATA_DIR"

Pdsh-w ^ dn_hosts "mkdir-p $DN_DATA_DIR & & chown hdfs:hadoop $DN_DATA_DIR"

Echo "Creating log directories on all hosts..."

Pdsh-w ^ all_hosts "mkdir-p $YARN_LOG_DIR & & chown yarn:hadoop $YARN_LOG_DIR"

Pdsh-w ^ all_hosts "mkdir-p $HADOOP_LOG_DIR & & chown hdfs:hadoop $HADOOP_LOG_DIR"

Pdsh-w ^ all_hosts "mkdir-p $HADOOP_MAPRED_LOG_DIR & & chown mapred:hadoop $HADOOP_MAPRED_LOG_DIR"

Echo "Creating pid directories on all hosts..."

Pdsh-w ^ all_hosts "mkdir-p $YARN_PID_DIR & & chown yarn:hadoop $YARN_PID_DIR"

Pdsh-w ^ all_hosts "mkdir-p $HADOOP_PID_DIR & & chown hdfs:hadoop $HADOOP_PID_DIR"

Pdsh-w ^ all_hosts "mkdir-p $HADOOP_MAPRED_PID_DIR & & chown mapred:hadoop $HADOOP_MAPRED_PID_DIR"

Echo "Editing Hadoop environment scripts for log directories on all hosts..."

Pdsh-w ^ all_hosts echo "export HADOOP_LOG_DIR=$HADOOP_LOG_DIR > > $HADOOP_HOME/etc/hadoop/hadoop-env.sh"

Pdsh-w ^ all_hosts echo "export YARN_LOG_DIR=$YARN_LOG_DIR > > $HADOOP_HOME/etc/hadoop/yarn-env.sh"

Pdsh-w ^ all_hosts echo "export HADOOP_MAPRED_LOG_DIR=$HADOOP_MAPRED_LOG_DIR > > $HADOOP_HOME/etc/hadoop/mapred-env.sh"

Echo "Editing Hadoop environment scripts for pid directories on all hosts..."

Pdsh-w ^ all_hosts echo "export HADOOP_PID_DIR=$HADOOP_PID_DIR > > $HADOOP_HOME/etc/hadoop/hadoop-env.sh"

Pdsh-w ^ all_hosts echo "export YARN_PID_DIR=$YARN_PID_DIR > > $HADOOP_HOME/etc/hadoop/yarn-env.sh"

Pdsh-w ^ all_hosts echo "export HADOOP_MAPRED_PID_DIR=$HADOOP_MAPRED_PID_DIR > > $HADOOP_HOME/etc/hadoop/mapred-env.sh"

Echo "Creating base Hadoop XML config files..."

Create_config-file core-site.xml

Put_config-file core-site.xml-property fs.default.name-value "hdfs://$nn:9000"

Put_config-file core-site.xml-property hadoop.http.staticuser.user-value "$HTTP_STATIC_USER"

Create_config-file hdfs-site.xml

Put_config-file hdfs-site.xml-property dfs.namenode.name.dir-value "$NN_DATA_DIR"

Put_config-file hdfs-site.xml-property fs.checkpoint.dir-value "$SNN_DATA_DIR"

Put_config-file hdfs-site.xml-property fs.checkpoint.edits.dir-value "$SNN_DATA_DIR"

Put_config-file hdfs-site.xml-property dfs.datanode.data.dir-value "$DN_DATA_DIR"

Put_config-file hdfs-site.xml-property dfs.namenode.http-address-value "$nn:50070"

Put_config-file hdfs-site.xml-property dfs.namenode.secondary.http-address-value "$snn:50090"

Create_config-file mapred-site.xml

Put_config-file mapred-site.xml-property mapreduce.framework.name-value yarn

Put_config-file mapred-site.xml-property mapreduce.jobhistory.address-value "$mr_hist:10020"

Put_config-file mapred-site.xml-property mapreduce.jobhistory.webapp.address-value "$mr_hist:19888"

Put_config-file mapred-site.xml-property yarn.app.mapreduce.am.staging-dir-value / mapred

Create_config-file yarn-site.xml

Put_config-file yarn-site.xml-property yarn.nodemanager.aux-services-value mapreduce.shuffle

Put_config-file yarn-site.xml-property yarn.nodemanager.aux-services.mapreduce.shuffle.class-value org.apache.hadoop.mapred.ShuffleHandler

Put_config-file yarn-site.xml-property yarn.web-proxy.address-value "$yarn_proxy:$YARN_PROXY_PORT"

Put_config-file yarn-site.xml-property yarn.resourcemanager.scheduler.address-value "$rmgr:8030"

Put_config-file yarn-site.xml-property yarn.resourcemanager.resource-tracker.address-value "$rmgr:8031"

Put_config-file yarn-site.xml-property yarn.resourcemanager.address-value "$rmgr:8032"

Put_config-file yarn-site.xml-property yarn.resourcemanager.admin.address-value "$rmgr:8033"

Put_config-file yarn-site.xml-property yarn.resourcemanager.webapp.address-value "$rmgr:8088"

Echo "Copying base Hadoop XML config files to all hosts..."

Pdcp-w ^ all_hosts core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $HADOOP_HOME/etc/hadoop/

Echo "Creating configuration, command, and script links on all hosts..."

Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/etc/hadoop / etc/hadoop"

Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/bin/* / usr/bin"

Pdsh-w ^ all_hosts "ln-s $HADOOP_HOME/libexec/* / usr/libexec"

Echo "Formatting the NameNode..."

Pdsh-w ^ nn_host "su-hdfs-c'$HADOOP_HOME/bin/hdfs namenode-format'"

Echo "Copying startup scripts to all hosts..."

Pdcp-w ^ nn_host hadoop-namenode / etc/init.d/

Pdcp-w ^ snn_host hadoop-secondarynamenode / etc/init.d/

Pdcp-w ^ dn_hosts hadoop-datanode / etc/init.d/

Pdcp-w ^ rm_host hadoop-resourcemanager / etc/init.d/

Pdcp-w ^ nm_hosts hadoop-nodemanager / etc/init.d/

Pdcp-w ^ mr_history_host hadoop-historyserver / etc/init.d/

Pdcp-w ^ yarn_proxy_host hadoop-proxyserver / etc/init.d/

Echo "Starting Hadoop $HADOOP_VERSION services on all hosts..."

Pdsh-w ^ nn_host "chmod 755 / etc/init.d/hadoop-namenode & & chkconfig hadoop-namenode on & & service hadoop-namenode start"

Pdsh-w ^ snn_host "chmod 755 / etc/init.d/hadoop-secondarynamenode & & chkconfig hadoop-secondarynamenode on & & service hadoop-secondarynamenode start"

Pdsh-w ^ dn_hosts "chmod 755 / etc/init.d/hadoop-datanode & & chkconfig hadoop-datanode on & & service hadoop-datanode start"

Pdsh-w ^ rm_host "chmod 755 / etc/init.d/hadoop-resourcemanager & & chkconfig hadoop-resourcemanager on & & service hadoop-resourcemanager start"

Pdsh-w ^ nm_hosts "chmod 755 / etc/init.d/hadoop-nodemanager & & chkconfig hadoop-nodemanager on & & service hadoop-nodemanager start"

Pdsh-w ^ yarn_proxy_host "chmod 755 / etc/init.d/hadoop-proxyserver & & chkconfig hadoop-proxyserver on & & service hadoop-proxyserver start"

Echo "Creating MapReduce Job History directories..."

Su-hdfs-c "hadoop fs-mkdir-p / mapred/history/done_intermediate"

Su-hdfs-c "hadoop fs-chown-R mapred:hadoop / mapred"

Su-hdfs-c "hadoop fs-chmod-R g+rwx / mapred"

Pdsh-w ^ mr_history_host "chmod 755 / etc/init.d/hadoop-historyserver & & chkconfig hadoop-historyserver on & & service hadoop-historyserver start"

Echo "Running YARN smoke test..."

Pdsh-w ^ all_hosts "usermod-a-G hadoop $(whoami)"

Su-hdfs-c "hadoop fs-mkdir-p / user/$ (whoami)"

Su-hdfs-c "hadoop fs-chown $(whoami): $(whoami) / user/$ (whoami)"

Source / etc/profile.d/java.sh

Source / etc/profile.d/hadoop.sh

Source / etc/hadoop/hadoop-env.sh

Source / etc/hadoop/yarn-env.sh

Hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-$HADOOP_VERSION.jar pi-Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory-libjars $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-$HADOOP_VERSION.jar 16 10000

}

Interactive ()

{

Echo-n "Enter NameNode hostname:"

Read nn

Echo-n "Enter Secondary NameNode hostname:"

Read snn

Echo-n "Enter ResourceManager hostname:"

Read rmgr

Echo-n "Enter Job History Server hostname:"

Read mr_hist

Echo-n "Enter YARN Proxy hostname:"

Read yarn_proxy

Echo-n "Enter DataNode hostnames (comma separated or hostlist syntax):"

Read dns

Echo-n "Enter NodeManager hostnames (comma separated or hostlist syntax):"

Read nms

Echo "$nn" > "$nn_host"

Echo "$snn" > "$snn_host"

Echo "$rmgr" > "$rm_host"

Echo "$mr_hist" > "$mr_history_host"

Echo "$yarn_proxy" > "$yarn_proxy_host"

Dn_hosts_var=$ (sed's /\, /\ nUniverse g'"$all_hosts"

}

Help ()

{

Cat

Welcome to subscribe "Shulou Technology Information " to get latest news, interesting things and hot topics in the IT industry, and controls the hottest and latest Internet news, technology news and IT industry trends.