Hadoop CDH 5 Installation

# Install database

## install postgresql

sudo yum install postgresql-server postgresql

## init database

sudo su - postgres
    initdb -D /var/lib/pgsql/data

## start service

systemctl status postgresql.service
    systemctl start postgresql.service
    systemctl stop postgresql.service

## remote access

vi /var/lib/pgsql/data/postgresql.conf
    listen_addresses ='*'
    
    vi /var/lib/pgsql/data/pg_hba.conf
    host all all 0.0.0.0/0 trust
    
## restart service

systemctl restart postgresql.service

## set password

su - postgres
    psql
    \password postgres
    xxxpgxxx

## create cloudera-manager database

Connect to PostgreSQL:

sudo -u postgres psql
    
If you are not using the Cloudera Manager installer, create a database for the Cloudera Manager Server. The database name, user name, and password can be any value. Record the names chosen because you will need them later when running the scm_prepare_database.sh script.

CREATE ROLE scm LOGIN PASSWORD 'scm';
    CREATE DATABASE scm OWNER scm ENCODING 'UTF8'; 
    
Create databases for Activity Monitor, Reports Manager, Hive Metastore Server, Sentry Server, Cloudera Navigator Audit Server, and Cloudera Navigator Metadata Server:

CREATE ROLE user LOGIN PASSWORD 'password';
    CREATE DATABASE databaseName OWNER user ENCODING 'UTF8';
    
where user, password, and databaseName can be any value. The examples shown match the default names provided in the Cloudera Manager configuration settings:
Role Database User Password

CREATE ROLE amon LOGIN PASSWORD 'amon_password';
    CREATE DATABASE amon OWNER amon ENCODING 'UTF8';
    CREATE ROLE rman LOGIN PASSWORD 'rman_password';
    CREATE DATABASE rman OWNER rman ENCODING 'UTF8';
    CREATE ROLE hive LOGIN PASSWORD 'hive_password';
    CREATE DATABASE metastore OWNER hive ENCODING 'UTF8';
    CREATE ROLE sentry LOGIN PASSWORD 'sentry_password';
    CREATE DATABASE sentry OWNER sentry ENCODING 'UTF8';
    CREATE ROLE nav LOGIN PASSWORD 'nav_password';
    CREATE DATABASE nav OWNER nav ENCODING 'UTF8';
    CREATE ROLE navms LOGIN PASSWORD 'navms_password';
    CREATE DATABASE navms OWNER navms ENCODING 'UTF8'; 
    CREATE ROLE oozie LOGIN PASSWORD 'oozie_password';
    CREATE DATABASE oozie OWNER oozie ENCODING 'UTF8';
    
For PostgreSQL 8.2.23 or higher, also run:

ALTER DATABASE Metastore SET standard_conforming_strings = off;
    
# Configure SSH

## configure hosts

vi /etc/hosts
    x.x.x.x node1
    x.x.x.x node2
    x.x.x.x node3
    
## hostname

hostnamectl set-hostname node1
    hostnamectl set-hostname node2
    hostnamectl set-hostname node3
    
## configure ssh

ssh-keygen -t rsa
    ssh-copy-id -i ~/.ssh/id_rsa.pub node1
    ssh-copy-id -i ~/.ssh/id_rsa.pub node2
    ssh-copy-id -i ~/.ssh/id_rsa.pub node3

# Configure NTP

[configure NTP](http://note.louyj.com/blog/post/louyj/NTP%E6%97%B6%E9%97%B4%E5%90%8C%E6%AD%A5)

# Disable firewall

systemctl stop firewalld.service
    systemctl disable firewalld.service
    
# Install Dependency

## install jdk

wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u79-b15/jdk-7u79-linux-x64.tar.gz
    echo "JAVA_HOME=/root/jdk1.7.0_79" >> /etc/environment 
    vi .bashrc 
    export JAVA_HOME=/root/jdk1.7.0_79
    export PATH=$PATH:$JAVA_HOME/bin
    source .bashrc

## other dependency

yum update
    #yum groupinstall "Development Tools"
    yum install psmisc -y
    yum install libxslt-devel -y
    yum install rpcbind -y
    service rpcbind start

# Configure swap

dd if=/dev/zero of=/swap-file bs=1M count=4096
    mkswap /swap-file
    swapon /swap-file
    swapon -s

# Install cloudera-manager

## download

wget http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.7.0_x86_64.tar.gz
    tar zxvf cloudera-manager-centos7-cm5.7.0_x86_64.tar.gz 
    mkdir cloudera-manager
    mv cloudera cloudera-manager
    mv cm-5.7.0/ cloudera-manager

## configuration

vi /root/cloudera-manager/cm-5.7.0/etc/cloudera-scm-server/db.properties
    
    com.cloudera.cmf.db.type=postgresql
    # The database host
    # If a non standard port is needed, use 'hostname:port'
    com.cloudera.cmf.db.host=node1
    # The database name
    com.cloudera.cmf.db.name=scm
    # The database user
    com.cloudera.cmf.db.user=scm
    # The database user's password
    com.cloudera.cmf.db.password=scm

vi /root/cloudera-manager/cm-5.7.0/etc/cloudera-scm-agent/config.ini
    server_host = node1
    
    scp -r cloudera-manager nodex:~/cloudera-manager
    
## create user cloudera-scm

useradd --system --home=/root/cloudera-manager/cm-5.7.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
    chown -R cloudera-scm:cloudera-scm /root/cloudera-manager/cm-5.7.0/run/cloudera-scm-server/
    chown root:root /root/
    
Create the Cloudera Manager Server Local Data Storage Directory

mkdir /var/log/cloudera-scm-server
    chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-server
    mkdir ~/cloudera-manager/cm-5.7.0/run/cloudera-scm-agent

## Create Parcel Directories

mkdir -p /opt/cloudera/parcel-repo
    chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
    mkdir -p /opt/cloudera/parcels
    chown cloudera-scm:cloudera-scm /opt/cloudera/parcels
    
## start server

./cloudera-manager/cm-5.7.0/etc/init.d/cloudera-scm-server start
    tail -100f ./cloudera-manager/cm-5.7.0/log/cloudera-scm-server/cloudera-scm-server.log
    
## start agent

./cloudera-manager/cm-5.7.0/etc/init.d/cloudera-scm-agent start
    tail -100f ./cloudera-manager/cm-5.7.0/log/cloudera-scm-agent/cloudera-scm-agent.log
    
## web ui

http://node1:7180/
    admin
    admin

Navigation

Recent Posts

Friend Links