CDH5 Installation Path-C

# Before You Begin

## SSH Configuration

vi /etc/hosts
    x.x.x.x linode01
    x.x.x.x linode02
    x.x.x.x linode03
    
    hostnamectl set-hostname linode01
    
    ssh-keygen -t rsa
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode01
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode02
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode03
    
## Disable Firewall

systemctl stop firewalld.service
    systemctl disable firewalld.service

## Dependency

yum install psmisc -y
    yum install libxslt-devel -y
    yum install chkconfig bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs redhat-lsb -y
    yum install python-psycopg2 -y 
    yum install snappy snappy-devel  -y
    
    #NFS
    yum install rpcbind -y
    service rpcbind start

## Install the Oracle JDK

cd /opt/
    sudo wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn/java/jdk/7u80-b15/jdk-7u80-linux-x64.tar.gz"
    
    scp /opt/jdk-7u80-linux-x64.tar.gz linode02:/opt/
    scp /opt/jdk-7u80-linux-x64.tar.gz linode03:/opt/
    
    cd /opt/
    tar zxf jdk-7u80-linux-x64.tar.gz
    cd
    
    echo 'export JAVA_HOME=/opt/jdk1.7.0_80' >> /etc/profile 
    echo 'export JRE_HOME=/opt/jdk1.7.0_80/jre' >> /etc/profile
    echo 'export PATH=$PATH:/opt/jdk1.7.0_80/bin' >> /etc/profile
    source /etc/profile
    echo 'JAVA_HOME=/opt/jdk1.7.0_80' >> /etc/environment

## Install and Configure External Databases

sudo yum install postgresql-server postgresql -y
    sudo su - postgres
    initdb -D /var/lib/pgsql/data
    
    #remote access
    vi /var/lib/pgsql/data/postgresql.conf
    listen_addresses ='*'

vi /var/lib/pgsql/data/pg_hba.conf
    host all all 0.0.0.0/0 trust
    
    #start service
    exit
    systemctl restart postgresql.service
    
    #set password
    su - postgres
    psql
    \password postgres
    
    #create cloudera-manager database
    sudo -u postgres psql
    
    CREATE ROLE scm LOGIN PASSWORD 'scm';
    CREATE DATABASE scm OWNER scm ENCODING 'UTF8'; 
    
    CREATE ROLE amon LOGIN PASSWORD 'amon_password';
    CREATE DATABASE amon OWNER amon ENCODING 'UTF8';
    CREATE ROLE rman LOGIN PASSWORD 'rman_password';
    CREATE DATABASE rman OWNER rman ENCODING 'UTF8';
    CREATE ROLE hive LOGIN PASSWORD 'hive_password';
    CREATE DATABASE metastore OWNER hive ENCODING 'UTF8';
    CREATE ROLE sentry LOGIN PASSWORD 'sentry_password';
    CREATE DATABASE sentry OWNER sentry ENCODING 'UTF8';
    CREATE ROLE nav LOGIN PASSWORD 'nav_password';
    CREATE DATABASE nav OWNER nav ENCODING 'UTF8';
    CREATE ROLE navms LOGIN PASSWORD 'navms_password';
    CREATE DATABASE navms OWNER navms ENCODING 'UTF8'; 
    CREATE ROLE oozie LOGIN PASSWORD 'oozie_password';
    CREATE DATABASE oozie OWNER oozie ENCODING 'UTF8';
    CREATE ROLE hue LOGIN PASSWORD 'hue_password';
    CREATE DATABASE hue OWNER hue ENCODING 'UTF8';
    
    #For PostgreSQL 8.2.23 or higher, also run:
    ALTER DATABASE Metastore SET standard_conforming_strings = off;

# Install the Cloudera Manager Server and Agents

## Download tarball

cd /opt/
    wget http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.8.0_x86_64.tar.gz
    tar zxf cloudera-manager-centos7-cm5.8.0_x86_64.tar.gz
    mkdir cloudera-manager
    mv cloudera cloudera-manager
    mv cm-5.8.0/ cloudera-manager

## Configure Cloudera Manager Server

vi /opt/cloudera-manager/cm-5.8.0/etc/cloudera-scm-server/db.properties
    
    -----------------------------------------------------------------------
    
    # Copyright (c) 2012 Cloudera, Inc. All rights reserved.
    #
    # This file describes the database connection.
    #
    
    # The database type
    # Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
    com.cloudera.cmf.db.type=postgresql
    
    # The database host
    # If a non standard port is needed, use 'hostname:port'
    com.cloudera.cmf.db.host=linode02
    
    # The database name
    com.cloudera.cmf.db.name=scm
    
    # The database user
    com.cloudera.cmf.db.user=scm
    
    # The database user's password
    com.cloudera.cmf.db.password=scm

## Configure Cloudera Manager Agents

vi /opt/cloudera-manager/cm-5.8.0/etc/cloudera-scm-agent/config.ini 
    server_host=linode01

## Copy to Each Cluster Host

scp -r /opt/cloudera-manager/  linode02:/opt/cloudera-manager
    scp -r /opt/cloudera-manager/  linode03:/opt/cloudera-manager

## Create Users

The Cloudera Manager Server and managed services require a user account to complete tasks. When installing Cloudera Manager from tarballs, you must create this user account on all hosts manually. Because Cloudera Manager Server and managed services are configured to use the user account cloudera-scm by default, creating a user with this name is the simplest approach. This created user, is used automatically after installation is complete.

sudo useradd --system --home=/opt/cloudera-manager/cm-5.8.0/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

Give the single user passwordless sudo access.

usermod -a -G sudo cloudera-scm
    
Sudo must be configured so that /usr/sbin is in the path when running sudo.

visudo  -f /etc/sudoers
     #add
     Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin
     
Set up per user limits for su prior to setting up the Agent.

vi /etc/pam.d/su
    #Uncomment
    session required pam_limits.so

## Create the Cloudera Manager Server Local Data Storage Directory

sudo mkdir /var/log/cloudera-scm-server
    sudo chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-server

## Configuring for a Custom Cloudera Manager User and Custom Directories

mkdir -p /var/log/cloudera-scm-headlamp
    chown -R cloudera-scm:cloudera-scm /var/log/cloudera-scm-headlamp
    mkdir -p /var/log/cloudera-scm-firehose
    chown -R cloudera-scm:cloudera-scm /var/log/cloudera-scm-firehose
    mkdir -p /var/log/cloudera-scm-alertpublisher
    chown -R cloudera-scm:cloudera-scm /var/log/cloudera-scm-alertpublisher
    mkdir -p /var/log/cloudera-scm-eventserver
    chown -R cloudera-scm:cloudera-scm /var/log/cloudera-scm-eventserver
    mkdir -p /var/lib/cloudera-scm-headlamp
    chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-headlamp
    mkdir -p /var/lib/cloudera-scm-firehose
    chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-firehose
    mkdir -p /var/lib/cloudera-scm-alertpublisher
    chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-alertpublisher
    mkdir -p /var/lib/cloudera-scm-eventserver
    chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-eventserver
    mkdir -p /var/lib/cloudera-scm-server
    chown -R cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server
    mkdir -p  /var/lib/hadoop-httpfs
    chown -R cloudera-scm:cloudera-scm  /var/lib/hadoop-httpfs
    mkdir -p /var/lib/oozie
    chown -R cloudera-scm:cloudera-scm /var/lib/oozie
    mkdir -p  /var/lib/sqoop2
    chown -R cloudera-scm:cloudera-scm  /var/lib/sqoop2
    mkdir -p /var/lib/solr
    chown -R cloudera-scm:cloudera-scm /var/lib/solr

## Create Parcel Directories

#Server host
    mkdir -p /opt/cloudera/parcel-repo
    chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
    
    #each cluster host
    mkdir -p /opt/cloudera/parcels
    chown cloudera-scm:cloudera-scm /opt/cloudera/parcels

## Swap

dd if=/dev/zero of=/swap-file bs=1M count=2048
    mkswap /swap-file
    swapon /swap-file
    swapon -s
    
## Start the Cloudera Manager Server

sudo ./cloudera-manager/cm-5.8.0/etc/init.d/cloudera-scm-server start
    tail -100f ./cloudera-manager/cm-5.8.0/log/cloudera-scm-server/cloudera-scm-server.log
    
## Start the Cloudera Manager Agents
    
    mkdir /opt/cloudera-manager/cm-5.8.0/run/cloudera-scm-agent
    sudo ./cloudera-manager/cm-5.8.0/etc/init.d/cloudera-scm-agent start
    tail -100f ./cloudera-manager/cm-5.8.0/log/cloudera-scm-agent/cloudera-scm-agent.log
    
## Web UI

http://node1:7180/
    admin
    admin

## Mount and export "/" by NFS

mount -t nfs -o vers=3,proto=tcp,nolock,noacl linode03:/  /hdfs

## spark configurati'o'ntion

vi /opt/cloudera/parcels/CDH/lib/spark/conf/spark-env.sh

Navigation

Recent Posts

Friend Links