CDH5 Installation Path-B

# Before You Begin

## SSH Configuration

vi /etc/hosts
    x.x.x.x linode01
    x.x.x.x linode02
    x.x.x.x linode03
    
    hostnamectl set-hostname linode01
    
    ssh-keygen -t rsa
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode01
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode02
    ssh-copy-id -i ~/.ssh/id_rsa.pub linode03
    
## Disable Firewall

systemctl stop firewalld.service
    systemctl disable firewalld.service

## Dependency

yum install psmisc -y
    yum install libxslt-devel -y
    yum install chkconfig bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs redhat-lsb -y
    yum install python-psycopg2 -y 
    yum install snappy snappy-devel  -y
    
    #NFS
    yum install rpcbind -y
    service rpcbind start

## Install and Configure External Databases

sudo yum install postgresql-server postgresql -y
    sudo su - postgres
    initdb -D /var/lib/pgsql/data
    
    #remote access
    vi /var/lib/pgsql/data/postgresql.conf
    listen_addresses ='*'

vi /var/lib/pgsql/data/pg_hba.conf
    host all all 0.0.0.0/0 trust
    
    #start service
    exit
    systemctl restart postgresql.service
    
    #set password
    su - postgres
    psql
    \password postgres
    
    #create cloudera-manager database
    sudo -u postgres psql
    
    CREATE ROLE scm LOGIN PASSWORD 'scm';
    CREATE DATABASE scm OWNER scm ENCODING 'UTF8'; 
    
    CREATE ROLE amon LOGIN PASSWORD 'amon_password';
    CREATE DATABASE amon OWNER amon ENCODING 'UTF8';
    CREATE ROLE rman LOGIN PASSWORD 'rman_password';
    CREATE DATABASE rman OWNER rman ENCODING 'UTF8';
    CREATE ROLE hive LOGIN PASSWORD 'hive_password';
    CREATE DATABASE metastore OWNER hive ENCODING 'UTF8';
    CREATE ROLE sentry LOGIN PASSWORD 'sentry_password';
    CREATE DATABASE sentry OWNER sentry ENCODING 'UTF8';
    CREATE ROLE nav LOGIN PASSWORD 'nav_password';
    CREATE DATABASE nav OWNER nav ENCODING 'UTF8';
    CREATE ROLE navms LOGIN PASSWORD 'navms_password';
    CREATE DATABASE navms OWNER navms ENCODING 'UTF8'; 
    CREATE ROLE oozie LOGIN PASSWORD 'oozie_password';
    CREATE DATABASE oozie OWNER oozie ENCODING 'UTF8';
    CREATE ROLE hue LOGIN PASSWORD 'hue_password';
    CREATE DATABASE hue OWNER hue ENCODING 'UTF8';
    
    #For PostgreSQL 8.2.23 or higher, also run:
    ALTER DATABASE Metastore SET standard_conforming_strings = off;

# Cloudera Manager Repository

vi cloudera-manager.repo
    ------
    [cloudera-manager]
    # Packages for Cloudera Manager, Version 5, on RedHat or CentOS 7 x86_64           	  
    name=Cloudera Manager
    baseurl=https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5/
    gpgkey =https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera    
    gpgcheck = 1
    
    
    mv cloudera-manager.repo /etc/yum.repos.d/

# Install Cloudera Manager Server Software

## Install the Oracle JDK on the Cloudera Manager Server Host

sudo yum install oracle-j2sdk1.7
    
## Install the Cloudera Manager Server Packages

sudo yum install cloudera-manager-daemons cloudera-manager-server
    
## Configure Cloudera Manager Server

vi /etc/cloudera-scm-server/db.properties 
    -----------------------------------------
    # Copyright (c) 2012 Cloudera, Inc. All rights reserved.
    #
    # This file describes the database connection.
    #
    # The database type
    # Currently 'mysql', 'postgresql' and 'oracle' are valid databases.
    com.cloudera.cmf.db.type=postgresql
    # The database host
    # If a non standard port is needed, use 'hostname:port'
    com.cloudera.cmf.db.host=linode02
    # The database name
    com.cloudera.cmf.db.name=scm
    # The database user
    com.cloudera.cmf.db.user=scm
    # The database user's password
    com.cloudera.cmf.db.password=scm
 
## Swap

dd if=/dev/zero of=/swap-file bs=2M count=2048
    mkswap /swap-file
    swapon /swap-file
    swapon -s
    free -m

## Start the Cloudera Manager Server

sudo service cloudera-scm-server start
    tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

## Start and Log into the Cloudera Manager Admin Console

The Cloudera Manager Server URL takes the following form `http://Server host:port`
The default port is 7180
 The default credentials are: Username: admin Password: admin

## Choose Cloudera Manager Hosts

You can specify multiple addresses and address ranges by separating them with commas, semicolons, tabs, or blank spaces, or by placing them on separate lines.

linode01,linode02,linode03

Click Search. Cloudera Manager identifies the hosts on your cluster to allow you to configure them for services.

## Monute NFS

Execute the following command to verify if all the services are up and running: 
    
    rpcinfo -p $nfs_server_ip
    
Verify if the HDFS namespace is exported and can be mounted.

showmount -e $nfs_server_ip
    
Mount the export "/"

mount -t nfs -o vers=3,proto=tcp,nolock,noacl linode0x:/  /hdfs

## Run Spark Shell

sudo -u spark spark-shell

Navigation

Recent Posts

Friend Links