##1)Environment initialization Note：At least three nodes, the Master node at least 6G of memory, and the other two at least 4G of memory |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| | Host | Server | Agent | zookeeper | namenode | secondarynamenode | datanode | |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh01 Master | Y | Y | Y | Y | | | |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh02 slave | | Y | Y | | Y | Y | |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh03 slave | | Y | Y | | | Y | |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh04 slave | | Y | Y | | | Y | |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh05 slave | | Y | Y | | | Y | |______________|___________|_____________|________________|___________|____________________|_____________| ####################################### ##Name resolution on every nodes ######################## cat > /etc/hosts <<EOF 127.0.0.1 localhost 192.168.51.6 cdh01 192.168.51.7 cdh02 192.168.51.16 cdh03 192.168.51.17 cdh04 192.168.51.4 cdh05 EOF ##We must to disable ipv6 on every nodes echo 'net.ipv6.conf.all.disable_ipv6 = 1'>>/etc/sysctl.conf echo 'net.ipv6.conf.default.disable_ipv6 = 1' >>/etc/sysctl.conf echo 'vm.swappiness = 0' >>/etc/sysctl.conf sysctl -p echo never > /sys/kernel/mm/transparent_hugepage/defrag echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local chmod +x /etc/rc.d/rc.local groupadd hadoop -g 700 useradd hadoop -g hadoop -u 700 echo "hadoop123"|passwd --stdin hadoop echo 'hadoop ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers echo "ulimit -SHn 204800" >> /etc/rc.local echo "ulimit -SHu 204800" >> /etc/rc.local cat >> /etc/security/limits.conf << EOF * soft nofile 204800 * hard nofile 204800 * soft nproc 204800 * hard nproc 204800 EOF #Install jdk on every nodes wget http://god.nongdingbang.net/downloads/auto_jdk_1.8.sh sh auto_jdk_1.8.sh #Install java-jdbc on every nodes cd /home/tools wget http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.39.tar.gz tar zxvf mysql-connector-java-5.1.39.tar.gz mkdir /usr/share/java/ /cp mysql-connector-java-5.1.39/mysql-connector-java-5.1.39-bin.jar /usr/share/java/mysql-connector-java.jar ######################################################################################################## ##2)Deploy local yum for CDH on cdh01 yum -y install epel-release yum -y install nginx createrepo cat > /etc/nginx/nginx.conf <<EOF user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; # Load dynamic modules. See /usr/share/nginx/README.dynamic. include /usr/share/nginx/modules/*.conf; events { worker_connections 102400; } http { log_format main '/$remote_addr - /$remote_user [/$time_local] "/$request" ' '/$status /$body_bytes_sent "/$http_referer" ' '"/$http_user_agent" "/$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 120; types_hash_max_size 2048; client_max_body_size 1G; client_header_timeout 10; client_body_timeout 10; fastcgi_connect_timeout 300; fastcgi_send_timeout 300; fastcgi_read_timeout 300; fastcgi_buffer_size 64k; fastcgi_buffers 4 64k; fastcgi_busy_buffers_size 128k; fastcgi_temp_file_write_size 128k; server_names_hash_bucket_size 64; gzip on; gzip_min_length 1024; gzip_buffers 4 64k; gzip_http_version 1.1; gzip_comp_level 9; gzip_types text/plain text/css text/xml text/x-component text/javascript application/x-javascript application/javascript application/json application/xhtml+xml image/svg+xml image/x-icon image/gif image/png image/jpeg video/mp4 video/x-flv video/x-m4v; gzip_disable "msie6"; gzip_vary on; include /etc/nginx/mime.types; default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; } EOF mkdir -p /data/www/html/ cat > /etc/nginx/conf.d/default.conf <<EOF server { listen 80; server_name localhost; #charset koi8-r; #access_log /var/log/nginx/log/host.access.log main; location / { root /data/www/html; autoindex on; #允许浏览文件 } #error_page 404 /404.html; # redirect server error pages to the static page /50x.html # error_page 500 502 503 504 /50x.html; location = /50x.html { root /usr/share/nginx/html; } } EOF cd /data/www/html/ && wget http://god.nongdingbang.net/downloads/cm5.11.0-centos7.tar.gz tar zxvf cm5.11.0-centos7.tar.gz && chown -R nginx.nginx /data/www/html cd /data/www/html/cm/5.11 && createrepo . #run this command on every nodes cat >/etc/yum.repos.d/cloudera-cm.repo <<EOF [cloudera-cm] # Packages for Cloudera's Distribution for cm, Version 5, on RedHat or CentOS 7 x86_64 name=Cloudera's Distribution for cm, Version 5 baseurl=http://192.168.51.6/cm/5.11 gpgcheck = 0 EOF #Start nginx on cdh01 service nginx start && chkconfig nginx on ##3)Install cloudera-agent on every nodes yum install -y vim wget openssh-clients python chrony cloudera-manager-agent cloudera-manager-daemons ##4)Install cloudera-magener on cdh01 yum install -y cloudera-manager-daemons cloudera-manager-server ##5)Install Mysql Server on cdh01 cd /home/tools yum -y install numactl libaio net-tools wget http://god.nongdingbang.net/downloads/mysql-5.7-el7.tgz && tar zxvf mysql-5.7-el7.tgz if [ $? -eq 0 ];then rpm -Uvh mysql*.rpm fi mkdir -p /data/{mysql_data,mysql_log,mysql_slow,mysql_undo} chown -R mysql.mysql /data/* cat >/etc/my.cnf<<EOF [client] port=3306 socket=/tmp/mysql.sock [mysql] no-auto-rehash [mysqld] port=3306 character-set-server=utf8 socket=/tmp/mysql.sock datadir=/data/mysql_data explicit_defaults_for_timestamp=true lower_case_table_names=1 sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' back_log=103 max_connections=3000 max_connect_errors=100000 table_open_cache=512 external-locking=FALSE max_allowed_packet=32M sort_buffer_size=2M join_buffer_size=2M thread_cache_size=51 query_cache_size=0 query_cache_type=0 transaction_isolation=REPEATABLE-READ tmp_table_size=96M max_heap_table_size=96M log-error=/data/mysql_log/error.log ###***slowqueryparameters long_query_time=2 slow_query_log=1 slow_query_log_file=/data/mysql_slow/slow.log ###***binlogparameters log-bin=mysql-bin binlog_cache_size=4M max_binlog_cache_size=4096M max_binlog_size=1024M binlog_format=ROW binlog_row_image=minimal expire_logs_days=3 sync_binlog=0 ###***undolog innodb_undo_directory=/data/mysql_undo innodb_undo_logs=128 innodb_undo_tablespaces=4 innodb_undo_log_truncate=1 innodb_max_undo_log_size=1G innodb_purge_rseg_truncate_frequency #***MyISAMparameters key_buffer_size=16M read_buffer_size=1M read_rnd_buffer_size=16M bulk_insert_buffer_size=1M ###***master-slavereplicationparameters server-id=1 #slave-skip-errors=all #***Innodbstorageengineparameters innodb_buffer_pool_dump_at_shutdown=1 innodb_buffer_pool_load_at_startup=1 innodb_buffer_pool_size=16G innodb_data_file_path=ibdata1:10M:autoextend innodb_temp_data_file_path = ibtmp1:12M:autoextend:max:5G #innodb_file_io_threads=8 innodb_thread_concurrency=0 innodb_flush_log_at_trx_commit=2 innodb_log_buffer_size=16M innodb_log_file_size=512M innodb_log_files_in_group=2 innodb_max_dirty_pages_pct=75 innodb_buffer_pool_dump_pct=50 innodb_lock_wait_timeout=50 innodb_file_per_table=on innodb_flush_neighbors=1 innodb_io_capacity_max = 2000 wait_timeout = 14400 interactive_timeout = 14400 [mysqldump] quick max_allowed_packet=32M [myisamchk] key_buffer=16M sort_buffer_size=16M read_buffer=8M write_buffer=8M [mysqld_safe] open-files-limit=28192 log-error=/data/mysql_log/error.log pid-file=/data/mysql_data/mysqld.pid EOF ##6)Initialize MySQL configuration mysqld --defaults-file=/etc/my.cnf --user=mysql / --datadir=/data/mysql_data --initialize-insecure ##7)Start mysql systemctl restart mysqld && systemctl enable mysqld ##8)setting root's password for mysql on cdh01 mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'Ad@sd119';" mysql -uroot -pAd@sd119 -Dmysql -e "select user,host,authentication_string,password_expired from user;" mysql -uroot -pAd@sd119 -e "create database scm default character set utf8;grant all privileges on *.* to scm@'%' identified by 'scm';" mysql -uroot -pAd@sd119 -e "create database amon default character set utf8;grant all privileges on amon.* to 'amon'@'%' identified by 'amon_password';" mysql -uroot -pAd@sd119 -e "create database rman default character set utf8;grant all privileges on rman.* to 'rman'@'%' identified by 'rman_password';" mysql -uroot -pAd@sd119 -e "create database metastore default character set utf8;grant all privileges on metastore.* to 'hive'@'%' identified by 'hive_password'" mysql -uroot -pAd@sd119 -e "create database sentry default character set utf8;grant all privileges on sentry.* to 'sentry'@'%' identified by 'sentry_password'" mysql -uroot -pAd@sd119 -e "create database nav default character set utf8;grant all privileges on nav.* to 'nav'@'%' identified by 'nav_password'" mysql -uroot -pAd@sd119 -e "create database navms default character set utf8;grant all privileges on navms.* to 'navms'@'%' identified by 'navms_password';" mysql -uroot -pAd@sd119 -e "create database hue default character set utf8;grant all privileges on hue.* to 'hue'@'%' identified by 'hue_password'" mysql -uroot -pAd@sd119 -e "flush privileges;" #9)Create this directory on every nodes mkdir -p /opt/cloudera/parcels chown cloudera-scm:cloudera-scm /opt/cloudera/parcels #10)Start Server on cdh01 ############################# cat >/etc/cloudera-scm-server/db.properties <<EOF com.cloudera.cmf.db.type=mysql com.cloudera.cmf.db.host=cdh01 com.cloudera.cmf.db.name=scm com.cloudera.cmf.db.user=scm com.cloudera.cmf.db.password=scm com.cloudera.cmf.db.setupType=EXTERNAL EOF cd /opt/cloudera/parcel-repo wget -c http://god.nongdingbang.net/downloads/parcel-repo.tgz if [ $? -eq 0 ];then tar zxvf parcel-repo.tgz fi mv parcel-repo/* . rm -rf parcel-repo* /etc/init.d/cloudera-scm-server start chkconfig cloudera-scm-server on #11)Start agentd on every nodes sed -i 's/server_host=localhost/server_host=192.168.51.6/' /etc/cloudera-scm-agent/config.ini sed -i 's#/# parcel_dir=/opt/cloudera/parcels#parcel_dir=/opt/cloudera/parcels#' /etc/cloudera-scm-agent/config.ini cat >/etc/cloudera-scm-agent/config.ini<<EOF server_host=cdh01 server_port=7182 parcel_dir=/opt/cloudera/parcels EOF /etc/init.d/cloudera-scm-agent start chkconfig cloudera-scm-agent on ##12)Browse your cdh http://192.168.51.6:7180/cmf/login
原创文章，作者：ItWorker，如若转载，请注明出处：https://blog.ytso.com/193996.html