##1)Environmentinitialization Note:Atleastthreenodes,theMasternodeatleast6Gofmemory,andtheothertwoatleast4Gofmemory |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |Host|Server|Agent|zookeeper|namenode|secondarynamenode|datanode| |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh01Master|Y|Y|Y|Y||| |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh02slave||Y|Y||Y|Y| |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh03slave||Y|Y|||Y| |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh04slave||Y|Y|||Y| |--------------|-----------|-------------|----------------|-----------|--------------------|-------------| |cdh05slave||Y|Y|||Y| |______________|___________|_____________|________________|___________|____________________|_____________| ####################################### ##Nameresolutiononeverynodes ######################## cat>/etc/hosts<<EOF 127.0.0.1localhost 192.168.51.6cdh01 192.168.51.7cdh02 192.168.51.16cdh03 192.168.51.17cdh04 192.168.51.4cdh05 EOF ##Wemusttodisableipv6oneverynodes echo'net.ipv6.conf.all.disable_ipv6=1'>>/etc/sysctl.conf echo'net.ipv6.conf.default.disable_ipv6=1'>>/etc/sysctl.conf echo'vm.swappiness=0'>>/etc/sysctl.conf sysctl-p echonever>/sys/kernel/mm/transparent_hugepage/defrag echo'echonever>/sys/kernel/mm/transparent_hugepage/defrag'>>/etc/rc.local chmod+x/etc/rc.d/rc.local groupaddhadoop-g700 useraddhadoop-ghadoop-u700 echo"hadoop123"|passwd--stdinhadoop echo'hadoopALL=(ALL)nopASSWD:ALL'>>/etc/sudoers echo"ulimit-SHn204800">>/etc/rc.local echo"ulimit-SHu204800">>/etc/rc.local cat>>/etc/security/limits.conf<<EOF *softnofile204800 *hardnofile204800 *softnproc204800 *hardnproc204800 EOF #Installjdkoneverynodes wgethttp://god.nongdingbang.net/downloads/auto_jdk_1.8.sh shauto_jdk_1.8.sh #Installjava-jdbconeverynodes cd/home/tools wgethttp://dev.MysqL.com/get/Downloads/connector-j/mysql-connector-java-5.1.39.tar.gz tarzxvfmysql-connector-java-5.1.39.tar.gz mkdir/usr/share/java/ \cpmysql-connector-java-5.1.39/mysql-connector-java-5.1.39-bin.jar/usr/share/java/mysql-connector-java.jar ######################################################################################################## ##2)DeploylocalyumforCDHoncdh01 yum-yinstallepel-release yum-yinstallNginxcreaterepo cat>/etc/Nginx/Nginx.conf<<EOF userNginx; worker_processesauto; error_log/var/log/Nginx/error.log; pid/run/Nginx.pid; #Loaddynamicmodules.See/usr/share/Nginx/README.dynamic. include/usr/share/Nginx/modules/*.conf; events{ worker_connections102400; } http{ log_formatmain'$remote_addr-$remote_user[$time_local]"$request"' '$status$body_bytes_sent"$http_referer"' '"$http_user_agent""$http_x_forwarded_for"'; access_log/var/log/Nginx/access.logmain; sendfileon; tcp_nopushon; tcp_nodelayon; keepalive_timeout120; types_hash_max_size2048; client_max_body_size1G; client_header_timeout10; client_body_timeout10; fastcgi_connect_timeout300; fastcgi_send_timeout300; fastcgi_read_timeout300; fastcgi_buffer_size64k; fastcgi_buffers464k; fastcgi_busy_buffers_size128k; fastcgi_temp_file_write_size128k; server_names_hash_bucket_size64; gzipon; gzip_min_length1024; gzip_buffers464k; gzip_http_version1.1; gzip_comp_level9; gzip_typestext/plaintext/csstext/xmltext/x-componenttext/javascriptapplication/x-javascriptapplication/javascriptapplication/jsonapplication/xhtml+xmlimage/svg+xmlimage/x-iconimage/gifimage/pngimage/jpegvideo/mp4video/x-flvvideo/x-m4v; gzip_disable"msie6"; gzip_varyon; include/etc/Nginx/mime.types; default_typeapplication/octet-stream; include/etc/Nginx/conf.d/*.conf; } EOF mkdir-p/data/www/html/ cat>/etc/Nginx/conf.d/default.conf<<EOF server{ listen80; server_namelocalhost; #charsetkoi8-r; #access_log/var/log/Nginx/log/host.access.logmain; location/{ root/data/www/html; autoindexon;#允许浏览文件 } #error_page404/404.html; #redirectservererrorpagestothestaticpage/50x.html # error_page500502503504/50x.html; location=/50x.html{ root/usr/share/Nginx/html; } } EOF cd/data/www/html/&&wgethttp://god.nongdingbang.net/downloads/cm5.11.0-centos7.tar.gz tarzxvfcm5.11.0-centos7.tar.gz&&chown-RNginx.Nginx/data/www/html cd/data/www/html/cm/5.11&&createrepo. #runthiscommandoneverynodes cat>/etc/yum.repos.d/cloudera-cm.repo<<EOF [cloudera-cm] #PackagesforCloudera'sdistributionforcm,Version5,onRedHatorCentOS7x86_64 name=Cloudera'sdistributionforcm,Version5 baseurl=http://192.168.51.6/cm/5.11 gpgcheck=0 EOF #StartNginxoncdh01 serviceNginxstart&&chkconfigNginxon ##3)Installcloudera-agentoneverynodes yuminstall-yvimwgetopenssh-clientspythonchronycloudera-manager-agentcloudera-manager-daemons ##4)Installcloudera-mageneroncdh01 yuminstall-ycloudera-manager-daemonscloudera-manager-server ##5)InstallMysqLServeroncdh01 cd/home/tools yum-yinstallnumactllibaionet-tools wgethttp://god.nongdingbang.net/downloads/MysqL-5.7-el7.tgz&&tarzxvfMysqL-5.7-el7.tgz if[$?-eq0];then rpm-UvhMysqL*.rpm fi mkdir-p/data/{MysqL_data,MysqL_log,MysqL_slow,MysqL_undo} chown-RMysqL.MysqL/data/* cat>/etc/my.cnf<<EOF [client] port=3306 socket=/tmp/MysqL.sock [MysqL] no-auto-rehash [MysqLd] port=3306 character-set-server=utf8 socket=/tmp/MysqL.sock datadir=/data/MysqL_data explicit_defaults_for_timestamp=true lower_case_table_names=1 sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION' back_log=103 max_connections=3000 max_connect_errors=100000 table_open_cache=512 external-locking=FALSE max_allowed_packet=32M sort_buffer_size=2M join_buffer_size=2M thread_cache_size=51 query_cache_size=0 query_cache_type=0 transaction_isolation=REPEATABLE-READ tmp_table_size=96M max_heap_table_size=96M log-error=/data/MysqL_log/error.log ###***slowqueryparameters long_query_time=2 slow_query_log=1 slow_query_log_file=/data/MysqL_slow/slow.log ###***binlogparameters log-bin=MysqL-bin binlog_cache_size=4M max_binlog_cache_size=4096M max_binlog_size=1024M binlog_format=ROW binlog_row_image=minimal expire_logs_days=3 sync_binlog=0 ###***undolog innodb_undo_directory=/data/MysqL_undo innodb_undo_logs=128 innodb_undo_tablespaces=4 innodb_undo_log_truncate=1 innodb_max_undo_log_size=1G innodb_purge_rseg_truncate_frequency #***MyISAMparameters key_buffer_size=16M read_buffer_size=1M read_rnd_buffer_size=16M bulk_insert_buffer_size=1M ###***master-slavereplicationparameters server-id=1 #slave-skip-errors=all #***Innodbstorageengineparameters innodb_buffer_pool_dump_at_shutdown=1 innodb_buffer_pool_load_at_startup=1 innodb_buffer_pool_size=16G innodb_data_file_path=ibdata1:10M:autoextend innodb_temp_data_file_path=ibtmp1:12M:autoextend:max:5G #innodb_file_io_threads=8 innodb_thread_concurrency=0 innodb_flush_log_at_trx_commit=2 innodb_log_buffer_size=16M innodb_log_file_size=512M innodb_log_files_in_group=2 innodb_max_dirty_pages_pct=75 innodb_buffer_pool_dump_pct=50 innodb_lock_wait_timeout=50 innodb_file_per_table=on innodb_flush_neighbors=1 innodb_io_capacity_max=2000 wait_timeout=14400 interactive_timeout=14400 [MysqLdump] quick max_allowed_packet=32M [myisamchk] key_buffer=16M sort_buffer_size=16M read_buffer=8M write_buffer=8M [MysqLd_safe] open-files-limit=28192 log-error=/data/MysqL_log/error.log pid-file=/data/MysqL_data/MysqLd.pid EOF ##6)InitializeMysqLconfiguration MysqLd--defaults-file=/etc/my.cnf--user=MysqL\ --datadir=/data/MysqL_data--initialize-insecure ##7)StartMysqL systemctlrestartMysqLd&&systemctlenableMysqLd ##8)settingroot'spasswordforMysqLoncdh01 MysqL-e"ALTERUSER'root'@'localhost'IDENTIFIEDBY'Ad@sd119';" MysqL-uroot-pAd@sd119-DMysqL-e"selectuser,host,authentication_string,password_expiredfromuser;" MysqL-uroot-pAd@sd119-e"createdatabasescmdefaultcharactersetutf8;grantallprivilegeson*.*toscm@'%'identifiedby'scm';" MysqL-uroot-pAd@sd119-e"createdatabaseamondefaultcharactersetutf8;grantallprivilegesonamon.*to'amon'@'%'identifiedby'amon_password';" MysqL-uroot-pAd@sd119-e"createdatabasermandefaultcharactersetutf8;grantallprivilegesonrman.*to'rman'@'%'identifiedby'rman_password';" MysqL-uroot-pAd@sd119-e"createdatabasemetastoredefaultcharactersetutf8;grantallprivilegesonmetastore.*to'hive'@'%'identifiedby'hive_password'" MysqL-uroot-pAd@sd119-e"createdatabasesentrydefaultcharactersetutf8;grantallprivilegesonsentry.*to'sentry'@'%'identifiedby'sentry_password'" MysqL-uroot-pAd@sd119-e"createdatabasenavdefaultcharactersetutf8;grantallprivilegesonnav.*to'nav'@'%'identifiedby'nav_password'" MysqL-uroot-pAd@sd119-e"createdatabasenavmsdefaultcharactersetutf8;grantallprivilegesonnavms.*to'navms'@'%'identifiedby'navms_password';" MysqL-uroot-pAd@sd119-e"createdatabasehuedefaultcharactersetutf8;grantallprivilegesonhue.*to'hue'@'%'identifiedby'hue_password'" MysqL-uroot-pAd@sd119-e"flushprivileges;" #9)Createthisdirectoryoneverynodes mkdir-p/opt/cloudera/parcels chowncloudera-scm:cloudera-scm/opt/cloudera/parcels #10)StartServeroncdh01 ############################# cat>/etc/cloudera-scm-server/db.properties<<EOF com.cloudera.cmf.db.type=MysqL com.cloudera.cmf.db.host=cdh01 com.cloudera.cmf.db.name=scm com.cloudera.cmf.db.user=scm com.cloudera.cmf.db.password=scm com.cloudera.cmf.db.setupType=EXTERNAL EOF cd/opt/cloudera/parcel-repo wget-chttp://god.nongdingbang.net/downloads/parcel-repo.tgz if[$?-eq0];then tarzxvfparcel-repo.tgz fi mvparcel-repo/*. rm-rfparcel-repo* /etc/init.d/cloudera-scm-serverstart chkconfigcloudera-scm-serveron #11)Startagentdoneverynodes sed-i's/server_host=localhost/server_host=192.168.51.6/'/etc/cloudera-scm-agent/config.ini sed-i's#\#parcel_dir=/opt/cloudera/parcels#parcel_dir=/opt/cloudera/parcels#'/etc/cloudera-scm-agent/config.ini cat>/etc/cloudera-scm-agent/config.ini<<EOF server_host=cdh01 server_port=7182 parcel_dir=/opt/cloudera/parcels EOF /etc/init.d/cloudera-scm-agentstart chkconfigcloudera-scm-agenton ##12)browseyourcdh http://192.168.51.6:7180/cmf/login
原文地址:https://www.jb51.cc/centos/376574.html
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。