本文导读
CDH 部署
主机映射
hostnamectl set-hostname cluster1-hadoop01-yx-daas-deepexi vim /etc/hosts # 192.168.0.225 cluster1-hadoop01-yx-daas-deepexi # 192.168.0.21 cluster1-hadoop02-yx-daas-deepexi # 192.168.0.232 cluster1-hadoop03-yx-daas-deepexi # 192.168.0.60 cluster2-hadoop01-yx-daas-deepexi # 192.168.0.97 cluster2-hadoop02-yx-daas-deepexi # 192.168.0.252 cluster2-hadoop03-yx-daas-deepexi
关闭防火墙
- 查看状态
systemctl status firewalld systemctl status iptables
- 停止
systemctl stop firewalld systemctl stop iptables
- 禁止开机启动
systemctl disable firewalld systemctl disable iptables
Ntp配置
- yum install ntp
vim /etc/ntp.conf # 允许内网其他机器同步时间，如果不添加该约束默认允许所有IP访问本机同步服务 # IP为本局域网内的子网IP restrict 192.168.100.0 mask 255.255.255.0 nomodify notrap #配置和上游标准时间同步 server 210.72.145.44 # 中国国家授时中心 server 0.cn.pool.ntp.org server 1.cn.pool.ntp.org server 2.cn.pool.ntp.org server 3.cn.pool.ntp.org #如果外部时间服务器不可用，NTP Server以本地时间作为时间服务器 server 127.127.1.0 # local clock fudge 127.127.1.0 stratum 10
- 配置slaves 服务器
#配置和上游标准时间同步 Server 192.168.100.45 #配置允许上游时间服务器主动修改本机的时间 restrict 192.168.100.45 nomodify notrap noquery timedatectl set-ntp yes
- 查看当前时间和NTP服务的同步状态
timedatectl set-ntp yes timedatectl ntpstat
SSH免密登录配置
- 设置config
vim /etc/ssh/sshd_config PermitRootLogin yes # 允许root用户登录 PasswordAuthentication no # 不使用密码登录 systemctl restart sshd.service
- 只在其中一台服务器生成密钥
ssh-keygen -t rsa
- 生成authorized_keys
cat /root/.ssh/id_rsa.pub >> authorized_keys
- 下载 私匙id_rsa 和 authorized_keys 文件,并上传到其他节点~/.ssh目录下
chmod 700 id_rsa mkdir /root/.ssh
- 验证
ssh cluster1-hadoop02-yx-daas-deepexi exit
JDK安装
- 必须安装在 /usr/java/
tar –zxvf jdk-8u181-linux-x64.tar.gz –C /usr/java/
- 配置环境变量
vim /etc/profile export JAVA_HOME=/usr/java/ jdk-1.8.0_181 export PATH=$PATH:$ JAVA_HOME/bin
jdbc 上传
上传jdbc到/usr/share/java/目录下，重命名为mysql-connector-java.jar
mkdir -p /usr/share/java/ cp mysql-connector-java-8.0.15-bin.jar /usr/share/java/mysql-connector-java.jar
禁用透明大页面压缩
- 查看
cat /sys/kernel/mm/transparent_hugepage/defrag
- 修改
echo never > /sys/kernel/mm/transparent_hugepage/defrag echo never > /sys/kernel/mm/transparent_hugepage/enabled
- 将命令写入开机启动
vim /etc/rc.local
Swapping交换区设置
- 查看
cat /proc/sys/vm/swappiness
- 修改
echo "vm.swappiness = 0" >> /etc/sysctl.conf
- 生效
sysctl -p
Tunned 系统调优服务禁用
- 查看tuned状态
systemctl status tuned
- 查看活动的服务
tuned-adm list
- 关闭tuned服务
tuned-adm off
- 确定没有活动的服务（No current active profile）
tuned-adm list
- 关闭并禁止开启启动
systemctl stop tuned systemctl disable tuned
MySQL安装（离线）
- 查看并卸载系统自带的Mariad
rpm –qa | grep mariadb rpm -e --nodeps mariadb-libs-5.5.52-1.el7.centos.x86_64
- 上传rpm包
mysql-community-client-8.0.15-1.el7.x86_64.rpm mysql-community-common-8.0.15-1.el7.x86_64.rpm mysql-community-libs-8.0.15-1.el7.x86_64.rpm mysql-community-server-8.0.15-1.el7.x86_64.rpm mysql-community-libs-compat-8.0.15-1.el7.x86_64.rpm
- 安装
rpm -ivh mysql-community-client-8.0.15-1.el7.x86_64.rpm
- 顺序
common -> libs -> client -> server -> libs-compat
- 如果提示缺少libaio依赖
yum install libaio
- 查看状态
systemctl status mysqld
- 备份数据配置文件并修改
cp /etc/my.cnf{,.bak} vim /etc/my.cnf [mysqld] datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock transaction-isolation = READ-COMMITTED key_buffer_size = 32M max_allowed_packet = 32M thread_stack = 256K thread_cache_size = 64 max_connections = 5000 binlog_expire_logs_seconds = 10 max_binlog_size = 200M log_bin=/var/lib/mysql/mysql_binary_log server_id=225 binlog_format = ROW read_buffer_size = 2M read_rnd_buffer_size = 16M sort_buffer_size = 8M join_buffer_size = 8M innodb_file_per_table = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 64M innodb_buffer_pool_size = 4G innodb_thread_concurrency = 8 innodb_flush_method = O_DIRECT innodb_log_file_size = 512M #lower_case_table_names = 1 [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid sql_mode=STRICT_ALL_TABLES - 启动和开机启动
sudo systemctl start mysqld sudo systemctl enable mysqld
- 查看初始密码
systemctl status mysqld -l
- 登录
mysql –uroot –p 输入密码
- 修改密码
alter user 'root'@'localhost' identified by '123456';
- 修改权限
use mysql; update user set host = "%" where user = 'root'; flush privileges; quit; # 如果提示密码简单 alter user 'root'@'localhost' identified by 'Abcd123456.'; set global validate_password.policy=0; set global validate_password.length=4; alter user 'root'@'localhost' identified by '123456';
- 修改连接限制
vim /usr/lib/systemd/system/mysqld.service LimitNOFILE = 65535 sysctl –p
- 登录MySQL建库
create database scm default character set utf8 default collate utf8_general_ci; create database amon default character set utf8 default collate utf8_general_ci; create database rman default character set utf8 default collate utf8_general_ci; create database hue default character set utf8 default collate utf8_general_ci; create database metastore default character set utf8 default collate utf8_general_ci; create database sentry default character set utf8 default collate utf8_general_ci; create database nav default character set utf8 default collate utf8_general_ci; create database navms default character set utf8 default collate utf8_general_ci; create database oozie default character set utf8 default collate utf8_general_ci;
- 创建用户
create user 'scm'@'%' identified by '123456'; create user 'amon'@'%' identified by '123456'; create user 'rman'@'%' identified by '123456'; create user 'hue'@'%' identified by '123456'; create user 'hive'@'%' identified by '123456'; create user 'sentry'@'%' identified by '123456'; create user 'nav'@'%' identified by '123456'; create user 'navms'@'%' identified by '123456'; create user 'oozie'@'%' identified by '123456';
- 修改密码验证方式
alter user 'scm'@'%' identified with mysql_native_password by '123456'; alter user 'amon'@'%' identified with mysql_native_password by '123456'; alter user 'rman'@'%' identified with mysql_native_password by '123456'; alter user 'hue'@'%' identified with mysql_native_password by '123456'; alter user 'hive'@'%' identified with mysql_native_password by '123456'; alter user 'sentry'@'%' identified with mysql_native_password by '123456'; alter user 'nav'@'%' identified with mysql_native_password by '123456'; alter user 'navms'@'%' identified with mysql_native_password by '123456'; alter user 'oozie'@'%' identified with mysql_native_password by '123456';
- 授权
grant all on scm.* to 'scm'@'%'; grant all on amon.* to 'amon'@'%'; grant all on rman.* to 'rman'@'%'; grant all on hue.* to 'hue'@'%'; grant all on metastore.* to 'hive'@'%'; grant all on sentry.* to 'sentry'@'%'; grant all on nav.* to 'nav'@'%'; grant all on navms.* to 'navms'@'%'; grant all on oozie.* to 'oozie'@'%'; flush privileges;
- 验证
show grants for 'scm'@'%'; show grants for 'amon'@'%'; show grants for 'rman'@'%'; show grants for 'hue'@'%'; show grants for 'hive'@'%'; show grants for 'sentry'@'%'; show grants for 'nav'@'%'; show grants for 'navms'@'%'; show grants for 'oozie'@'%';
Cloudera Manager（6.x安装方式相同）
- 离线下载CM安装包上传到服务器，主节点上传server和daemons，从节点上传agent和daemons
cloudera-manager-agent-6.2.1-1426065.el7.x86_64.rpm cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm cloudera-manager-server-6.2.1-1426065.el7.x86_64.rpm
- 安装CM
yum install bind-utils psmisc cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs /lib/lsb/init-functions httpd mod_ssl openssl-devel python-psycopg2 MySQL-python libxslt rpm -ivh cloudera-manager-daemons-6.0.1-610811.el7.x86_64.rpm rpm -ivh cloudera-manager-server-6.0.1-610811.el7.x86_64.rpm
- 修改服务主机（所有agent）
vim /etc/cloudera-scm-agent/config.ini server_host=master # server主机host server_port=7182
- 设置CM数据库
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm # 输入scm密码
- 启动server
sudo systemctl start cloudera-scm-server
- 查看启动情况
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
- 查看server状态
sudo systemctl status cloudera-scm-server 显示以下信息说明启动完成 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
- 登录web
web：http://master:7180 账号密码：admin
CDH集群部署
- 单集群部署，上传CDH离线安装包
CDH-6.0.1-1.cdh6.0.1.p0.590678-el7.parcel CDH-6.0.1-1.cdh6.0.1.p0.590678-el7.parcel.sha256 manifest.json
- 上传到 /opt/cloudera/parcel-repo/，没有则手动创建，修改.sha256文件后缀
mv CDH-6.0.1-1.cdh6.0.1.p0.590678-el7.parcel.sha256 CDH-6.0.1-1.cdh6.0.1.p0.590678-el7.parcel.sha
- 修改用户和组
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/*
- 登录界面
图1 登录界面1
图2 登录界面2
图3 登录界面3
- 选择版本，免费版即可
图4 选择版本1
图5 选择版本2
- 单击继续，可输入和搜索主机，也可通过“当前管理的主机”看到已连接到server的agent，直接选择即可
图6 当前管理的主机1
图7 当前管理的主机2
图8 当前管理的主机3
图9 当前管理的主机4
图10 当前管理的主机5
图11 当前管理的主机6
- 集群安装
图12 集群安装1
图13 集群安装2
图14 集群安装3
图15 集群安装4
图16 集群安装5
图17 集群安装6
图18 集群安装7
图19 集群安装8
- 安装Httpd（集群中一台服务器上即可）
yum install –y httpd
- 创建cloudera目录
mkdir -p /var/www/html/cloudera
- 复制安装包到 /var/www/html/cloudera
allkeys.asc cloudera-manager-agent-6.2.1-1426065.el7.x86_64.rpm cloudera-manager-daemons-6.2.1-1426065.el7.x86_64.rpm cloudera-manager-server-6.2.1-1426065.el7.x86_64.rpm cloudera-manager-server-db-2-6.2.1-1426065.el7.x86_64.rpm enterprise-debuginfo-6.2.1-1426065.el7.x86_64.rpm RPM-GPG-KEY-cloudera
- 安装createrepo服务，用于创建自定义仓库
yum install -y yum-utils createrepo
- 生成rpm元数据
cd /var/www/html/cloudera createrepo .
- 浏览器验证
- 制作Cloudera Manager的repo源
vim /etc/yum.repos.d/cm6.1.1_repo [cm6.2.1_repo] name = cm6.2.1_repo baseurl=http://master/cloudera enable = true gpgcheck = false
- 重启CM
systemctl start cloudera-scm-server
- 登录web
账号密码：admin
图20 登录web
- 选择服务器步骤与离线安装相同，从Paecel选择开始不同
图21 选择1
- 选择自定义存储库，输入上面配置的cloudera的http地址
图22 选择2
- Parcel选择，单击“更多选项”,单击“-”删除其它所有地址，输入http://master/cloudera，单击“保存更改”
图23 选择3
图24 选择4
- 单击继续，选择jdk，上面已经安装jdk，取消勾选即可
图25 选择5
- 单击继续，配置ssh账号密码
图26 配置ssh账号密码
- 单击“继续”，进入下一步，安装cm agent相关到各个节点
图27 下一步1
图28 下一步2
- Agent安装完以后，大数据组件安装与单集群离线安装相同
父主题： 实施步骤
