[mysqld] datadir=/var/lib/mysql socket=/var/lib/mysql/mysql.sock transaction-isolation = READ-COMMITTED # Disabling symbolic-links is recommended to prevent assorted security risks; # to do so, uncomment this line: symbolic-links = 0 key_buffer_size = 32M max_allowed_packet = 32M thread_stack = 256K thread_cache_size = 64 query_cache_limit = 8M query_cache_size = 64M query_cache_type = 1 max_connections = 550 #expire_logs_days = 10 #max_binlog_size = 100M #log_bin should be on a disk with enough free space. #Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your #system and chown the specified folder to the mysql user. log_bin=/var/lib/mysql/mysql_binary_log #In later versions of MySQL, if you enable the binary log and do not set #a server_id, MySQL will not start. The server_id must be unique within #the replicating group. server_id=1 skip-ssl binlog_format = mixed read_buffer_size = 2M read_rnd_buffer_size = 16M sort_buffer_size = 8M join_buffer_size = 8M # InnoDB settings innodb_file_per_table = 1 innodb_flush_log_at_trx_commit = 2 innodb_log_buffer_size = 64M innodb_buffer_pool_size = 4G innodb_thread_concurrency = 8 innodb_flush_method = O_DIRECT innodb_log_file_size = 512M [mysqld_safe] log-error=/var/log/mysqld.log pid-file=/var/run/mysqld/mysqld.pid sql_mode=STRICT_ALL_TABLES
系统优化
关闭禁用透明重复页面
临时生效
1 2
echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag
永久生效
1 2
echo'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.d/rc.local echo'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.d/rc.local
修改文件句柄数
修改本次运行期间的句柄数
1
ulimit -n 65535
修改操作系统重启时默认的句柄数
1 2 3 4 5 6 7 8
vim /etc/security/limits.conf
* soft nofile 65536 * hard nofile 131072 # 打开进程数 * soft nproc 65535 * hard nproc 65535
[root@master data]# yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server Loaded plugins: fastestmirror, langpacks Loading mirror speeds from cached hostfile * base: mirrors.tuna.tsinghua.edu.cn * extras: mirrors.tuna.tsinghua.edu.cn * updates: mirrors.bfsu.edu.cn Resolving Dependencies --> Running transaction check ---> Package cloudera-manager-agent.x86_64 0:6.3.1-1466458.el7 will be installed --> Processing Dependency: python-psycopg2 for package: cloudera-manager-agent-6.3.1-1466458.el7.x86_64 --> Processing Dependency: mod_ssl for package: cloudera-manager-agent-6.3.1-1466458.el7.x86_64 --> Processing Dependency: MySQL-python for package: cloudera-manager-agent-6.3.1-1466458.el7.x86_64 --> Processing Dependency: /lib/lsb/init-functions for package: cloudera-manager-agent-6.3.1-1466458.el7.x86_64 --> Processing Dependency: libpq.so.5()(64bit) for package: cloudera-manager-agent-6.3.1-1466458.el7.x86_64 ---> Package cloudera-manager-daemons.x86_64 0:6.3.1-1466458.el7 will be installed ---> Package cloudera-manager-server.x86_64 0:6.3.1-1466458.el7 will be installed --> Running transaction check ---> Package MySQL-python.x86_64 0:1.2.5-1.el7 will be installed ---> Package mod_ssl.x86_64 1:2.4.6-97.el7.centos will be installed ---> Package postgresql-libs.x86_64 0:9.2.24-7.el7_9 will be installed ---> Package python-psycopg2.x86_64 0:2.5.1-4.el7 will be installed ---> Package redhat-lsb-core.x86_64 0:4.1-27.el7.centos.1 will be installed --> Processing Dependency: redhat-lsb-submod-security(x86-64) = 4.1-27.el7.centos.1 for package: redhat-lsb-core-4.1-27.el7.centos.1.x86_64 --> Processing Dependency: spax for package: redhat-lsb-core-4.1-27.el7.centos.1.x86_64 --> Running transaction check ---> Package redhat-lsb-submod-security.x86_64 0:4.1-27.el7.centos.1 will be installed ---> Package spax.x86_64 0:1.5.2-13.el7 will be installed --> Finished Dependency Resolution
Dependencies Resolved
============================================================================================================================================================================================================ Package Arch Version Repository Size ============================================================================================================================================================================================================ Installing: cloudera-manager-agent x86_64 6.3.1-1466458.el7 cloudera-manager 10 M cloudera-manager-daemons x86_64 6.3.1-1466458.el7 cloudera-manager 1.1 G cloudera-manager-server x86_64 6.3.1-1466458.el7 cloudera-manager 11 k Installing for dependencies: MySQL-python x86_64 1.2.5-1.el7 base 90 k mod_ssl x86_64 1:2.4.6-97.el7.centos updates 114 k postgresql-libs x86_64 9.2.24-7.el7_9 updates 235 k python-psycopg2 x86_64 2.5.1-4.el7 base 132 k redhat-lsb-core x86_64 4.1-27.el7.centos.1 base 38 k redhat-lsb-submod-security x86_64 4.1-27.el7.centos.1 base 15 k spax x86_64 1.5.2-13.el7 base 260 k
mysql数据库与CM Server是同一台主机 执行命令：/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
mysql数据库与CM Server不在同一台主机上 执行命令：/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h <mysql-host-ip> --scm-host <cm-server-ip> scm scm
1 2 3 4 5 6 7 8 9
[root@master parcel-repo]# /opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h 10.10.100.197 --scm-host 10.10.100.197 scm scm Enter SCM password: JAVA_HOME=/usr/local/java/jdk1.8.0_212 Verifying that we can write to /etc/cloudera-scm-server Creating SCM configuration file in /etc/cloudera-scm-server Executing: /usr/local/java/jdk1.8.0_212/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/java/postgresql-connector-java.jar:/opt/cloudera/cm/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db. Wed Jul 21 13:46:01 CST 2021 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. [ main] DbCommandExecutor INFO Successfully connected to database. All done, your SCM database is configured correctly!
2021-07-21 13:48:55,430 INFO WebServerImpl:org.eclipse.jetty.server.Server: jetty-9.4.14.v20181114; built: 2018-11-14T21:20:31.478Z; git: c4550056e785fb5665914545889f21dc136ad9e6; jvm 1.8.0_222-b10 2021-07-21 13:48:55,446 INFO WebServerImpl:org.eclipse.jetty.server.AbstractConnector: Started ServerConnector@64610985{HTTP/1.1,[http/1.1]}{0.0.0.0:7180} 2021-07-21 13:48:55,447 INFO WebServerImpl:org.eclipse.jetty.server.Server: Started @60970ms 2021-07-21 13:48:55,447 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server. 2021-07-21 13:48:55,642 ERROR ParcelUpdateService:com.cloudera.parcel.components.ParcelDownloaderImpl: Unable to retrieve remote parcel repository manifest java.util.concurrent.ExecutionException: java.net.ConnectException: connection timed out: archive.cloudera.com/151.101.72.167:443 at com.ning.http.client.providers.netty.future.NettyResponseFuture.abort(NettyResponseFuture.java:231) at com.ning.http.client.providers.netty.request.NettyConnectListener.onFutureFailure(NettyConnectListener.java:137) at com.ning.http.client.providers.netty.request.NettyConnectListener.operationComplete(NettyConnectListener.java:145) at org.jboss.netty.channel.DefaultChannelFuture.notifyListener(DefaultChannelFuture.java:409) at org.jboss.netty.channel.DefaultChannelFuture.notifyListeners(DefaultChannelFuture.java:400) at org.jboss.netty.channel.DefaultChannelFuture.setFailure(DefaultChannelFuture.java:362) at org.jboss.netty.channel.socket.nio.NioClientBoss.processConnectTimeout(NioClientBoss.java:142) at org.jboss.netty.channel.socket.nio.NioClientBoss.process(NioClientBoss.java:83) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.net.ConnectException: connection timed out: archive.cloudera.com/151.101.72.167:443 at com.ning.http.client.providers.netty.request.NettyConnectListener.onFutureFailure(NettyConnectListener.java:133) ... 13 more Caused by: org.jboss.netty.channel.ConnectTimeoutException: connection timed out: archive.cloudera.com/151.101.72.167:443 at org.jboss.netty.channel.socket.nio.NioClientBoss.processConnectTimeout(NioClientBoss.java:139) ... 8 more
启动cloudera-scm-agent的所有节点
1 2 3
systemctl start cloudera-scm-agent # 或者 service cloudera-scm-agent start
已启用透明大页面压缩，可能会导致重大性能问题。请运行“echo never > /sys/kernel/mm/transparent_hugepage/defrag”和“echo never > /sys/kernel/mm/transparent_hugepage/enabled”以禁用此设置，然后将同一命令添加到 /etc/rc.local 等初始化脚本中，以便在系统重启时予以设置。以下主机将受到影响:
wechat
alipay
