GreatSQL社区

搜索

[已解决] 在使用sysbench 导入数据的时候成员离线

709 7 2024-4-18 22:47
日志已经做附件上传

其中 mgr-node2 执行过reset master 然后重新加入集群,还是报错


版本 greatesql-8.0.32-25

mgrlog.zip

188.67 KB, 下载次数: 3, 下载积分: 金币 -1

全部回复(7)
KAiTO 2024-4-19 10:22:52
看了你的日志,好像MGR都没搭建成功吧
可以跟着手册手工搭建哈:https://greatsql.cn/docs/8032-25 ... install-deploy.html
yejr 2024-4-19 11:03:42
请补充提供几个信息
1. 几个节点所在机器配置
2. 几个节点my.cnf配置
3. sysbench导入数据期间各节点系统负载数据

较大的可能性是:你的机器配置较一般,导入数据量太大或并发太高,导致各节点负载太高,MGR通信异常,多次超时或异常后,就会退出
mabai 2024-4-19 17:07:18
KAiTO 发表于 2024-4-19 10:22
看了你的日志,好像MGR都没搭建成功吧
可以跟着手册手工搭建哈:https://greatsql.cn/docs/8032-25/user-ma ...

MGR是搭建成功的,中间vip也启动了
mabai 2024-4-19 17:09:26
本帖最后由 mabai 于 2024-4-19 17:31 编辑
yejr 发表于 2024-4-19 11:03
请补充提供几个信息
1. 几个节点所在机器配置
2. 几个节点my.cnf配置

机器配置96vcpu   251G  21TSSD  
[mysql]
loose-skip-binary-as-hex
prompt = "(\\D)[\\u@mgr-node3][\\d]>"
no-auto-rehash

[mysqld]
user    = mysql
port    = 3306
server_id = 330603
basedir = /usr/local/mysql
datadir    = /data/mysql/data
socket    = /data/mysql/run/mysql.sock
pid-file = /data/mysql/run/mysql.pid
secure_file_priv = /data/mysql/tmp
character-set-server = UTF8MB4
skip_name_resolve = 1
skip_external_locking = 1
lower_case_table_names = 1
default_time_zone = "+8:00"
#启用admin_port,连接数爆满等紧急情况下给管理员留个后门
admin_address = '127.0.0.1'
admin_port = 33063
local_infile = 1
default_authentication_plugin = mysql_native_password
sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'



#performance setttings
lock_wait_timeout = 1800
open_files_limit = 65535
back_log = 1024
max_connections = 6000
max_connect_errors = 1000000
table_open_cache = 2048
table_definition_cache = 2048
thread_stack = 512K
sort_buffer_size = 4M
join_buffer_size = 4M
read_buffer_size = 8M
read_rnd_buffer_size = 16M
bulk_insert_buffer_size = 16M
thread_cache_size = 768
interactive_timeout = 600
wait_timeout = 60
tmp_table_size = 96M
max_heap_table_size = 96M
max_allowed_packet = 64M
net_buffer_shrink_interval = 180
#GIPK
loose-sql_generate_invisible_primary_key = ON

#log settings
log_timestamps = SYSTEM
log_error = /data/mysql/log/error.log
log_error_verbosity = 3
slow_query_log = 1
log_slow_extra = 1
slow_query_log_file = /data/mysql/log/slow.log
long_query_time = 2
log_queries_not_using_indexes = 1
log_throttle_queries_not_using_indexes = 60
min_examined_row_limit = 100
log_slow_admin_statements = 1
log_slow_slave_statements = 1
log_slow_verbosity = FULL
log_bin = /data/mysql/log/binlog/mysql-bin
log_bin_index = /data/mysql/log/binlog/mysql-bin-index
binlog_format = ROW
sync_binlog = 1
binlog_cache_size = 4M
max_binlog_cache_size = 6G
max_binlog_size = 1G
binlog_rows_query_log_events = 1
binlog_expire_logs_seconds = 15552000
binlog_checksum = CRC32
gtid_mode = ON
enforce_gtid_consistency = TRUE

#myisam settings
key_buffer_size = 32M
myisam_sort_buffer_size = 128M

#replication settings
relay_log_recovery = 1
slave_parallel_type = LOGICAL_CLOCK
#并行复制线程数可以设置为逻辑CPU数量的2倍
slave_parallel_workers = 96
binlog_transaction_dependency_tracking = WRITESET
slave_preserve_commit_order = 1
slave_checkpoint_period = 2

#启用InnoDB并行查询优化功能
loose-force_parallel_execute = ON
#设置每个SQL语句的并行查询最大并发度
loose-parallel_default_dop = 8
#设置系统中总的并行查询线程数,可以和最大逻辑CPU数量一样
loose-parallel_max_threads = 96
#并行执行时leader线程和worker线程使用的总内存大小上限,可以设置物理内存的5-10%左右
loose-parallel_memory_limit = 12G

#parallel load data
loose-gdb_parallel_load_chunk_size = 4M

#rapid engine
loose-rapid_memory_limit = 12G
loose-rapid_worker_threads = 32
loose-rapid_hash_table_memory_limit = 30
loose-secondary_engine_parallel_load_workers = 16

#mgr settings
loose-plugin_load_add = 'mysql_clone.so'
loose-plugin_load_add = 'group_replication.so'
loose-group_replication_group_name="c2e8b9d4-91e4-47a0-b51e-c1f4b4e5eacf"
loose-group_replication_local_address = "172.16.179.22:33061"
loose-group_replication_group_seeds = '172.16.179.20:33061,172.16.179.21:33061,172.16.179.22:33061'
loose-group_replication_start_on_boot = ON
loose-group_replication_bootstrap_group = OFF
loose-group_replication_exit_state_action = READ_ONLY
loose-group_replication_flow_control_mode = "DISABLED"
loose-group_replication_single_primary_mode = ON
loose-group_replication_enforce_update_everywhere_checks=0
loose-group_replication_majority_after_mode = ON
loose-group_replication_communication_max_message_size = 10M
loose-group_replication_arbitrator = 0
loose-group_replication_single_primary_fast_mode = 1
loose-group_replication_request_time_threshold = 100
loose-group_replication_primary_election_mode = GTID_FIRST
loose-group_replication_unreachable_majority_timeout = 0
loose-group_replication_member_expel_timeout = 5
loose-group_replication_autorejoin_tries = 288
loose-group_replication_recovery_get_public_key = ON

#mgr vip
loose-plugin_load_add = 'greatdb_ha.so'
loose-greatdb_ha_enable_mgr_vip = 1
loose-greatdb_ha_mgr_vip_nic = 'bond1'
loose-greatdb_ha_mgr_vip_ip = '172.16.179.120'
loose-greatdb_ha_mgr_vip_mask = '255.255.255.0'
loose-greatdb_ha_port = 33062
loose-greatdb_ha_mgr_read_vip_ips = "172.16.179.220"
loose-greatdb_ha_mgr_read_vip_floating_type = "TO_ANOTHER_SECONDARY"
loose-greatdb_ha_send_arp_packge_times = 5
report_host = 172.16.179.22
report_port = 3306

#MGR切主后是否断开旧Priamry节点上的所有应用连接
loose-greatdb_ha_mgr_exit_primary_kill_connection_mode = 0

#innodb settings
innodb_buffer_pool_size = 188G
innodb_buffer_pool_instances = 8
innodb_data_file_path = ibdata1:12M:autoextend
innodb_flush_log_at_trx_commit = 1
innodb_log_buffer_size = 32M
innodb_log_file_size = 2G
innodb_log_files_in_group = 3
innodb_redo_log_capacity = 6G
innodb_doublewrite_files = 2
innodb_max_undo_log_size = 4G
# 根据您的服务器IOPS能力适当调整
# 一般配普通SSD盘的话,可以调整到 10000 - 20000
# 配置高端PCIe SSD卡的话,则可以调整的更高,比如 50000 - 80000
innodb_io_capacity = 50000
innodb_io_capacity_max = 80000
innodb_open_files = 65534
#提醒:当需要用CLONE加密特性时,不要选用O_DIRECT模式,否则会比较慢
innodb_flush_method = O_DIRECT
innodb_lru_scan_depth = 4000
innodb_lock_wait_timeout = 10
innodb_rollback_on_timeout = 1
innodb_print_all_deadlocks = 1
innodb_online_alter_log_max_size = 4G
innodb_print_ddl_logs = 1
innodb_status_file = 1
innodb_status_output = 0
innodb_status_output_locks = 1
innodb_sort_buffer_size = 64M
innodb_adaptive_hash_index = 0
#开启NUMA支持
innodb_numa_interleave = ON
innodb_print_lock_wait_timeout_info = 1
#自动杀掉超过2分钟不活跃事务,避免行锁被长时间持有
kill_idle_transaction = 120
#异步清理大表
#innodb_data_file_async_purge = ON

innodb_status_output = 0
innodb_status_output_locks = 1


#innodb monitor settings
innodb_monitor_enable = "module_innodb,module_server,module_dml,module_ddl,module_trx,module_os,module_purge,module_log,module_lock,module_buffer,module_index,module_ibuf_system,module_buffer_page,module_adaptive_hash"

#pfs settings
performance_schema = 1
performance_schema_instrument = '%memory%=on'
performance_schema_instrument = '%lock%=on'
performance-schema-instrument = 'memory/%=COUNTED'
performance_schema_consumer_events_statements_history_long = ON
performance_schema_consumer_events_transactions_current = ON
performance_schema_consumer_events_transactions_history = ON



3、sysbench导入时候  cpu没有太大的波动,磁盘 io在40%左右 ,内存被打满
mabai 2024-4-19 17:29:28
本帖最后由 mabai 于 2024-4-19 17:33 编辑
yejr 发表于 2024-4-19 11:03
请补充提供几个信息
1. 几个节点所在机器配置
2. 几个节点my.cnf配置

我发现一点 开始做压测导数据的时间 内存急剧降低  导致oom。原生mgr没有这样的问题

导入命令为:
sysbench oltp_read_write --db-driver=mysql --mysql-host=172.16.179.120 --mysql-port=3306 --mysql-db=lizi --mysql-user=username --mysql-password='password' --table_size=5000000 --tables=100 --threads=50 --report-interval=1 --rand-type=uniform --time=1800 prepare
yejr 2024-4-20 13:55:50
mabai 发表于 2024-4-19 17:29
我发现一点 开始做压测导数据的时间 内存急剧降低  导致oom。原生mgr没有这样的问题

导入命令为:

请问你用的是哪个版本
yejr 2024-4-20 13:58:12
mabai 发表于 2024-4-19 17:09
机器配置96vcpu   251G  21TSSD  
[mysql]
loose-skip-binary-as-hex

压测期间请运行

  1. vmstat -S m 1
复制代码

记录下系统负载数据发出来
mabai

15

主题

0

博客

91

贡献

版主

Rank: 7Rank: 7Rank: 7

金币
2878
贡献
91

合作电话:010-64087828

社区邮箱:greatsql@greatdb.com

社区公众号
社区小助手
QQ群
GMT+8, 2024-12-27 02:00 , Processed in 0.032567 second(s), 23 queries , Redis On.
快速回复 返回顶部 返回列表