点击参与 GreatSQL 8.0.32-26 文档捉虫~~~

GreatSQL社区

搜索

[讨论中] 使用source命令导入慢问题

301 3 2025-2-28 16:07
GreatSQL版本：8.0.32-26

导入文件大小：1.7G
导入时间：15:26 ~ 16:05
耗时：

配置：
  1. [client]
  2. socket    = /data/soft/GreatSQL/mysql.sock
  3. default-character-set = utf8mb4

  5. [mysql]
  6. loose-skip-binary-as-hex
  7. prompt = "(\\D)[\\u@GreatSQL][\\d]>"
  8. no-auto-rehash
  9. default-character-set = utf8mb4

  11. [mysqld]
  12. user    = mysql
  13. #port    = 3306
  14. #server_id = 3306
  15. basedir = /usr/
  16. #datadir    = /data/soft/GreatSQL
  17. #socket    = /data/soft/GreatSQL/mysql.sock
  18. pid-file = mysql.pid
  19. character-set-server = utf8mb4
  20. collation-server = utf8mb4_0900_ai_ci
  21. init_connect = "SET NAMES utf8mb4 COLLATE utf8mb4_0900_ai_ci"
  22. skip-character-set-client-handshake
  23. skip_name_resolve = ON
  24. default_time_zone = "+8:00"
  25. bind_address = "0.0.0.0"
  26. secure_file_priv = /data/soft/GreatSQL

  28. # Performance
  29. lock_wait_timeout = 3600
  30. open_files_limit    = 65535
  31. back_log = 1024
  32. max_connections = 1024
  33. max_connect_errors = 1000000
  34. table_open_cache = 2048
  35. table_definition_cache = 2048
  36. sort_buffer_size = 4M
  37. join_buffer_size = 4M
  38. read_buffer_size = 8M
  39. read_rnd_buffer_size = 4M
  40. bulk_insert_buffer_size = 64M
  41. thread_cache_size = 768
  42. interactive_timeout = 600
  43. wait_timeout = 600
  44. tmp_table_size = 96M
  45. max_heap_table_size = 96M
  46. max_allowed_packet = 64M
  47. net_buffer_shrink_interval = 180
  48. sql_generate_invisible_primary_key = ON
  49. loose-lock_ddl_polling_mode = ON
  50. loose-lock_ddl_polling_runtime = 200

  52. # Logs
  53. log_timestamps = SYSTEM
  54. log_error = mysqld.log
  55. log_error_verbosity = 3

  57. # 慢查询日志
  58. slow_query_log = ON
  59. log_slow_extra = ON
  60. slow_query_log_file = slow.log
  61. long_query_time = 0.01
  62. log_queries_not_using_indexes = ON
  63. log_throttle_queries_not_using_indexes = 60
  64. min_examined_row_limit = 100
  65. log_slow_admin_statements = ON
  66. log_slow_replica_statements = ON
  67. log_slow_verbosity = FULL

  69. # binlog
  70. log_bin = binlog
  71. binlog_format = ROW
  72. sync_binlog = 1
  73. binlog_cache_size = 4M
  74. max_binlog_cache_size = 6G
  75. max_binlog_size = 1G
  76. binlog_space_limit = 500G
  77. binlog_rows_query_log_events = ON
  78. binlog_expire_logs_seconds = 604800
  79. binlog_checksum = CRC32
  80. binlog_order_commits = OFF
  81. gtid_mode = ON
  82. enforce_gtid_consistency = ON

  84. # InnoDB
  85. innodb_buffer_pool_size = 8G
  86. innodb_buffer_pool_instances = 8
  87. innodb_data_file_path = ibdata1:12M:autoextend
  88. innodb_flush_log_at_trx_commit = 1
  89. innodb_log_buffer_size = 32M
  90. innodb_redo_log_capacity = 4G
  91. innodb_doublewrite_files = 2
  92. innodb_max_undo_log_size = 4G
  93. innodb_io_capacity = 4000
  94. innodb_io_capacity_max = 8000
  95. innodb_open_files = 65534
  96. innodb_flush_method = O_DIRECT
  97. innodb_lru_scan_depth = 4000
  98. innodb_lock_wait_timeout = 10
  99. innodb_rollback_on_timeout = ON
  100. innodb_print_all_deadlocks = ON
  101. innodb_online_alter_log_max_size = 4G
  102. innodb_print_ddl_logs = ON
  103. innodb_status_file = ON
  104. innodb_status_output = OFF
  105. innodb_status_output_locks = ON
  106. innodb_sort_buffer_size = 64M
  107. innodb_adaptive_hash_index = OFF
  108. innodb_numa_interleave = OFF
  109. innodb_spin_wait_delay = 20
  110. innodb_print_lock_wait_timeout_info = ON
  111. innodb_change_buffering = none
  112. kill_idle_transaction = 300
  113. innodb_data_file_async_purge = ON

  115. #innodb monitor settings
  116. #innodb_monitor_enable = "module_innodb,module_server,module_dml,module_ddl,module_trx,module_os,module_purge,module_log,module_lock,module_buffer,module_index,module_ibuf_system,module_buffer_page,module_adaptive_hash"

  118. #pfs settings
  119. performance_schema = 1
  120. #performance_schema_instrument = '%memory%=on'
  121. performance_schema_instrument = '%lock%=on'

  123. [mysqld@3306]
  124. datadir=/data/soft/GreatSQL/3306
  125. socket=/data/soft/GreatSQL/3306/mysql.sock
  126. port=3306
  127. server_id=103306
  128. log-error=/data/soft/GreatSQL/3306/mysqld.log

  130. [mysqld@3307]
  131. datadir=/data/soft/GreatSQL/3307
  132. socket=/data/soft/GreatSQL/3307/mysql.sock
  133. port=3307
  134. server_id=103307
  135. log-error=/data/soft/GreatSQL/3307/mysqld.log
复制代码
服务器IO：
2.png


MySQL版本：5.7.44
导入文件大小：1.7G
导入时间：15:29 ~ 15:35
耗时：6分钟

配置：
  1. [client]
  2. #password       = your_password
  3. #port            = 3306
  4. #socket          = /data/soft/mysql3306/mysql.sock
  5. #default-character-set = utf8mb4

  7. [mysqld@3306]
  8. port            = 3306
  9. socket          = /data/soft/mysql/3306/mysql.sock
  10. datadir         = /data/soft/mysql/3306
  11. #skip-external-locking
  12. #skip_name_resolve = 1
  13. key_buffer_size = 256M
  14. max_allowed_packet = 512M
  15. table_open_cache = 512
  16. sort_buffer_size = 16M
  17. read_buffer_size = 16M
  18. read_rnd_buffer_size = 64M
  19. myisam_sort_buffer_size = 64M
  20. thread_cache_size = 4
  21. query_cache_size= 16M
  22. query_cache_type=1
  23. character_set_server = utf8mb4
  24. collation_server=utf8mb4_general_ci
  25. max_connections = 500
  26. innodb_read_io_threads = 4
  27. innodb_write_io_threads = 4
  28. wait_timeout=2880000
  29. interactive_timeout = 2880000
  30. lower_case_table_names=1

  32. event_scheduler=ON

  34. log-bin=mysql-bin
  35. binlog_format=mixed

  37. log_error = /data/soft/mysql/3306/mysql.log
  38. slow_query_log = 1
  39. long_query_time = 1
  40. slow_query_log_file = /data/soft/mysql/3306/slow.log
  41. log_slow_admin_statements = 1

  43. server-id       = 100
  44. expire_logs_days = 30

  46. innodb_buffer_pool_size = 8G
  47. innodb_buffer_pool_instances = 1
  48. innodb_buffer_pool_load_at_startup = 1
  49. innodb_flush_neighbors = 1
  50. innodb_print_all_deadlocks = 1

  52. innodb_sort_buffer_size = 16M
  53. innodb_flush_log_at_trx_commit=2

  55. [mysqld@3307]
  56. port            = 3307
  57. socket          = /data/soft/mysql/3307/mysql.sock
  58. datadir         = /data/soft/mysql/3307
  59. #skip-external-locking
  60. #skip_name_resolve = 1
  61. key_buffer_size = 256M
  62. max_allowed_packet = 512M
  63. table_open_cache = 512
  64. sort_buffer_size = 16M
  65. read_buffer_size = 16M
  66. read_rnd_buffer_size = 64M
  67. myisam_sort_buffer_size = 64M
  68. thread_cache_size = 4
  69. query_cache_size= 16M
  70. query_cache_type=1
  71. character_set_server = utf8mb4
  72. collation_server=utf8mb4_general_ci
  73. max_connections = 500
  74. innodb_read_io_threads = 4
  75. innodb_write_io_threads = 4
  76. wait_timeout=2880000
  77. interactive_timeout = 2880000
  78. lower_case_table_names=1

  80. event_scheduler=ON

  82. log-bin=mysql-bin
  83. binlog_format=mixed

  85. log_error = /data/soft/mysql/3307/mysql.log
  86. slow_query_log = 1
  87. long_query_time = 3
  88. slow_query_log_file = /data/soft/mysql/3307/slow.log
  89. log_slow_admin_statements = 1

  91. server-id       = 101
  92. expire_logs_days = 30

  94. innodb_buffer_pool_size = 8G
  95. innodb_buffer_pool_instances = 1
  96. innodb_buffer_pool_load_at_startup = 1
  97. innodb_flush_neighbors = 1
  98. innodb_print_all_deadlocks = 1

  100. innodb_sort_buffer_size = 16M
  101. innodb_flush_log_at_trx_commit=2

  103. [mysqldump]
  104. quick
  105. max_allowed_packet = 512M
  106. user=bakuser
  107. password=Ba0k1@db_pass

  109. [mysql]
  110. no-auto-rehash

  112. [myisamchk]
  113. key_buffer_size = 128M
  114. sort_buffer_size = 128M
  115. read_buffer = 2M
  116. write_buffer = 2M

  118. [mysqlhotcopy]
  119. interactive-timeout
复制代码
服务器IO：

GreatSQL
全部回复(3)
只看楼主
yejr 2025-2-28 22:01:17
1. 可以用下面的方法跑dd简单测试I/O性能（你上面的方法要加上oflag参数）
time dd oflag=direct if=/dev/zero of=./zero bs=1M count=20480

2. source方式导入的是串行，不是并行

3. 导入期间，可以关闭双1以提速。不知道什么是双1的话，参考：https://greatsql.cn/docs/8.0.32- ... E6%88%96-%E5%8F%8C0
回复 采纳

举报

earl86 2025-3-3 09:35:38
source 慢正常,可以试试 mydumper 并行导入导出
回复 采纳

举报

caihe.li 2025-3-3 11:23:34
如果允许没有特殊字符的话，可以考虑使用LOAD DATA的方式导入导出，greatsql有并行LOAD DATA，可以参考这个链接:https://greatsql.cn/docs/8.0.32- ... -parallel-load.html
回复 采纳

举报

alexyj

5

主题

0

博客

22

贡献

新手上路

Rank: 1

积分
40
发消息
互动与交流
社区活动
论坛问答
加入社区
贡献者指南
学习与应用
文章博客
文档资料
视频资料
常见问题
发现社区
社区准则
建议反馈
社区之星

合作电话：010-64087828

社区邮箱：greatsql@greatdb.com

社区公众号
社区小助手
QQ群
友情链接
万里数据库开源中国墨天轮开放原子开源基金会OpenEuler
©2022 GreatSQL All Rights Reserved. ( 京ICP备06057874号 )
GMT+8, 2025-4-3 23:56 , Processed in 0.024352 second(s), 18 queries , Redis On.
快速回复 返回顶部 返回列表