GreatSQL发布Rapid引擎（Rapid Engine），TPC-H 100G不到80秒，牛~

yejr 发表于 2024-3-1 21:00
请详细描述你编译的过程

1. 
   
2. #!/bin/bash
   
3. MAJOR_VERSION=8
   
4. MINOR_VERSION=0
   
5. PATCH_VERSION=32
   
6. RELEASE=25
   
7. REVISION=db07cc5cb73
   
8. GLIBC=`ldd --version | head -n 1 | awk '{print $NF}'`
   
9. ARCH=`uname -p`
   
10. OS=`grep '^ID=' /etc/os-release | sed 's/.*"\(.*\)".*/\1/ig'`
    
11. #OS=Linux
    
12. PKG_NAME=GreatSQL-${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}-${RELEASE}-${OS}-glibc${GLIBC}-${ARCH}
    
13. # 修改安装目录
    
14. BASE_DIR=/usr/local/mysql
    
15. BOOST_VERSION=1_77_0
    
16. MAKELOG=/tmp/greatsql-automake.log
    
17. #使用原来的源代码目录
    
18. SOURCE_DIR=greatsql-8.0.32-24
    
19. CMAKE_EXE_LINKER_FLAGS=""
    
20. JOBS=`lscpu | grep '^CPU(s)'|awk '{print $NF}'`
    
21. if [ ${JOBS} -ge 16 ] ; then
    
22.   JOBS=`expr ${JOBS} - 4`
    
23. else
    
24.   JOBS=`expr ${JOBS} - 2`
    
25. fi
    
26. 
    
27. if [ ${ARCH} = "x86_64" ] ; then
    
28.   CMAKE_EXE_LINKER_FLAGS=" -ljemalloc "
    
29. fi
    
30. 
    
31. if [ ${ARCH} = "loongarch64" ] ; then
    
32.   cd /opt/${SOURCE_DIR}
    
33.   sed -i 's/\(.*defined.*mips.*\) \\/\1 defined(__loongarch__) || \\/ig' extra/icu/source/i18n/double-conversion-utils.h
    
34. fi
    
35. 
    
36. 
    
37. LIBLIST="libcrypto.so libssl.so libreadline.so libtinfo.so libsasl2.so libbrotlidec.so libbrotlicommon.so libgssapi_krb5.so libkrb5.so libkrb5support.so libk5crypto.so librtmp.so libgssapi.so libssl3.so libsmime3.so libnss3.so libnssutil3.so libplc4.so libnspr4.so libssl3.so libplds4.so libncurses.so libjemalloc.so"
    
38. DIRLIST="bin lib lib/private lib/plugin lib/mysqlrouter/plugin lib/mysqlrouter/private"
    
39. 
    
40. LIBPATH=""
    
41. 
    
42. function gather_libs {
    
43.     local elf_path=$1
    
44.     for lib in ${LIBLIST}; do
    
45.         for elf in $(find ${elf_path} -maxdepth 1 -exec file {} \; | grep 'ELF ' | cut -d':' -f1); do
    
46.             IFS=$'\n'
    
47.             for libfromelf in $(ldd ${elf} | grep ${lib} | awk '{print $3}'); do
    
48.                 lib_realpath="$(readlink -f ${libfromelf})"
    
49.                 lib_realpath_basename="$(basename $(readlink -f ${libfromelf}))"
    
50.                 lib_without_version_suffix=$(echo ${lib_realpath_basename} | awk -F"." 'BEGIN { OFS = "." }{ print $1, $2}')
    
51. 
    
52.                 # Some libraries may have dependencies on earlier openssl libraries, such as authentication_ldap_simple.so,
    
53.                 # thus we need to treat them specially here, other than stripping version suffix.
    
54.                 if [[ "${lib_realpath_basename}" =~ ^libcrypto.so.1.0.* ]] || [[ "${lib_realpath_basename}" =~ ^libssl.so.1.0.* ]];
    
55.                 then
    
56.                   lib_without_version_suffix=$(basename ${libfromelf})
    
57.                 fi
    
58. 
    
59. 
    
60.                 if [ ! -f "lib/private/${lib_realpath_basename}" ] && [ ! -L "lib/private/${lib_realpath_basename}" ]; then
    
61. 
    
62.                     echo "Copying lib ${lib_realpath_basename}"
    
63.                     cp ${lib_realpath} lib/private
    
64. 
    
65.                     echo "Symlinking lib from ${lib_realpath_basename} to ${lib_without_version_suffix}"
    
66.                     cd lib/
    
67.                     ln -s private/${lib_realpath_basename} ${lib_without_version_suffix}
    
68.                     cd -
    
69.                     if [ ${lib_realpath_basename} != ${lib_without_version_suffix} ]; then
    
70.                         cd lib/private
    
71.                         ln -s ${lib_realpath_basename} ${lib_without_version_suffix}
    
72.                         cd -
    
73.                     fi
    
74. 
    
75.                     patchelf --set-soname ${lib_without_version_suffix} lib/private/${lib_realpath_basename}
    
76. 
    
77.                     LIBPATH+=" $(echo ${libfromelf} | grep -v $(pwd))"
    
78.                 fi
    
79.             done
    
80.             unset IFS
    
81.         done
    
82.     done
    
83. }
    
84. 
    
85. function set_runpath {
    
86.     # Set proper runpath for bins but check before doing anything
    
87.     local elf_path=$1
    
88.     local r_path=$2
    
89.     for elf in $(find ${elf_path} -maxdepth 1 -exec file {} \; | grep 'ELF ' | cut -d':' -f1); do
    
90.         echo "Checking LD_RUNPATH for ${elf}"
    
91.         if [[ -z $(patchelf --print-rpath ${elf}) ]]; then
    
92.             echo "Changing RUNPATH for ${elf}"
    
93.             patchelf --set-rpath ${r_path} ${elf}
    
94.         fi
    
95.         if [[ ! -z "${override}" ]] && [[ "${override}" == "true" ]]; then
    
96.             echo "Overriding RUNPATH for ${elf}"
    
97.             patchelf --set-rpath ${r_path} ${elf}
    
98.         fi
    
99.     done
    
100. }
     
101. 
     
102. function replace_libs {
     
103.     local elf_path=$1
     
104.     for libpath_sorted in ${LIBPATH}; do
     
105.         for elf in $(find ${elf_path} -maxdepth 1 -exec file {} \; | grep 'ELF ' | cut -d':' -f1); do
     
106.             LDD=$(ldd ${elf} | grep ${libpath_sorted}|head -n1|awk '{print $1}')
     
107.             lib_realpath_basename="$(basename $(readlink -f ${libpath_sorted}))"
     
108.             lib_without_version_suffix="$(echo ${lib_realpath_basename} | awk -F"." 'BEGIN { OFS = "." }{ print $1, $2}')"
     
109.             if [[ ! -z $LDD  ]] && [[ "${minimal}" == "false" ]]; then
     
110.                 echo "Replacing lib ${lib_realpath_basename} to ${lib_without_version_suffix} for ${elf}"
     
111.                 patchelf --replace-needed ${LDD} ${lib_without_version_suffix} ${elf}
     
112.             fi
     
113.         done
     
114.     done
     
115. }
     
116. 
     
117. function check_libs {
     
118.     local elf_path=$1
     
119.     for elf in $(find ${elf_path} -maxdepth 1 -exec file {} \; | grep 'ELF ' | cut -d':' -f1); do
     
120.         if ! ldd ${elf}; then
     
121.             exit 1
     
122.         fi
     
123.     done
     
124. }
     
125. 
     
126. function link {
     
127.     if [ ! -d lib/private ]; then
     
128.         mkdir -p lib/private
     
129.     fi
     
130.     # Gather libs
     
131.     for DIR in ${DIRLIST}; do
     
132.         gather_libs ${DIR}
     
133.     done
     
134.     # Set proper runpath
     
135.     export override=false
     
136.     set_runpath bin '$ORIGIN/../lib/private/'
     
137.     set_runpath lib '$ORIGIN/private/'
     
138.     set_runpath lib/plugin '$ORIGIN/../private/'
     
139.     set_runpath lib/private '$ORIGIN'
     
140.     # LIBS MYSQLROUTER
     
141.     unset override && export override=true && set_runpath lib/mysqlrouter/plugin '$ORIGIN/:$ORIGIN/../private/:$ORIGIN/../../private/'
     
142.     unset override && export override=true && set_runpath lib/mysqlrouter/private '$ORIGIN/:$ORIGIN/../plugin/:$ORIGIN/../../private/'
     
143.     #  BINS MYSQLROUTER
     
144.     unset override && export override=true && set_runpath bin/mysqlrouter_passwd '$ORIGIN/../lib/mysqlrouter/private/:$ORIGIN/../lib/mysqlrouter/plugin/:$ORIGIN/../lib/private/'
     
145.     unset override && export override=true && set_runpath bin/mysqlrouter_plugin_info '$ORIGIN/../lib/mysqlrouter/private/:$ORIGIN/../lib/mysqlrouter/plugin/:$ORIGIN/../lib/private/'
     
146.     unset override && export override=true && set_runpath bin/mysqlrouter '$ORIGIN/../lib/mysqlrouter/private/:$ORIGIN/../lib/mysqlrouter/plugin/:$ORIGIN/../lib/private/'
     
147.     unset override && export override=true && set_runpath bin/mysqlrouter_keyring '$ORIGIN/../lib/mysqlrouter/private/:$ORIGIN/../lib/mysqlrouter/plugin/:$ORIGIN/../lib/private/'
     
148.     # Replace libs
     
149.     for DIR in ${DIRLIST}; do
     
150.         replace_libs ${DIR}
     
151.     done
     
152.     # Make final check in order to determine any error after linkage
     
153.     for DIR in ${DIRLIST}; do
     
154.         check_libs ${DIR}
     
155.     done
     
156. }
     
157. 
     
158. rm -fr ${MAKELOG}
     
159. 
     
160. cd /opt/${SOURCE_DIR} && \
     
161. rm -fr bld && \
     
162. mkdir bld && \
     
163. cd bld && \
     
164. cmake .. -DBOOST_INCLUDE_DIR=/opt/boost_${BOOST_VERSION} \
     
165. -DLOCAL_BOOST_DIR=/opt/boost_${BOOST_VERSION} \
     
166. -DCMAKE_INSTALL_PREFIX=${BASE_DIR} -DWITH_ZLIB=bundled \
     
167. -DWITH_NUMA=ON -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
     
168. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_release \
     
169. -DWITH_TOKUDB=OFF -DWITH_ROCKSDB=OFF \
     
170. -DROCKSDB_DISABLE_AVX2=1 \
     
171. -DROCKSDB_DISABLE_MARCH_NATIVE=1 \
     
172. -DGROUP_REPLICATION_WITH_ROCKSDB=OFF \
     
173. -DALLOW_NO_SSE42=ON \
     
174. -DMYSQL_MAINTAINER_MODE=OFF \
     
175. -DFORCE_INSOURCE_BUILD=1 \
     
176. -DCOMPILATION_COMMENT="GreatSQL, Release ${RELEASE}, Revision ${REVISION}" \
     
177. -DMAJOR_VERSION=${MAJOR_VERSION} -DMINOR_VERSION=${MINOR_VERSION} -DPATCH_VERSION=${PATCH_VERSION} \
     
178. -DWITH_NDB=OFF -DWITH_NDBCLUSTER_STORAGE_ENGINE=OFF -DWITH_NDBCLUSTER=OFF \
     
179. -DWITH_UNIT_TESTS=OFF -DWITH_SSL=system -DWITH_SYSTEMD=ON \
     
180. -DWITH_AUTHENTICATION_LDAP=OFF \
     
181. -DWITH_PAM=1 \
     
182. -DWITH_LIBEVENT=bundled \
     
183. -DWITH_LDAP=system \
     
184. -DWITH_SYSTEM_LIBS=ON \
     
185. -DWITH_LZ4=bundled \
     
186. -DWITH_PROTOBUF=bundled \
     
187. -DWITH_RAPIDJSON=bundled \
     
188. -DWITH_ICU=bundled \
     
189. -DWITH_READLINE=system \
     
190. -DWITH_ZSTD=bundled \
     
191. -DWITH_FIDO=bundled \
     
192. -DWITH_KEYRING_VAULT=ON \
     
193. >> ${MAKELOG} 2>&1 && make -j${JOBS} >> ${MAKELOG} 2>&1 && make -j${JOBS} install >> ${MAKELOG} 2>&1
     
194. 
     
195. rm -fr ${BASE_DIR}/mysql-test 2 > /dev/null
     
196. 
     
197. # strip binaries to get minimal package
     
198. minimal=true
     
199. echo "minimal = ${minimal}" >> ${MAKELOG} 2>&1
     
200. echo "link ${BASE_DIR}-minimal" >> ${MAKELOG} 2>&1
     
201. (
     
202.   cp -rp ${BASE_DIR} ${BASE_DIR}-minimal
     
203.   cd ${BASE_DIR}-minimal
     
204.   find . -type f -exec file '{}' \; | grep ': ELF ' | cut -d':' -f1 | xargs strip --strip-unneeded
     
205.   link >> ${MAKELOG} 2>&1
     
206. )
     
207. #如果要打包压缩，就把下面两行注释去掉
     
208. tar -cf ${BASE_DIR}-minimal.tar ${BASE_DIR}-minimal
     
209. xz -9 -f -T${JOBS} ${BASE_DIR}-minimal.tar ${BASE_DIR}-minimal
     
210. 
     
211. minimal=false
     
212. (
     
213.   cd ${BASE_DIR}
     
214.   link >> ${MAKELOG} 2>&1
     
215. )
     
216. #如果要打包压缩，就把下面两行注释去掉
     
217. tar -cf ${BASE_DIR}.tar ${BASE_DIR}
     
218. xz -9 -f -T${JOBS} ${BASE_DIR}.tar.xz ${BASE_DIR}
     

复制代码

mlovewt 发表于 2024-3-4 10:00

您好，我需要的是详细编译过程，包括这个过程中产生的日志，不只是编译脚本。

yejr 发表于 2024-3-4 11:25
您好，我需要的是详细编译过程，包括这个过程中产生的日志，不只是编译脚本。 ...

编译日志，和dockerfile,docker镜像是以前做的，没有重新构建。

mlovewt 发表于 2024-3-4 15:10
编译日志，和dockerfile,docker镜像是以前做的，没有重新构建。

看了日志，您的编译环境有确实的依赖包未安装

1. CMake Warning at cmake/fido2.cmake:70 (MESSAGE):
   
2.   Cannot find development libraries.  You need to install the required
   
3.   packages:
   
4. 
   
5.     Debian/Ubuntu:              apt install libudev-dev
   
6.     RedHat/Fedora/Oracle Linux: yum install libudev-devel
   
7.     SuSE:                       zypper install libudev-devel

复制代码

建议使用我们提供的GreatSQL-Build Docker镜像，详情参考 https://gitee.com/GreatSQL/Great ... ster/GreatSQL-Build