diff --git a/CMakeLists.txt b/CMakeLists.txt index 2324f9d64..2cec9e66d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,23 @@ if (WIN32) endif () endif () endif () + + if (HAVE_firebird) + option (WITH_FIREBIRD "compile with firebird rdbms support" ON) + if (WITH_FIREBIRD) + set (USE_FIREBIRD 1) + include_directories (${firebird_INCLUDE}) + + CMAKE_DEPENDENT_OPTION (DL_FIREBIRD "load firebird rdbms library dynamically" OFF "USE_FIREBIRD" ON) + if (DL_FIREBIRD) + set (DL_FIREBIRD 1) + set (FIREBIRD_LIB fbclient.dll) + else( DL_FIREBIRD ) + list (APPEND EXTRA_LIBRARIES ${firebird_LIB}) + endif () + install (FILES ${firebird_ROOT}/fbclient.dll DESTINATION bin COMPONENT APPLICATIONS) + endif () + endif () mark_as_advanced(CMAKE_CONFIGURATION_TYPES) else (WIN32) @@ -185,7 +202,8 @@ else (WIN32) ac_check_funcs ("backtrace;backtrace_symbols") ac_check_funcs ("mremap") ac_check_funcs ("nanosleep;pthread_mutex_timedlock") - ac_check_funcs ( "eventfd") + ac_check_funcs ("eventfd") + ac_check_funcs ("kqueue") check_function_exists (epoll_ctl HAVE_EPOLL) ac_search_libs ("rt" "clock_gettime" HAVE_CLOCK_GETTIME EXTRA_LIBRARIES) @@ -363,6 +381,55 @@ else (WIN32) endif (PostgreSQL_FOUND) endif (WITH_PGSQL) + # test for FIREBIRD RDBMS + message (STATUS "Option WITH_FIREBIRD ${WITH_FIREBIRD}") + option (WITH_FIREBIRD "compile with Firebird RDBMS support" OFF) + set (WITH_FIREBIRD_INCLUDES "" CACHE PATH "path to Firebird RDBMS header files") + set (WITH_FIREBIRD_LIBS "" CACHE PATH "path to Firebird RDBMS library") + + if (WITH_FIREBIRD) + if (WITH_FIREBIRD_INCLUDES) + set (Firebird_INCLUDE_DIR ${WITH_FIREBIRD_INCLUDES}) + endif (WITH_FIREBIRD_INCLUDES) + + if (WITH_FIREBIRD_LIBS) + set (Firebird_LIBRARIES ${WITH_FIREBIRD_LIBS}/libfbclient.so) + endif (WITH_FIREBIRD_LIBS) + + if (NOT (WITH_FIREBIRD_INCLUDES AND WITH_FIREBIRD_LIBS)) + find_package (Firebird) + endif () + + if (Firebird_FOUND) + set (USE_FIREBIRD 1) + include_directories (${Firebird_INCLUDE_DIRS}) + + CMAKE_DEPENDENT_OPTION (DL_FIREBIRD "load firebird rdbms library dynamically" ON "Firebird_FOUND;HAVE_DL;NOT STATIC_FIREBIRD" OFF) + CMAKE_DEPENDENT_OPTION (STATIC_FIREBIRD "link to firebird rdbms library statically" OFF "Firebird_FOUND;NOT DL_FIREBIRD" OFF) + if (STATIC_FIREBIRD) + message (STATUS "Firebird RDBMS will be linked statically") + string (REGEX REPLACE "fbclient" "libfbclient.a" FIREBIRD_LIBRARIES "${FIREBIRD_LIBRARIES}") + endif (STATIC_FIREBIRD) + + if (DL_FIREBIRD) + message (STATUS "Firebird RDBMS will not be linked (will be loaded at runtime)") + set (DL_FIREBIRD 1) + GET_FILENAME_COMPONENT (FIREBIRD_LIB ${Firebird_LIBRARY} NAME) + else ( DL_FIREBIRD ) + list (APPEND EXTRA_LIBRARIES ${Firebird_LIBRARIES}) + endif () + else (Firebird_FOUND) + message (SEND_ERROR + "******************************************************************************** + ERROR: cannot find Firebird RDBMS libraries. If you want to compile with Firebird RDBMS support, + you must either specify file locations explicitly using + -D WITH_FIREBIRD_INCLUDES=... and -D WITH_FIREBIRD_LIBS=... options, or make sure path to + fb_config is listed in your PATH environment variable. If you want to + disable Firebird RDBMS support, use -D WITH_FIREBIRD=OFF option. + ********************************************************************************") + endif (Firebird_FOUND) + endif (WITH_FIREBIRD) + endif (WIN32) if (NOT CMAKE_BUILD_TYPE MATCHES Debug) @@ -581,6 +648,9 @@ LIST (APPEND BANNER "WITH_ODBC=${WITH_ODBC}") LIST (APPEND BANNER "WITH_PGSQL=${WITH_PGSQL}") LIST (APPEND BANNER "WITH_PGSQL_INCLUDES=${WITH_PGSQL_INCLUDES}") LIST (APPEND BANNER "WITH_PGSQL_LIBS=${WITH_PGSQL_LIBS}") +LIST (APPEND BANNER "WITH_FIREBIRD=${WITH_FIREBIRD}") +LIST (APPEND BANNER "WITH_FIREBIRD_INCLUDES=${WITH_FIREBIRD_INCLUDES}") +LIST (APPEND BANNER "WITH_FIREBIRD_LIBS=${WITH_FIREBIRD_LIBS}") LIST (APPEND BANNER "WITH_RE2=${WITH_RE2}") LIST (APPEND BANNER "WITH_RE2_INCLUDES=${WITH_RE2_INCLUDES}") LIST (APPEND BANNER "WITH_RE2_LIBS=${WITH_RE2_LIBS}") diff --git a/Makefile.in b/Makefile.in index 0e6ede8b9..ccbf42489 100644 --- a/Makefile.in +++ b/Makefile.in @@ -197,6 +197,8 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ PGSQL_CFLAGS = @PGSQL_CFLAGS@ PGSQL_LIBS = @PGSQL_LIBS@ +FIREBIRD_CFLAGS = @FIREBIRD_CFLAGS@ +FIREBIRD_LIBS = @FIREBIRD_LIBS@ RANLIB = @RANLIB@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ diff --git a/acinclude.m4 b/acinclude.m4 index e09697ead..a53ea7139 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -297,6 +297,82 @@ ERROR: cannot find PostgreSQL libraries. If you want to compile with PosgregSQL fi ]) +dnl --------------------------------------------------------------------------- +dnl Macro: AC_CHECK_FIREBIRD +dnl First check for custom Firebird RDBMS paths in --with-firebird-* options. +dnl If some paths are missing, check if fb_config exists. +dnl --------------------------------------------------------------------------- + +AC_DEFUN([AC_CHECK_FIREBIRD],[ + +# Check for custom includes path +if test [ -z "$ac_cv_firebird_includes" ] +then + AC_ARG_WITH([firebird-includes], + AC_HELP_STRING([--with-firebird-includes], [path to Firebird RDBMS header files]), + [ac_cv_firebird_includes=$withval]) +fi +if test [ -n "$ac_cv_firebird_includes" ] +then + AC_CACHE_CHECK([Firebird RDBMS includes], [ac_cv_firebird_includes], [ac_cv_firebird_includes=""]) + FIREBIRD_CFLAGS="-I$ac_cv_firebird_includes" +fi + +# Check for custom library path +if test [ -z "$ac_cv_firebird_libs" ] +then + AC_ARG_WITH([firebird-libs], + AC_HELP_STRING([--with-firebird-libs], [path to Firebird RDBMS libraries]), + [ac_cv_firebird_libs=$withval]) +fi +if test [ -n "$ac_cv_firebird_libs" ] +then + AC_CACHE_CHECK([Firebird RDBMS libraries], [ac_cv_firebird_libs], [ac_cv_firebird_libs=""]) + FIREBIRD_PKGLIBDIR="$ac_cv_firebird_libs" + FIREBIRD_LIBS="-L$FIREBIRD_PKGLIBDIR -lfbclient" +fi + +# If some path is missing, try to autodetermine with firebird_config +if test [ -z "$ac_cv_firebird_includes" -o -z "$ac_cv_firebird_libs" ] +then + if test [ -z "$fbconfig" ] + then + AC_PATH_PROG(fbconfig,fb_config) + fi + if test [ -z "$fbconfig" ] + then + AC_MSG_ERROR([fb_config executable not found +******************************************************************************** +ERROR: cannot find Firebird RDBMS libraries. If you want to compile with Firebird RDBMS support, + you must either specify file locations explicitly using + --with-firebird-includes and --with-firebird-libs options, or make sure path to + fb_config is listed in your PATH environment variable. If you want to + disable Firebird RDBMS support, use --without-firebird option. +******************************************************************************** +]) + else + if test [ -z "$ac_cv_firebird_includes" ] + then + AC_MSG_CHECKING(Firebird RDBMS C flags) + FIREBIRD_CFLAGS="`${fbconfig} --cflags`" + ### /opt/firebird/bin/fb_config --cflags: + ### -I/opt/firebird/include + AC_MSG_RESULT($FIREBIRD_CFLAGS) + fi + if test [ -z "$ac_cv_firebird_libs" ] + then + AC_MSG_CHECKING(Firebird RDBMS linker flags) + FIREBIRD_PKGLIBDIR=`${fbconfig} --libs` + ### /opt/firebird/bin/fb_config --libs: + ### -L/opt/firebird/lib -lfbclient + FIREBIRD_LIBS="$FIREBIRD_PKGLIBDIR" + AC_MSG_RESULT($FIREBIRD_LIBS) + fi + fi +fi +]) + + dnl --------------------------------------------------------------------------- dnl Macro: AC_CHECK_LIBSTEMMER dnl Check the libstemmer first in custom include path in --with-libstemmer=* diff --git a/cmake/select_library_dir.cmake b/cmake/select_library_dir.cmake index f1ff5c713..b12794bcf 100644 --- a/cmake/select_library_dir.cmake +++ b/cmake/select_library_dir.cmake @@ -12,7 +12,7 @@ if (LIBS_BUNDLE) # here is the list of the libs we expect to find foreach (req_lib expat iconv mysql pq) - file(GLOB list_libs "${LIBS_BUNDLE}/*${req_lib}*" "${LIBS_BUNDLE}/*pgsql*") + file(GLOB list_libs "${LIBS_BUNDLE}/*${req_lib}*" "${LIBS_BUNDLE}/*pgsql*" "${LIBS_BUNDLE}/*firebird*") SET (flib FALSE) # select whether we need -x64 or simple lib for our current arch FOREACH (lib ${list_libs}) diff --git a/config/config.h.in b/config/config.h.in index 901d15839..a4faca302 100644 --- a/config/config.h.in +++ b/config/config.h.in @@ -15,6 +15,9 @@ /* Define to 1 if runtime load PosgreSQL using dlopen */ #undef DL_PGSQL +/* Define to 1 if runtime load Firebird RDBMS using dlopen */ +#undef DL_FIREBIRD + /* Define to 1 if you want runtime load odbc/iodbc using dlopen */ #undef DL_UNIXODBC @@ -288,6 +291,9 @@ /* Define to 1 if you want to compile with PostgreSQL support */ #undef USE_PGSQL +/* Define to 1 if you want to compile with Firebird RDBMS support */ +#undef USE_FIREBIRD + /* RE2 library support */ #undef USE_RE2 diff --git a/config/config_cmake.h.in b/config/config_cmake.h.in index 7041de4ec..e87f88722 100644 --- a/config/config_cmake.h.in +++ b/config/config_cmake.h.in @@ -24,6 +24,10 @@ #cmakedefine DL_PGSQL ${DL_PGSQL} #cmakedefine PGSQL_LIB "${PGSQL_LIB}" +/* Define to 1 if runtime load Firebird RDBMS using dlopen */ +#cmakedefine DL_FIREBIRD ${DL_FIREBIRD} +#cmakedefine FIREBIRD_LIB "${FIREBIRD_LIB}" + /* Define to 1 if you want runtime load odbc/iodbc using dlopen */ #cmakedefine DL_UNIXODBC ${DL_UNIXODBC} #cmakedefine UNIXODBC_LIB "${UNIXODBC_LIB}" @@ -43,6 +47,9 @@ /* Define if your system supports the epoll system calls */ #cmakedefine HAVE_EPOLL ${HAVE_EPOLL} +/* Define if your system supports the kqueue system calls */ +#cmakedefine HAVE_KQUEUE ${HAVE_KQUEUE} + /* Define if your system supports the eventfd system calls */ #cmakedefine01 HAVE_EVENTFD @@ -121,6 +128,9 @@ /* Define to 1 if you want to compile with PostgreSQL support */ #cmakedefine USE_PGSQL ${USE_PGSQL} +/* Define to 1 if you want to compile with Firebird RDBMS support */ +#cmakedefine USE_FIREBIRD ${USE_FIREBIRD} + /* re2 library support */ #cmakedefine USE_RE2 ${USE_RE2} diff --git a/configure b/configure index b552f8c26..1903b40f0 100755 --- a/configure +++ b/configure @@ -624,6 +624,11 @@ USE_PGSQL_TRUE PGSQL_CFLAGS PGSQL_LIBS pgconfig +USE_FIREBIRD_FALSE +USE_FIREBIRD_TRUE +FIREBIRD_CFLAGS +FIREBIRD_LIBS +fbconfig USE_MYSQL_FALSE USE_MYSQL_TRUE MYSQL_CFLAGS @@ -743,6 +748,10 @@ with_pgsql with_static_pgsql with_pgsql_includes with_pgsql_libs +with_firebird +with_static_firebird +with_firebird_includes +with_firebird_libs enable_id64 with_libstemmer with_libexpat @@ -1414,6 +1423,11 @@ Optional Packages: no) --with-pgsql-includes path to PostgreSQL header files --with-pgsql-libs path to PostgreSQL libraries + --with-firebird compile with Firebird RDBMS support (default is + disabled) + --with-static-firebird link statically with Firebird RDBMS library (default is no) + --with-firebird-includes path to Firebird RDBMS header files + --with-firebird-libs path to Firebird RDBMS libraries --with-libstemmer compile with libstemmer support (default is disabled) --with-libexpat compile with expat XML library suppport (default is @@ -7925,6 +7939,458 @@ else fi + +# check if we should compile with Firebird support + +# Check whether --with-firebird was given. +if test "${with_firebird+set}" = set; then : + withval=$with_firebird; ac_cv_use_firebird=$withval +else + ac_cv_use_firebird=no + +fi + +# check if we should statically link the Firebird RDBMS library + +# Check whether --with-static-firebird was given. +if test "${with_static_firebird+set}" = set; then : + withval=$with_static_firebird; ac_cv_use_static_firebird=$withval +else + ac_cv_use_static_firebird=no + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to compile with Firebird RDBMS support" >&5 +$as_echo_n "checking whether to compile with Firebird RDBMS support... " >&6; } +if test x$ac_cv_use_static_firebird != xno -o x$ac_cv_use_firebird != xno +then + dl_firebird=0 + if test x$ac_cv_use_static_firebird != xno ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: static" >&5 +$as_echo "static" >&6; } + + +# Check for custom includes path +if test -z "$ac_cv_firebird_includes" +then + +# Check whether --with-firebird-includes was given. +if test "${with_firebird_includes+set}" = set; then : + withval=$with_firebird_includes; ac_cv_firebird_includes=$withval +fi + +fi +if test -n "$ac_cv_firebird_includes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS includes" >&5 +$as_echo_n "checking Firebird RDBMS includes... " >&6; } +if ${ac_cv_firebird_includes+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_includes="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_includes" >&5 +$as_echo "$ac_cv_firebird_includes" >&6; } + FIREBIRD_CFLAGS="-I$ac_cv_firebird_includes" +fi + +# Check for custom library path +if test -z "$ac_cv_firebird_libs" +then + +# Check whether --with-firebird-libs was given. +if test "${with_firebird_libs+set}" = set; then : + withval=$with_firebird_libs; ac_cv_firebird_libs=$withval +fi + +fi +if test -n "$ac_cv_firebird_libs" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird libraries" >&5 +$as_echo_n "checking Firebird libraries... " >&6; } +if ${ac_cv_firebird_libs+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_libs="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_libs" >&5 +$as_echo "$ac_cv_firebird_libs" >&6; } + FIREBIRD_PKGLIBDIR="$ac_cv_firebird_libs" + FIREBIRD_LIBS="-L$FIREBIRD_PKGLIBDIR -lfbclient" +fi + +# If some path is missing, try to autodetermine with fb_config +if test -z "$ac_cv_firebird_includes" -o -z "$ac_cv_firebird_libs" +then + if test -z "$fbconfig" + then + # Extract the first word of "fb_config", so it can be a program name with args. +set dummy fb_config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_fbconfig+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $fbconfig in + [\\/]* | ?:[\\/]*) + ac_cv_path_fbconfig="$fbconfig" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_fbconfig="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +fbconfig=$ac_cv_path_fbconfig +if test -n "$fbconfig"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $fbconfig" >&5 +$as_echo "$fbconfig" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi + if test -z "$fbconfig" + then + as_fn_error $? "fb_config executable not found +******************************************************************************** +ERROR: cannot find Firebird RDBMS libraries. If you want to compile with Firebird support, + you must either specify file locations explicitly using + --with-firebird-includes and --with-firebird-libs options, or make sure path to + fb_config is listed in your PATH environment variable. If you want to + disable Firebird RDBMS support, use --without-firebird option. +******************************************************************************** +" "$LINENO" 5 + else + if test -z "$ac_cv_firebird_includes" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS C flags" >&5 +$as_echo_n "checking Firebird RDBMS C flags... " >&6; } + FIREBIRD_CFLAGS="`${fbconfig} --cflags`" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_CFLAGS" >&5 +$as_echo "$FIREBIRD_CFLAGS" >&6; } + fi + if test -z "$ac_cv_firebird_libs" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS linker flags" >&5 +$as_echo_n "checking Firebird RDBMS linker flags... " >&6; } + FIREBIRD_PKGLIBDIR=`${fbconfig} --libs` + FIREBIRD_LIBS="$FIREBIRD_PKGLIBDIR" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_LIBS" >&5 +$as_echo "$FIREBIRD_LIBS" >&6; } + fi + fi +fi + + FIREBIRD_LIBS=`echo $FIREBIRD_LIBS | sed -e "sX-lfbclientX$FIREBIRD_PKGLIBDIR/libfbclient.aXg"` + else + if test x$sph_usedl == xyes ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: runtime dynamic" >&5 +$as_echo "runtime dynamic" >&6; } + + +# Check for custom includes path +if test -z "$ac_cv_firebird_includes" +then + +# Check whether --with-firebird-includes was given. +if test "${with_firebird_includes+set}" = set; then : + withval=$with_firebird_includes; ac_cv_firebird_includes=$withval +fi + +fi +if test -n "$ac_cv_firebird_includes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS includes" >&5 +$as_echo_n "checking Firebird RDBMS includes... " >&6; } +if ${ac_cv_firebird_includes+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_includes="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_includes" >&5 +$as_echo "$ac_cv_firebird_includes" >&6; } + FIREBIRD_CFLAGS="-I$ac_cv_firebird_includes" +fi + +# Check for custom library path +if test -z "$ac_cv_firebird_libs" +then + +# Check whether --with-firebird-libs was given. +if test "${with_firebird_libs+set}" = set; then : + withval=$with_firebird_libs; ac_cv_firebird_libs=$withval +fi + +fi +if test -n "$ac_cv_firebird_libs" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS libraries" >&5 +$as_echo_n "checking Firebird RDBMS libraries... " >&6; } +if ${ac_cv_firebird_libs+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_libs="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_libs" >&5 +$as_echo "$ac_cv_firebird_libs" >&6; } + FIREBIRD_PKGLIBDIR="$ac_cv_firebird_libs" + FIREBIRD_LIBS="-L$FIREBIRD_PKGLIBDIR -lfbclient" +fi + +# If some path is missing, try to autodetermine with firebird_config +if test -z "$ac_cv_firebird_includes" -o -z "$ac_cv_firebird_libs" +then + if test -z "$fbconfig" + then + # Extract the first word of "fb_config", so it can be a program name with args. +set dummy fb_config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_fbconfig+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $fbconfig in + [\\/]* | ?:[\\/]*) + ac_cv_path_fbconfig="$fbconfig" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_fbconfig="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +fbconfig=$ac_cv_path_fbconfig +if test -n "$fbconfig"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $fbconfig" >&5 +$as_echo "$fbconfig" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi + if test -z "$fbconfig" + then + as_fn_error $? "fb_config executable not found +******************************************************************************** +ERROR: cannot find Firebird RDBMS libraries. If you want to compile with Firebird RDBMS support, + you must either specify file locations explicitly using + --with-firebird-includes and --with-firebird-libs options, or make sure path to + fb_config is listed in your PATH environment variable. If you want to + disable Firebird RDBMS support, use --without-firebird option. +******************************************************************************** +" "$LINENO" 5 + else + if test -z "$ac_cv_firebird_includes" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS C flags" >&5 +$as_echo_n "checking Firebird RDBMS C flags... " >&6; } + FIREBIRD_CFLAGS="`${fbconfig} --cflags`" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_CFLAGS" >&5 +$as_echo "$FIREBIRD_CFLAGS" >&6; } + fi + if test -z "$ac_cv_firebird_libs" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS linker flags" >&5 +$as_echo_n "checking Firebird RDBMS linker flags... " >&6; } + FIREBIRD_PKGLIBDIR=`${fbconfig} --libs` + FIREBIRD_LIBS="$FIREBIRD_PKGLIBDIR" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_LIBS" >&5 +$as_echo "$FIREBIRD_LIBS" >&6; } + fi + fi +fi + + FIREBIRD_LIBS="" + dl_firebird=1 + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: dynamic" >&5 +$as_echo "dynamic" >&6; } + + +# Check for custom includes path +if test -z "$ac_cv_firebird_includes" +then + +# Check whether --with-firebird-includes was given. +if test "${with_firebird_includes+set}" = set; then : + withval=$with_firebird_includes; ac_cv_firebird_includes=$withval +fi + +fi +if test -n "$ac_cv_firebird_includes" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS includes" >&5 +$as_echo_n "checking Firebird RDBMS includes... " >&6; } +if ${ac_cv_firebird_includes+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_includes="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_includes" >&5 +$as_echo "$ac_cv_firebird_includes" >&6; } + FIREBIRD_CFLAGS="-I$ac_cv_firebird_includes" +fi + +# Check for custom library path +if test -z "$ac_cv_firebird_libs" +then + +# Check whether --with-firebird-libs was given. +if test "${with_firebird_libs+set}" = set; then : + withval=$with_firebird_libs; ac_cv_firebird_libs=$withval +fi + +fi +if test -n "$ac_cv_firebird_libs" +then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS libraries" >&5 +$as_echo_n "checking Firebird RDBMS libraries... " >&6; } +if ${ac_cv_firebird_libs+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_firebird_libs="" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_firebird_libs" >&5 +$as_echo "$ac_cv_firebird_libs" >&6; } + FIREBIRD_PKGLIBDIR="$ac_cv_firebird_libs" + FIREBIRD_LIBS="-L$FIREBIRD_PKGLIBDIR -lfbclient" +fi + +# If some path is missing, try to autodetermine with firebird_config +if test -z "$ac_cv_firebird_includes" -o -z "$ac_cv_firebird_libs" +then + if test -z "$fbconfig" + then + # Extract the first word of "fb_config", so it can be a program name with args. +set dummy fb_config; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_fbconfig+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $fbconfig in + [\\/]* | ?:[\\/]*) + ac_cv_path_fbconfig="$fbconfig" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_path_fbconfig="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +fbconfig=$ac_cv_path_fbconfig +if test -n "$fbconfig"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $fbconfig" >&5 +$as_echo "$fbconfig" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + fi + if test -z "$fbconfig" + then + as_fn_error $? "fb_config executable not found +******************************************************************************** +ERROR: cannot find Firebird RDBMS libraries. If you want to compile with Firebird RDBMS support, + you must either specify file locations explicitly using + --with-firebird-includes and --with-firebird-libs options, or make sure path to + fb_config is listed in your PATH environment variable. If you want to + disable Firebird RDBMS support, use --without-firebird option. +******************************************************************************** +" "$LINENO" 5 + else + if test -z "$ac_cv_firebird_includes" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS C flags" >&5 +$as_echo_n "checking Firebird RDBMS C flags... " >&6; } + FIREBIRD_CFLAGS="`${fbconfig} --cflags`" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_CFLAGS" >&5 +$as_echo "$FIREBIRD_CFLAGS" >&6; } + fi + if test -z "$ac_cv_firebird_libs" + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking Firebird RDBMS linker flags" >&5 +$as_echo_n "checking Firebird RDBMS linker flags... " >&6; } + FIREBIRD_PKGLIBDIR=`${fbconfig} --libs` + FIREBIRD_LIBS="$FIREBIRD_PKGLIBDIR" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FIREBIRD_LIBS" >&5 +$as_echo "$FIREBIRD_LIBS" >&6; } + fi + fi +fi + + fi + fi + +$as_echo "#define USE_FIREBIRD 1" >>confdefs.h + + +cat >>confdefs.h <<_ACEOF +#define DL_FIREBIRD $dl_firebird +_ACEOF + + + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + if test x$ac_cv_use_firebird != xno -o x$ac_cv_use_static_firebird != xno ; then + USE_FIREBIRD_TRUE= + USE_FIREBIRD_FALSE='#' +else + USE_FIREBIRD_TRUE='#' + USE_FIREBIRD_FALSE= +fi + + + # add macports include directory if (echo $MYSQL_LIBS | grep -q -- -L/opt/local/lib); then MYSQL_CFLAGS="$MYSQL_CFLAGS -I/opt/local/include" @@ -8089,7 +8555,7 @@ fi # we can now set preprocessor flags for both C and C++ compilers -CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $LIBSTEMMER_CFLAGS" +CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $FIREBIRD_CFLAGS $LIBSTEMMER_CFLAGS" @@ -8255,7 +8721,7 @@ fi # we can now set preprocessor flags for both C and C++ compilers -CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $LIBRE2_CFLAGS" +CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $FIREBIRD_CFLAGS $LIBRE2_CFLAGS" # Check whether --with-rlp was given. @@ -9160,6 +9626,10 @@ if test -z "${USE_PGSQL_TRUE}" && test -z "${USE_PGSQL_FALSE}"; then as_fn_error $? "conditional \"USE_PGSQL\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${USE_FIREBIRD_TRUE}" && test -z "${USE_FIREBIRD_FALSE}"; then + as_fn_error $? "conditional \"USE_FIREBIRD\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${USE_LIBSTEMMER_TRUE}" && test -z "${USE_LIBSTEMMER_FALSE}"; then as_fn_error $? "conditional \"USE_LIBSTEMMER\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 diff --git a/configure.ac b/configure.ac index 39624407c..ee63f7819 100644 --- a/configure.ac +++ b/configure.ac @@ -437,7 +437,58 @@ then else AC_MSG_RESULT([no]) fi -AM_CONDITIONAL(USE_PGSQL, test x$ac_cv_use_pgsql != xno -o x$ac_cv_use_satic_pgsql != xno ) +AM_CONDITIONAL(USE_PGSQL, test x$ac_cv_use_pgsql != xno -o x$ac_cv_use_static_pgsql != xno ) + +# check if we should compile with Firebird RDBMS support +AC_ARG_WITH([firebird], + AC_HELP_STRING([--with-firebird], [compile with Firebird RDBMS support (default is disabled)]), + [ac_cv_use_firebird=$withval], [ac_cv_use_firebird=no] +) +# check if we should statically link the Firebird RDBMS library +AC_ARG_WITH([static-firebird], + AC_HELP_STRING([--with-static-firebird], [link statically with Firebird RDBMS library (default is no)]), + [ac_cv_use_static_firebird=$withval], [ac_cv_use_static_firebird=no] +) +AC_MSG_CHECKING([whether to compile with Firebird RDBMS support]) +if test x$ac_cv_use_static_firebird != xno -o x$ac_cv_use_firebird != xno +then + dl_firebird=0 + if test x$ac_cv_use_static_firebird != xno ; then + AC_MSG_RESULT([static]) + AC_CHECK_FIREBIRD([$ac_cv_use_static_firebird]) + FIREBIRD_LIBS=`echo $FIREBIRD_LIBS | sed -e "sX-lfbclientX$FIREBIRD_PKGLIBDIR/libfbclient.aXg"` +### +### L_VV Try do it as other DB does: +### /opt/firebird/bin/fb_config --libs: +### -L/opt/firebird/lib -lfbclient +### file: /opt/firebird/lib/libfbclient.a (".a" file is a static library, while a ".so" file is a shared object (dynamic) library) +### PGSQL_LIBS =`echo $PGSQL_LIBS | sed -e "sX-lpqX$PGSQL_PKGLIBDIR/libpq.aXg"` +### MYSQL_LIBS =`echo $MYSQL_LIBS | sed -e "sX-lmysqlclientX\$MYSQL_PKGLIBDIR/libmysqlclient.aXg"` +### +### In FIREBIRD_LIBS finally we get the string: +### -L/opt/firebird/lib /opt/firebird/lib/libfbclient.a +### The next line give the same result (in it present '\' after 'X' in the middle of the line): +### FIREBIRD_LIBS=`echo $FIREBIRD_LIBS | sed -e "sX-lfbclientX\$FIREBIRD_PKGLIBDIR/libfbclient.aXg"` +### + else + if test x$sph_usedl == xyes ; then + AC_MSG_RESULT([runtime dynamic]) + AC_CHECK_FIREBIRD([ac_cv_use_firebird]) + FIREBIRD_LIBS="" + dl_firebird=1 + else + AC_MSG_RESULT([dynamic]) + AC_CHECK_FIREBIRD([ac_cv_use_firebird]) + fi + fi + AC_DEFINE(USE_FIREBIRD,1,[Define to 1 if you want to compile with Firebird RDBMS support]) + AC_DEFINE_UNQUOTED(DL_FIREBIRD,$dl_firebird,[Define to 1 if runtime load Firebird RDBMS using dlopen]) + AC_SUBST([FIREBIRD_LIBS]) + AC_SUBST([FIREBIRD_CFLAGS]) +else + AC_MSG_RESULT([no]) +fi +AM_CONDITIONAL(USE_FIREBIRD, test x$ac_cv_use_firebird != xno -o x$ac_cv_use_static_firebird != xno ) # add macports include directory if (echo $MYSQL_LIBS | grep -q -- -L/opt/local/lib); then @@ -493,7 +544,7 @@ AM_CONDITIONAL(USE_INTERNAL_LIBSTEMMER, test x$ac_cv_use_internal_libstemmer != dnl --- # we can now set preprocessor flags for both C and C++ compilers -CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $LIBSTEMMER_CFLAGS" +CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $FIREBIRD_CFLAGS $LIBSTEMMER_CFLAGS" AC_ARG_WITH([libexpat], @@ -542,7 +593,7 @@ AM_CONDITIONAL(USE_RE2, test x$ac_cv_use_re2 != xno) dnl --- # we can now set preprocessor flags for both C and C++ compilers -CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $LIBRE2_CFLAGS" +CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS $PGSQL_CFLAGS $FIREBIRD_CFLAGS $LIBRE2_CFLAGS" AC_ARG_WITH([rlp], AC_HELP_STRING([--with-rlp], [compile with RLP library support (default is disabled)]), diff --git a/sphinx.spec b/sphinx.spec index 3863a53c1..08d9906a9 100644 --- a/sphinx.spec +++ b/sphinx.spec @@ -93,7 +93,7 @@ sed -i 's/\r//' api/ruby/lib/sphinx/response.rb %build -%configure --sysconfdir=/etc/sphinx --with-mysql --with-re2 --with-libstemmer --with-unixodbc --with-iconv --enable-id64 --with-pgsql --with-syslog +%configure --sysconfdir=/etc/sphinx --with-mysql --with-re2 --with-libstemmer --with-unixodbc --with-iconv --enable-id64 --with-pgsql --with-firebird --with-syslog make %{?_smp_mflags} diff --git a/src/Makefile.am b/src/Makefile.am index c7ae07cb3..d6c659a09 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,7 @@ SRC_SPHINX = sphinx.cpp sphinxexcerpt.cpp sphinxquery.cpp \ sphinxsearch.cpp sphinxrt.cpp sphinxjson.cpp sphinxudf.c sphinxaot.cpp sphinxplugin.cpp sphinxqcache.cpp \ sphinxrlp.cpp -ARFLAGS = crU +ARFLAGS = cr noinst_LIBRARIES = libsphinx.a libsphinx_a_SOURCES = $(SRC_SPHINX) @@ -32,5 +32,5 @@ RLP_INC = endif AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\"" -COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) +COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(FIREBIRD_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) LDADD = $(COMMON_LIBS) diff --git a/src/Makefile.in b/src/Makefile.in index e1540d10d..ad87e8ab5 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -196,6 +196,8 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ PGSQL_CFLAGS = @PGSQL_CFLAGS@ PGSQL_LIBS = @PGSQL_LIBS@ +FIREBIRD_CFLAGS = @FIREBIRD_CFLAGS@ +FIREBIRD_LIBS = @FIREBIRD_LIBS@ RANLIB = @RANLIB@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ @@ -251,7 +253,7 @@ SRC_SPHINX = sphinx.cpp sphinxexcerpt.cpp sphinxquery.cpp \ sphinxsearch.cpp sphinxrt.cpp sphinxjson.cpp sphinxudf.c sphinxaot.cpp sphinxplugin.cpp sphinxqcache.cpp \ sphinxrlp.cpp -ARFLAGS = crU +ARFLAGS = cr noinst_LIBRARIES = libsphinx.a libsphinx_a_SOURCES = $(SRC_SPHINX) indexer_SOURCES = indexer.cpp @@ -266,7 +268,7 @@ BUILT_SOURCES = extract-version @USE_RLP_FALSE@RLP_INC = @USE_RLP_TRUE@RLP_INC = -I$(top_srcdir)/rlp/rlp/include -I$(top_srcdir)/rlp/utilities/include -D_REENTRANT AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\"" -COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) +COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(FIREBIRD_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) LDADD = $(COMMON_LIBS) all: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) all-am diff --git a/src/indexer.cpp b/src/indexer.cpp index 3d136ad2e..1f49af5be 100644 --- a/src/indexer.cpp +++ b/src/indexer.cpp @@ -705,6 +705,28 @@ CSphSource * SpawnSourceMySQL ( const CSphConfigSection & hSource, const char * #endif // USE_MYSQL +#if USE_FIREBIRD +CSphSource * SpawnSourceFBSQL( const CSphConfigSection & hSource, const char * sSourceName, bool bProxy ) +{ + assert ( hSource["type"]=="firebird" ); + + CSphSourceParams_FBSQL tParams; + if ( !SqlParamsConfigure ( tParams, hSource, sSourceName ) ) + return NULL; + + LOC_GETS ( tParams.m_sCharset, "sql_charset" ); + LOC_GETS ( tParams.m_sRole, "sql_role" ); + + CSphSource_FBSQL * pSrcFBSQL = CreateSourceWithProxy ( sSourceName, bProxy ); + if ( !pSrcFBSQL->Setup ( tParams ) ) + SafeDelete ( pSrcFBSQL ); + + return pSrcFBSQL; +} +#endif // USE_FIREBIRD + + + #if USE_ODBC CSphSource * SpawnSourceODBC ( const CSphConfigSection & hSource, const char * sSourceName, bool bProxy ) { @@ -837,6 +859,11 @@ CSphSource * SpawnSource ( const CSphConfigSection & hSource, const char * sSour return SpawnSourceMySQL ( hSource, sSourceName, bBatchedRLP ); #endif + #if USE_FIREBIRD + if ( hSource["type"]=="firebird") + return SpawnSourceFBSQL( hSource, sSourceName, bBatchedRLP ); + #endif + #if USE_ODBC if ( hSource["type"]=="odbc" ) return SpawnSourceODBC ( hSource, sSourceName, bBatchedRLP ); diff --git a/src/searchd.cpp b/src/searchd.cpp index 2cb01e3dc..9058de59e 100644 --- a/src/searchd.cpp +++ b/src/searchd.cpp @@ -208,71 +208,6 @@ static int g_iAgentRetryCount = 0; static int g_iAgentRetryDelay = MAX_RETRY_DELAY/2; // global (default) values. May be override by the query options 'retry_count' and 'retry_timeout' bool g_bHostnameLookup = false; - -struct ListNode_t -{ - ListNode_t * m_pPrev; - ListNode_t * m_pNext; - ListNode_t () - : m_pPrev ( NULL ) - , m_pNext ( NULL ) - { } -}; - - -class List_t -{ -public: - List_t () - { - m_tStub.m_pPrev = &m_tStub; - m_tStub.m_pNext = &m_tStub; - m_iCount = 0; - } - - void Add ( ListNode_t * pNode ) - { - assert ( !pNode->m_pNext && !pNode->m_pPrev ); - pNode->m_pNext = m_tStub.m_pNext; - pNode->m_pPrev = &m_tStub; - m_tStub.m_pNext->m_pPrev = pNode; - m_tStub.m_pNext = pNode; - - m_iCount++; - } - - void Remove ( ListNode_t * pNode ) - { - assert ( pNode->m_pNext && pNode->m_pPrev ); - pNode->m_pNext->m_pPrev = pNode->m_pPrev; - pNode->m_pPrev->m_pNext = pNode->m_pNext; - pNode->m_pNext = NULL; - pNode->m_pPrev = NULL; - - m_iCount--; - } - - int GetLength () const - { - return m_iCount; - } - - const ListNode_t * Begin () const - { - return m_tStub.m_pNext; - } - - const ListNode_t * End () const - { - return &m_tStub; - } - -private: - ListNode_t m_tStub; // stub node - int m_iCount; // elements counter -}; - - enum ThdState_e { THD_HANDSHAKE = 0, @@ -1957,17 +1892,23 @@ void SetSignalHandlers ( bool bAllowCtrlC=false ) const int WIN32_PIPE_BUFSIZE = 32; /// on Windows, the wrapper just prevents the warnings + +#pragma warning(push) // store current warning values +#pragma warning(disable:4127) // conditional expr is const +#pragma warning(disable:4389) // signed/unsigned mismatch + void sphFDSet ( int fd, fd_set * fdset ) { - #pragma warning(push) // store current warning values - #pragma warning(disable:4127) // conditional expr is const - #pragma warning(disable:4389) // signed/unsigned mismatch - FD_SET ( fd, fdset ); +} - #pragma warning(pop) // restore warnings +void sphFDClr ( int fd, fd_set * fdset ) +{ + FD_SET ( fd, fdset ); } +#pragma warning(pop) // restore warnings + #else // !USE_WINDOWS #define SPH_FDSET_OVERFLOW(_fd) ( (_fd)<0 || (_fd)>=(int)FD_SETSIZE ) @@ -1981,6 +1922,14 @@ void sphFDSet ( int fd, fd_set * set ) FD_SET ( fd, set ); } +void sphFDClr ( int fd, fd_set * set ) +{ + if ( SPH_FDSET_OVERFLOW( fd ) ) + sphFatal ( "sphFDClr() failed fd=%d, FD_SETSIZE=%d", fd, FD_SETSIZE ); + else + FD_CLR ( fd, set ); +} + #endif // USE_WINDOWS @@ -2341,27 +2290,11 @@ int sphSetSockNB ( int iSock ) #endif } -class CloseOnDestroy : public ISphNoncopyable -{ - int m_id; -public: - explicit CloseOnDestroy ( int id ) : m_id ( id ) {} - ~CloseOnDestroy() { close ( m_id ); } -}; - /// wait until socket is readable or writable int sphPoll ( int iSock, int64_t tmTimeout, bool bWrite=false ) { -#if HAVE_EPOLL - int eid = epoll_create ( 1 ); - CloseOnDestroy dEid ( eid ); - epoll_event dEvent; - dEvent.events = bWrite ? EPOLLOUT : EPOLLIN; - dEvent.data.ptr = NULL; // to keep leak checker silent - epoll_ctl ( eid, EPOLL_CTL_ADD, iSock, &dEvent ); - // do poll - return ::epoll_wait ( eid, &dEvent, 1, int ( tmTimeout/1000 ) ); -#elif HAVE_POLL + // don't need any epoll/kqueue here, since we check only 1 socket +#if HAVE_POLL struct pollfd pfd; pfd.fd = iSock; pfd.events = bWrite ? POLLOUT : POLLIN; @@ -2380,50 +2313,6 @@ int sphPoll ( int iSock, int64_t tmTimeout, bool bWrite=false ) #endif } -/// check if a socket is still connected -bool sphSockEof ( int iSock ) -{ - if ( iSock<0 ) - return true; - - char cBuf; -#if HAVE_EPOLL - int eid = epoll_create ( 1 ); - CloseOnDestroy dEid ( eid ); - epoll_event dEvent; - dEvent.events = EPOLLPRI | EPOLLIN; - dEvent.data.ptr = NULL; // to keep leak checker silent - epoll_ctl ( eid, EPOLL_CTL_ADD, iSock, &dEvent ); - if ( ::epoll_wait ( eid, &dEvent, 1, 0 )<0 ) - return true; - - if ( dEvent.events & (EPOLLPRI|EPOLLIN) ) -#elif HAVE_POLL - struct pollfd pfd; - pfd.fd = iSock; - pfd.events = POLLPRI | POLLIN; - if ( ::poll ( &pfd, 1, 0 )<0 ) - return true; - - if ( pfd.revents & (POLLIN|POLLPRI) ) -#else - fd_set fdrSet, fdeSet; - FD_ZERO ( &fdrSet ); - FD_ZERO ( &fdeSet ); - sphFDSet ( iSock, &fdrSet ); - sphFDSet ( iSock, &fdeSet ); - struct timeval tv = {0}; - if ( ::select ( iSock+1, &fdrSet, NULL, &fdeSet, &tv )<0 ) - return true; - - if ( FD_ISSET ( iSock, &fdrSet ) || FD_ISSET ( iSock, &fdeSet ) ) -#endif - if ( ::recv ( iSock, &cBuf, sizeof(cBuf), MSG_PEEK )<=0 ) - if ( sphSockGetErrno()!=EWOULDBLOCK ) - return true; - return false; -} - int sphSockRead ( int iSock, void * buf, int iLen, int iReadTimeout, bool bIntr ) { @@ -19837,327 +19726,119 @@ struct EventsIterator_t }; -#if HAVE_POLL -class CSphEventsPoll +class NetActionsPoller { private: - CSphVector m_dWork; - int m_iIter; EventsIterator_t m_tIter; + ISphNetEvents* m_pPoll; - CSphVector m_dEvents; - int m_iLastReportedErrno; - int m_iReady; - -public: - CSphEventsPoll () - : m_iIter ( -1 ) - , m_iLastReportedErrno ( -1 ) - , m_iReady ( 0 ) + inline DWORD TranslateEvents ( DWORD uNetEvents ) { - m_dWork.Reserve ( 1000 ); - m_tIter.m_pWork = NULL; - m_tIter.m_uEvents = 0; - } - - ~CSphEventsPoll() - { - ARRAY_FOREACH ( i, m_dWork ) - SafeDelete ( m_dWork[i] ); - } - - void SetupEvent ( ISphNetAction * pWork, int64_t tmNow ) - { - assert ( pWork && pWork->m_iSock>=0 ); - - NetEvent_e eSetup = pWork->Setup ( tmNow ); - assert ( eSetup==NE_IN || eSetup==NE_OUT ); - - pollfd tEvent; - tEvent.fd = pWork->m_iSock; - tEvent.events = ( eSetup==NE_IN ? POLLIN : POLLOUT ); - - m_dEvents.Add ( tEvent ); - m_dWork.Add ( pWork ); - } - - bool Wait ( int timeoutMs ) - { - // need positive timeout for communicate threads back and shutdown - m_iReady = ::poll ( m_dEvents.Begin(), m_dEvents.GetLength(), timeoutMs ); - - if ( m_iReady<0 ) - { - int iErrno = sphSockGetErrno(); - // common recoverable errors - if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) - return false; - - if ( m_iLastReportedErrno!=iErrno ) - { - sphWarning ( "poll tick failed: %s", sphSockError(iErrno) ); - m_iLastReportedErrno = iErrno; - } - return false; - } - - return ( m_iReady>0 ); - } - - bool IterateNextAll () - { - assert ( m_dEvents.GetLength()==m_dWork.GetLength() ); - - m_iIter++; - m_tIter.m_pWork = ( m_iIter=m_dEvents.GetLength() ) - return false; - - for ( ;; ) - { - m_iIter++; - if ( m_iIter>=m_dEvents.GetLength() ) - return false; - - if ( m_dEvents[m_iIter].revents==0 ) - continue; - - m_iReady--; - - m_tIter.m_pWork = m_dWork[m_iIter]; - m_tIter.m_uEvents = 0; - - pollfd & tEv = m_dEvents[m_iIter]; - if ( tEv.revents & POLLIN ) - m_tIter.m_uEvents |= NE_IN; - if ( tEv.revents & POLLOUT ) - m_tIter.m_uEvents |= NE_OUT; - if ( tEv.revents & POLLHUP ) - m_tIter.m_uEvents |= NE_HUP; - if ( tEv.revents & POLLERR ) - m_tIter.m_uEvents |= NE_ERR; - - tEv.revents = 0; - - return true; - } - } - - void IterateChangeEvent ( NetEvent_e eEv, ISphNetAction * ) - { - assert ( m_iIter>=0 && m_iIter=0 && m_iIter=0 && m_iIter m_dReady; - int m_iLastReportedErrno; - int m_iReady; - int m_iEFD; - EventsIterator_t m_tIter; - int m_iIterEv; - public: - CSphEventsEpoll () - : m_iLastReportedErrno ( -1 ) - , m_iReady ( 0 ) + NetActionsPoller () { - m_iEFD = epoll_create ( 1000 ); - if ( m_iEFD==-1 ) - sphDie ( "failed to create epoll main FD, errno=%d, %s", errno, strerror(errno) ); - - m_dReady.Reserve ( 1000 ); m_tIter.m_pWork = NULL; m_tIter.m_uEvents = 0; - m_iIterEv = -1; + m_pPoll = sphCreatePoll ( 1000 ); } - ~CSphEventsEpoll() + ~NetActionsPoller() { - const ListNode_t * pIt = m_tWork.Begin(); - while ( pIt!=m_tWork.End() ) + if (m_pPoll) { - ISphNetAction * pWork = (ISphNetAction *)pIt; - pIt = pIt->m_pNext; - SafeDelete ( pWork ); + m_pPoll->IterateStart (); + while (m_pPoll->IterateNextAll ()) + { + ISphNetAction * pWork = (ISphNetAction *)m_pPoll->IterateGet ().m_pData; + SafeDelete (pWork); + } } - - SafeClose ( m_iEFD ); + SafeDelete ( m_pPoll ); } void SetupEvent ( ISphNetAction * pWork, int64_t tmNow ) { + assert ( m_pPoll ); assert ( pWork && pWork->m_iSock>=0 ); NetEvent_e eSetup = pWork->Setup ( tmNow ); assert ( eSetup==NE_IN || eSetup==NE_OUT ); - m_tWork.Add ( pWork ); - - epoll_event tEv; - tEv.data.ptr = pWork; - tEv.events = ( eSetup==NE_IN ? EPOLLIN : EPOLLOUT ); - - sphLogDebugv ( "%p epoll setup, ev=0x%u, sock=%d", pWork, tEv.events, pWork->m_iSock ); - - int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_ADD, pWork->m_iSock, &tEv ); - if ( iRes==-1 ) - sphWarning ( "failed to setup epoll event for sock %d, errno=%d, %s", pWork->m_iSock, errno, strerror(errno) ); + m_pPoll->SetupEvent ( pWork->m_iSock, + ( eSetup==NE_IN ? ISphNetEvents::SPH_POLL_RD : ISphNetEvents::SPH_POLL_WR ), pWork ); } bool Wait ( int timeoutMs ) { - m_dReady.Resize ( m_tWork.GetLength() ); - // need positive timeout for communicate threads back and shutdown - m_iReady = epoll_wait ( m_iEFD, m_dReady.Begin(), m_dReady.GetLength(), timeoutMs ); - - if ( m_iReady<0 ) - { - int iErrno = sphSockGetErrno(); - // common recoverable errors - if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) - return false; - - if ( m_iLastReportedErrno!=iErrno ) - { - sphWarning ( "poll tick failed: %s", sphSockError(iErrno) ); - m_iLastReportedErrno = iErrno; - } - return false; - } - - return ( m_iReady>0 ); + assert ( m_pPoll ); + return m_pPoll->Wait ( timeoutMs ); } bool IterateNextAll () { - if ( !m_tIter.m_pWork ) - { - if ( m_tWork.Begin()==m_tWork.End() ) - return false; - - m_tIter.m_pWork = (ISphNetAction *)m_tWork.Begin(); - return true; - } else + assert ( m_pPoll ); + m_tIter.m_pWork = nullptr; + m_tIter.m_uEvents = 0; + bool bRes = m_pPoll->IterateNextAll (); + if ( bRes ) { - m_tIter.m_pWork = (ISphNetAction *)m_tIter.m_pWork->m_pNext; - if ( m_tIter.m_pWork!=m_tWork.End() ) - return true; - - m_tIter.m_pWork = NULL; - return false; + const NetEventsIterator_t& tBackendIterator = m_pPoll->IterateGet (); + m_tIter.m_pWork = (ISphNetAction*) tBackendIterator.m_pData; + m_tIter.m_uEvents = TranslateEvents( tBackendIterator.m_uEvents ); } + return bRes; } bool IterateNextReady () { - m_iIterEv++; - if ( m_iReady<=0 || m_iIterEv>=m_iReady ) + assert ( m_pPoll ); + m_tIter.m_pWork = nullptr; + m_tIter.m_uEvents = 0; + bool bRes = m_pPoll->IterateNextReady (); + if ( bRes ) { - m_tIter.m_pWork = NULL; - m_tIter.m_uEvents = 0; - return false; + const NetEventsIterator_t& tBackendIterator = m_pPoll->IterateGet (); + m_tIter.m_pWork = (ISphNetAction*) tBackendIterator.m_pData; + m_tIter.m_uEvents = TranslateEvents ( tBackendIterator.m_uEvents ); } - - const epoll_event & tEv = m_dReady[m_iIterEv]; - - m_tIter.m_pWork = (ISphNetAction *)tEv.data.ptr; - m_tIter.m_uEvents = 0; - - if ( tEv.events & EPOLLIN ) - m_tIter.m_uEvents |= NE_IN; - if ( tEv.events & EPOLLOUT ) - m_tIter.m_uEvents |= NE_OUT; - if ( tEv.events & EPOLLHUP ) - m_tIter.m_uEvents |= NE_HUP; - if ( tEv.events & EPOLLERR ) - m_tIter.m_uEvents |= NE_ERR; - - return true; + return bRes; } - void IterateChangeEvent ( NetEvent_e eEv, ISphNetAction * pWork ) + void IterateChangeEvent ( NetEvent_e eEv, ISphNetAction * pAction) { - epoll_event tEv; - tEv.data.ptr = pWork; - tEv.events = ( ( eEv & NE_IN )!=0 ? EPOLLIN : EPOLLOUT ); - - sphLogDebugv ( "%p epoll change, ev=0x%u, sock=%d", pWork, tEv.events, pWork->m_iSock ); - - int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_MOD, pWork->m_iSock, &tEv ); - if ( iRes==-1 ) - sphWarning ( "failed to modify epoll event for sock %d, errno=%d, %s", pWork->m_iSock, errno, strerror(errno) ); + assert ( m_pPoll ); + m_pPoll->IterateChangeEvent ( pAction->m_iSock, ( eEv==NE_IN ? ISphNetEvents::SPH_POLL_RD : ISphNetEvents::SPH_POLL_WR ) ); } void IterateRemove () { - sphLogDebugv ( "%p epoll remove, ev=0x%u, sock=%d", m_tIter.m_pWork, m_tIter.m_uEvents, m_tIter.m_pWork->m_iSock ); - assert ( m_tIter.m_pWork ); - - epoll_event tEv; - int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_DEL, m_tIter.m_pWork->m_iSock, &tEv ); + assert ( m_pPoll ); + const NetEventsIterator_t& tBackendIterator = m_pPoll->IterateGet (); + ISphNetAction* pAction = (ISphNetAction*) tBackendIterator.m_pData; - // might be already closed by worker from thread pool - if ( iRes==-1 ) - sphLogDebugv ( "failed to remove epoll event for sock %d(%p), errno=%d, %s", m_tIter.m_pWork->m_iSock, m_tIter.m_pWork, errno, strerror(errno) ); - - ISphNetAction * pPrev = (ISphNetAction *)m_tIter.m_pWork->m_pPrev; - m_tWork.Remove ( m_tIter.m_pWork ); - m_tIter.m_pWork = pPrev; - // SafeDelete ( m_tIter.m_pWork ); + m_pPoll->IterateRemove ( pAction->m_iSock ); + m_tIter.m_pWork = NULL; } int IterateStart () { + assert ( m_pPoll ); + m_tIter.m_pWork = NULL; m_tIter.m_uEvents = 0; - m_iIterEv = -1; - return m_iReady; + return m_pPoll->IterateStart (); } EventsIterator_t & IterateGet () @@ -20165,23 +19846,6 @@ class CSphEventsEpoll return m_tIter; } }; -#endif - -class CSphEventsDummy -{ - EventsIterator_t m_tIter; - -public: - void SetupEvent ( ISphNetAction * , int64_t ) {} - bool Wait ( int ) { return false; } // NOLINT - bool IterateNextAll () { return false; } - bool IterateNextReady () { return false; } - void IterateChangeEvent ( NetEvent_e , ISphNetAction * ) {} - void IterateRemove () {} - int IterateStart () { return 0; } - EventsIterator_t & IterateGet () { return m_tIter; } -}; - // event that wakes-up poll net loop from finished thread pool job class CSphWakeupEvent : public ISphNetAction @@ -20558,13 +20222,7 @@ class CSphNetLoop CSphWakeupEvent * m_pWakeupExternal; // FIXME!!! owned\deleted by event loop CSphMutex m_tExtLock; LoopProfiler_t m_tPrf; -#if HAVE_EPOLL - CSphEventsEpoll m_tEvents; -#elif HAVE_POLL - CSphEventsPoll m_tEvents; -#else - CSphEventsDummy m_tEvents; -#endif + NetActionsPoller m_tPoller; explicit CSphNetLoop ( CSphVector & dListeners ) { @@ -20572,7 +20230,7 @@ class CSphNetLoop ARRAY_FOREACH ( i, dListeners ) { NetActionAccept_t * pCur = new NetActionAccept_t ( dListeners[i] ); - m_tEvents.SetupEvent ( pCur, tmNow ); + m_tPoller.SetupEvent ( pCur, tmNow ); } int iRead = -1; @@ -20590,7 +20248,7 @@ class CSphNetLoop if ( iRead>=0 && iWrite>=0 ) { m_pWakeupExternal = new CSphWakeupEvent ( iRead, iWrite ); - m_tEvents.SetupEvent ( m_pWakeupExternal, tmNow ); + m_tPoller.SetupEvent ( m_pWakeupExternal, tmNow ); } m_bGotExternal = false; @@ -20629,7 +20287,7 @@ class CSphNetLoop m_tPrf.StartPoll(); // need positive timeout for communicate threads back and shutdown - bool bGot = m_tEvents.Wait ( iSpinWait ); + bool bGot = m_tPoller.Wait ( iSpinWait ); m_tPrf.EndTask(); m_uTick++; @@ -20659,18 +20317,18 @@ class CSphNetLoop // handle events and collect stats m_tPrf.StartTick(); - int iGotEvents = m_tEvents.IterateStart(); + int iGotEvents = m_tPoller.IterateStart(); sphLogDebugv ( "got events=%d, tick=%u", iGotEvents, m_uTick ); int iConnections = 0; int iMaxIters = 0; - while ( m_tEvents.IterateNextReady() && ( !g_iThrottleAction || iMaxItersTick ( m_tEvents.IterateGet().m_uEvents, dWorkNext, this ); + NetEvent_e eEv = pWork->Tick ( m_tPoller.IterateGet().m_uEvents, dWorkNext, this ); pWork->GetStats ( iConnections ); m_tPrf.m_iPerfEv++; iMaxIters++; @@ -20689,7 +20347,7 @@ class CSphNetLoop dCleanup.Add ( pWork ); } else { - m_tEvents.IterateChangeEvent ( eEv, pWork ); + m_tPoller.IterateChangeEvent ( eEv, pWork ); } m_tPrf.EndTask(); m_tPrf.EndTask(); @@ -20705,7 +20363,7 @@ class CSphNetLoop ARRAY_FOREACH ( i, dWorkNext ) { ISphNetAction * pWork = dWorkNext[i]; - m_tEvents.SetupEvent ( pWork, tmNow ); + m_tPoller.SetupEvent ( pWork, tmNow ); if ( pWork->m_tmTimeout ) tmNextCheck = Min ( tmNextCheck, pWork->m_tmTimeout ); } @@ -20736,11 +20394,11 @@ class CSphNetLoop // remove outdated items on no signals tmNextCheck = INT64_MAX; - m_tEvents.IterateStart(); - while ( m_tEvents.IterateNextAll() ) + m_tPoller.IterateStart(); + while ( m_tPoller.IterateNextAll() ) { - assert ( m_tEvents.IterateGet().m_pWork ); - ISphNetAction * pWork = m_tEvents.IterateGet().m_pWork; + assert ( m_tPoller.IterateGet().m_pWork ); + ISphNetAction * pWork = m_tPoller.IterateGet().m_pWork; if ( !pWork->m_tmTimeout ) continue; @@ -20751,7 +20409,7 @@ class CSphNetLoop } sphLogDebugv ( "%p bailing on timeout no signal, sock=%d", pWork, pWork->m_iSock ); - m_tEvents.IterateRemove (); + m_tPoller.IterateRemove (); // SafeDelete ( pWork ); // close socket immediately to prevent write by client into persist connection that just timed out // that does not need in case Work got removed at IterateRemove + SafeDelete ( pWork ); @@ -20790,7 +20448,7 @@ class CSphNetLoop void RemoveIterEvent () { - m_tEvents.IterateRemove(); + m_tPoller.IterateRemove(); } // main thread wrapper diff --git a/src/searchdaemon.h b/src/searchdaemon.h index 608b0625d..6ac5aadcd 100644 --- a/src/searchdaemon.h +++ b/src/searchdaemon.h @@ -58,6 +58,7 @@ #undef EINPROGRESS #undef ECONNRESET #undef ECONNABORTED + #undef EAGAIN #define LOCK_EX 0 #define LOCK_UN 1 #define STDIN_FILENO fileno(stdin) @@ -70,6 +71,7 @@ #define ECONNRESET WSAECONNRESET #define ECONNABORTED WSAECONNABORTED #define ESHUTDOWN WSAESHUTDOWN + #define EAGAIN WSATRY_AGAIN #define socklen_t int #define ftruncate _chsize @@ -102,8 +104,8 @@ int sphSockGetErrno (); void sphSockSetErrno ( int ); int sphSockPeekErrno (); int sphSetSockNB ( int ); -bool sphSockEof ( int ); void sphFDSet ( int fd, fd_set * fdset ); +void sphFDClr ( int fd, fd_set * fdset ); DWORD sphGetAddress ( const char * sHost, bool bFatal=false ); ///////////////////////////////////////////////////////////////////////////// diff --git a/src/searchdha.cpp b/src/searchdha.cpp index 09043ac39..a62baa512 100644 --- a/src/searchdha.cpp +++ b/src/searchdha.cpp @@ -1225,7 +1225,7 @@ void RemoteConnectToAgent ( AgentConn_t & tAgent ) if ( tAgent.m_iSock>=0 ) // already connected { - if ( !sphSockEof ( tAgent.m_iSock ) ) + if ( !sphNBSockEof ( tAgent.m_iSock ) ) { tAgent.m_eState = AGENT_ESTABLISHED; tAgent.m_iStartQuery = sphMicroTimer(); @@ -1328,8 +1328,6 @@ void RemoteConnectToAgent ( AgentConn_t & tAgent ) } } -#if HAVE_EPOLL -// copy-pasted version with epoll; plain version below // process states AGENT_CONNECTING, AGENT_HANDSHAKE, AGENT_ESTABLISHED and notes AGENT_QUERYED // called in serial order with RemoteConnectToAgents (so, the context is NOT changed during the call). int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) @@ -1341,9 +1339,7 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) int iAgents = 0; int64_t tmMaxTimer = sphMicroTimer() + pCtx->m_iTimeout*1000; // in microseconds - int eid = epoll_create ( pCtx->m_iAgentCount ); - CSphVector dEvents ( pCtx->m_iAgentCount ); - epoll_event dEvent; + ISphNetEvents* pEvents = sphCreatePoll ( pCtx->m_iAgentCount, true ); int iEvents = 0; bool bTimeout = false; @@ -1366,9 +1362,9 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) // tAgent.m_eState = AGENT_RETRY; // do retry on connect() failures // commented out since already set by Fail continue; } - dEvent.events = ( tAgent.m_eState==AGENT_CONNECTING || tAgent.m_eState==AGENT_ESTABLISHED ) ? EPOLLOUT : EPOLLIN; - dEvent.data.ptr = &tAgent; - epoll_ctl ( eid, EPOLL_CTL_ADD, tAgent.m_iSock, &dEvent ); + pEvents->SetupEvent( tAgent.m_iSock, (tAgent.m_eState==AGENT_CONNECTING + || tAgent.m_eState==AGENT_ESTABLISHED) + ? ISphNetEvents::SPH_POLL_WR : ISphNetEvents::SPH_POLL_RD, &tAgent); ++iEvents; } } @@ -1397,27 +1393,28 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) // do poll - int iSelected = ::epoll_wait ( eid, dEvents.Begin(), dEvents.GetLength(), int( tmMicroLeft/1000 ) ); + bool bHaveSelected = pEvents->Wait( int( tmMicroLeft/1000 ) ); // update counters, and loop again if nothing happened pCtx->m_pAgents->m_iWaited += sphMicroTimer() - tmSelect; // todo: do we need to check for EINTR here? Or the fact of timeout is enough anyway? - if ( iSelected<=0 ) + if ( !bHaveSelected ) continue; + pEvents->IterateStart(); + // ok, something did happen, so loop the agents and do them checks - for ( int i=0; iIterateNextReady() ) { - AgentConn_t & tAgent = *(AgentConn_t*)dEvents[i].data.ptr; - bool bReadable = ( dEvents[i].events & EPOLLIN )!=0; - bool bWriteable = ( dEvents[i].events & EPOLLOUT )!=0; - bool bErr = ( ( dEvents[i].events & ( EPOLLERR | EPOLLHUP ) )!=0 ); + NetEventsIterator_t & tEvent = pEvents->IterateGet (); + AgentConn_t & tAgent = *(AgentConn_t*)tEvent.m_pData; + bool bErr = ( (tEvent.m_uEvents & (ISphNetEvents::SPH_POLL_ERR | ISphNetEvents::SPH_POLL_HUP ) )!=0 ); - if ( tAgent.m_eState==AGENT_CONNECTING && ( bWriteable || bErr ) ) + if ( tAgent.m_eState==AGENT_CONNECTING && ( tEvent.m_bWritable || bErr ) ) { if ( bErr ) { - epoll_ctl ( eid, EPOLL_CTL_DEL, tAgent.m_iSock, &dEvent ); + pEvents->IterateRemove (tAgent.m_iSock); --iEvents; int iErr = 0; @@ -1435,24 +1432,21 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) NetOutputBuffer_c tOut ( tAgent.m_iSock ); tOut.SendDword ( SPHINX_CLIENT_VERSION ); tOut.Flush (); // FIXME! handle flush failure? - dEvent.events = EPOLLIN; - dEvent.data.ptr = &tAgent; - epoll_ctl ( eid, EPOLL_CTL_MOD, tAgent.m_iSock, &dEvent ); - + pEvents->IterateChangeEvent ( tAgent.m_iSock, ISphNetEvents::SPH_POLL_RD ); tAgent.m_eState = AGENT_HANDSHAKE; continue; } // check if hello was received - if ( tAgent.m_eState==AGENT_HANDSHAKE && bReadable ) + if ( tAgent.m_eState==AGENT_HANDSHAKE && tEvent.m_bReadable ) { // read reply int iRemoteVer; int iRes = sphSockRecv ( tAgent.m_iSock, (char*)&iRemoteVer, sizeof(iRemoteVer) ); if ( iRes!=sizeof(iRemoteVer) ) { - epoll_ctl ( eid, EPOLL_CTL_DEL, tAgent.m_iSock, &dEvent ); + pEvents->IterateRemove (tAgent.m_iSock); --iEvents; if ( iRes<0 ) { @@ -1498,13 +1492,11 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) } tAgent.m_eState = AGENT_ESTABLISHED; - dEvent.events = EPOLLOUT; - dEvent.data.ptr = &tAgent; - epoll_ctl ( eid, EPOLL_CTL_MOD, tAgent.m_iSock, &dEvent ); + pEvents->IterateChangeEvent ( tAgent.m_iSock, ISphNetEvents::SPH_POLL_WR ); continue; } - if ( tAgent.m_eState==AGENT_ESTABLISHED && bWriteable ) + if ( tAgent.m_eState==AGENT_ESTABLISHED && tEvent.m_bWritable ) { // send request NetOutputBuffer_c tOut ( tAgent.m_iSock ); @@ -1516,26 +1508,24 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) continue; } tAgent.m_eState = AGENT_QUERYED; - iAgents++; - dEvent.events = EPOLLIN; - dEvent.data.ptr = &tAgent; - epoll_ctl ( eid, EPOLL_CTL_MOD, tAgent.m_iSock, &dEvent ); + ++iAgents; + pEvents->IterateChangeEvent ( tAgent.m_iSock, ISphNetEvents::SPH_POLL_RD ); continue; } // check if queried agent replied while we were querying others - if ( tAgent.m_eState==AGENT_QUERYED && bReadable ) + if ( tAgent.m_eState==AGENT_QUERYED && tEvent.m_bReadable ) { // do not account agent wall time from here; agent is probably ready tAgent.m_iWall += sphMicroTimer(); tAgent.m_eState = AGENT_PREREPLY; - epoll_ctl ( eid, EPOLL_CTL_DEL, tAgent.m_iSock, &dEvent ); + pEvents->IterateRemove(tAgent.m_iSock); --iEvents; continue; } } } - close ( eid ); + SafeDelete (pEvents); // check if connection timed out for ( int i=0; im_iAgentCount; i++ ) @@ -1557,9 +1547,8 @@ int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) return iAgents; } -// epoll version. Plain version below // processing states AGENT_QUERY, AGENT_PREREPLY and AGENT_REPLY -// may work in parallel with RemoteQueryAgents, so the state MAY change duirng a call. +// may work in parallel with RemoteQueryAgents, so the state MAY change during a call. int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IReplyParser_t & tParser ) { assert ( iTimeout>=0 ); @@ -1567,9 +1556,7 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl int iAgents = 0; int64_t tmMaxTimer = sphMicroTimer() + iTimeout*1000; // in microseconds - int eid = epoll_create ( dAgents.GetLength() ); - CSphVector dEvents ( dAgents.GetLength() ); - epoll_event dEvent; + ISphNetEvents * pEvents = sphCreatePoll ( dAgents.GetLength (), true ); int iEvents = 0; bool bTimeout = false; @@ -1588,9 +1575,7 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl { assert ( !tAgent.m_sPath.IsEmpty() || tAgent.m_iPort>0 ); assert ( tAgent.m_iSock>0 ); - dEvent.events = EPOLLIN; - dEvent.data.ptr = &tAgent; - epoll_ctl ( eid, EPOLL_CTL_ADD, tAgent.m_iSock, &dEvent ); + pEvents->SetupEvent ( tAgent.m_iSock, ISphNetEvents::SPH_POLL_RD, &tAgent); ++iEvents; bDone = false; } @@ -1600,8 +1585,6 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl break; } - - int64_t tmSelect = sphMicroTimer(); int64_t tmMicroLeft = tmMaxTimer - tmSelect; if ( tmMicroLeft<=0 ) // FIXME? what about iTimeout==0 case? @@ -1610,21 +1593,23 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl break; } - int iSelected = ::epoll_wait ( eid, dEvents.Begin(), dEvents.GetLength(), int( tmMicroLeft/1000 ) ); + bool bHaveAnswered = pEvents->Wait( int( tmMicroLeft/1000 ) ); dAgents.Begin()->m_iWaited += sphMicroTimer() - tmSelect; - if ( iSelected<=0 ) + if ( !bHaveAnswered ) continue; - for ( int i=0; iIterateStart (); + while ( pEvents->IterateNextReady () ) { - AgentConn_t & tAgent = *(AgentConn_t*)dEvents[i].data.ptr; + NetEventsIterator_t &tEvent = pEvents->IterateGet (); + AgentConn_t & tAgent = *(AgentConn_t*) tEvent.m_pData; if ( tAgent.m_bBlackhole ) continue; if (!( tAgent.m_eState==AGENT_QUERYED || tAgent.m_eState==AGENT_REPLY || tAgent.m_eState==AGENT_PREREPLY )) continue; - if (!( dEvents[i].events & EPOLLIN )) + if (!(tEvent.m_bReadable )) continue; // if there was no reply yet, read reply header @@ -1746,7 +1731,7 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl break; } - epoll_ctl ( eid, EPOLL_CTL_DEL, tAgent.m_iSock, &dEvent ); + pEvents->IterateRemove ( tAgent.m_iSock ); --iEvents; // all is well iAgents++; @@ -1760,7 +1745,7 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl if ( bFailure ) { - epoll_ctl ( eid, EPOLL_CTL_DEL, tAgent.m_iSock, &dEvent ); + pEvents->IterateRemove ( tAgent.m_iSock ); --iEvents; tAgent.Close (); tAgent.m_dResults.Reset (); @@ -1773,7 +1758,7 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl } } - close ( eid ); + SafeDelete(pEvents); // close timed-out agents ARRAY_FOREACH ( iAgent, dAgents ) @@ -1793,951 +1778,1221 @@ int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IRepl return iAgents; } -#else // !HAVE_EPOLL +struct AgentWorkContext_t; +typedef void ( *ThdWorker_fn ) ( AgentWorkContext_t * ); -// process states AGENT_CONNECTING, AGENT_HANDSHAKE, AGENT_ESTABLISHED and notes AGENT_QUERYED -// called in serial order with RemoteConnectToAgents (so, the context is NOT changed during the call). -int RemoteQueryAgents ( AgentConnectionContext_t * pCtx ) +struct AgentWorkContext_t : public AgentConnectionContext_t { - assert ( pCtx->m_iTimeout>=0 ); - assert ( pCtx->m_pAgents ); - assert ( pCtx->m_iAgentCount ); - - int iAgents = 0; - int64_t tmMaxTimer = sphMicroTimer() + pCtx->m_iTimeout*1000; // in microseconds - CSphVector dWorkingSet; - dWorkingSet.Reserve ( pCtx->m_iAgentCount ); - -#if HAVE_POLL - CSphVector fds; - fds.Reserve ( pCtx->m_iAgentCount ); -#endif - - // main connection loop - // break if a) all connects in AGENT_QUERY state, or b) timeout - for ( ;; ) - { - // prepare socket sets - dWorkingSet.Reset(); -#if HAVE_POLL - fds.Reset(); -#else - int iMax = 0; - fd_set fdsRead, fdsWrite; - FD_ZERO ( &fdsRead ); - FD_ZERO ( &fdsWrite ); -#endif + ThdWorker_fn m_pfn; ///< work functor & flag of dummy element + int64_t m_tmWait; + int m_iAgentsDone; - bool bDone = true; - for ( int i=0; im_iAgentCount; i++ ) - { - AgentConn_t & tAgent = pCtx->m_pAgents[i]; - // select only 'initial' agents - which are not send query response. - if ( tAgent.m_eStateAGENT_QUERYED ) - continue; + AgentWorkContext_t () + : m_pfn ( NULL ) + , m_tmWait ( 0 ) + , m_iAgentsDone ( 0 ) + {} +}; - assert ( !tAgent.m_sPath.IsEmpty() || tAgent.m_iPort>0 ); - assert ( tAgent.m_iSock>0 ); - if ( tAgent.m_iSock<=0 || ( tAgent.m_sPath.IsEmpty() && tAgent.m_iPort<=0 ) ) - { - tAgent.Fail ( eConnectFailures, "invalid agent in querying. Socket %d, Path %s, Port %d", - tAgent.m_iSock, tAgent.m_sPath.cstr(), tAgent.m_iPort ); - // tAgent.m_eState = AGENT_RETRY; // do retry on connect() failures // commented out since done by Fail() - continue; - } - bool bWr = ( tAgent.m_eState==AGENT_CONNECTING || tAgent.m_eState==AGENT_ESTABLISHED ); - dWorkingSet.Add(i); -#if HAVE_POLL - pollfd& pfd = fds.Add(); - pfd.fd = tAgent.m_iSock; - pfd.events = bWr ? POLLOUT : POLLIN; -#else - sphFDSet ( tAgent.m_iSock, bWr ? &fdsWrite : &fdsRead ); - iMax = Max ( iMax, tAgent.m_iSock ); -#endif - if ( tAgent.m_eState!=AGENT_QUERYED ) - bDone = false; - } +class ThdWorkPool_c : ISphNoncopyable +{ +private: + CSphMutex m_tDataLock; + CSphMutex m_tStatLock; +public: + CSphAutoEvent m_tChanged; + CSphAtomic m_iActiveThreads; - if ( bDone ) - break; +private: + AgentWorkContext_t * m_dData; // works array + int m_iLen; - // compute timeout - int64_t tmSelect = sphMicroTimer(); - int64_t tmMicroLeft = tmMaxTimer - tmSelect; - if ( tmMicroLeft<=0 ) - break; // FIXME? what about iTimeout==0 case? + int m_iHead; // ring buffer begin + int m_iTail; // ring buffer end - // do poll -#if HAVE_POLL - int iSelected = ::poll ( fds.Begin(), fds.GetLength(), int( tmMicroLeft/1000 ) ); -#else - struct timeval tvTimeout; - tvTimeout.tv_sec = (int)( tmMicroLeft/ 1000000 ); // full seconds - tvTimeout.tv_usec = (int)( tmMicroLeft % 1000000 ); // microseconds - int iSelected = ::select ( 1+iMax, &fdsRead, &fdsWrite, NULL, &tvTimeout ); // exceptfds are OOB only -#endif + CSphAtomic m_iWorksCount; // count of works to be done + volatile int m_iAgentsDone; // count of agents that finished their works + volatile int m_iAgentsReported; // count of agents that reported of their work done + volatile bool m_bIsDestroying; // help to keep at least 1 worker thread active - // update counters, and loop again if nothing happened - pCtx->m_pAgents->m_iWaited += sphMicroTimer() - tmSelect; - // todo: do we need to check for EINTR here? Or the fact of timeout is enough anyway? - if ( iSelected<=0 ) - continue; + CrashQuery_t m_tCrashQuery; // query that got reported on crash - // ok, something did happen, so loop the agents and do them checks - ARRAY_FOREACH ( i, dWorkingSet ) - { - AgentConn_t & tAgent = pCtx->m_pAgents[dWorkingSet[i]]; +public: + explicit ThdWorkPool_c ( int iLen ); + ~ThdWorkPool_c (); -#if HAVE_POLL - bool bReadable = ( fds[i].revents & POLLIN )!=0; - bool bWriteable = ( fds[i].revents & POLLOUT )!=0; - bool bErr = ( ( fds[i].revents & ( POLLERR | POLLHUP ) )!=0 ); -#else - bool bReadable = FD_ISSET ( tAgent.m_iSock, &fdsRead )!=0; - bool bWriteable = FD_ISSET ( tAgent.m_iSock, &fdsWrite )!=0; - bool bErr = !bWriteable; // just poll and check for error -#endif - if ( tAgent.m_eState==AGENT_CONNECTING && ( bWriteable || bErr ) ) - { - if ( bErr ) - { - // check if connection completed - // tricky part, with select, we MUST use write-set ONLY here at this check - // even though we can't tell connect() success from just OS send buffer availability - // but any check involving read-set just never ever completes, so... - int iErr = 0; - socklen_t iErrLen = sizeof(iErr); - getsockopt ( tAgent.m_iSock, SOL_SOCKET, SO_ERROR, (char*)&iErr, &iErrLen ); - if ( iErr ) - { - // connect() failure - tAgent.Fail ( eConnectFailures, "connect() failed: errno=%d, %s", iErr, sphSockError(iErr) ); - } - continue; - } + AgentWorkContext_t Pop(); - assert ( bWriteable ); // should never get empty or readable state - // connect() success - track_processing_time ( tAgent ); + void Push ( const AgentWorkContext_t & tElem ); + void RawPush ( const AgentWorkContext_t & tElem ); + int GetReadyCount () const; + int FetchReadyCount (); // each call returns new portion of number - // send the client's proto version right now to avoid w-w-r pattern. - NetOutputBuffer_c tOut ( tAgent.m_iSock ); - tOut.SendDword ( SPHINX_CLIENT_VERSION ); - tOut.Flush (); // FIXME! handle flush failure? + int GetReadyTotal () const + { + return m_iAgentsDone; + } - tAgent.m_eState = AGENT_HANDSHAKE; - continue; - } + bool HasIncompleteWorks () const + { + return ( m_iWorksCount.GetValue ()>0 ); + } - // check if hello was received - if ( tAgent.m_eState==AGENT_HANDSHAKE && bReadable ) - { - // read reply - int iRemoteVer; - int iRes = sphSockRecv ( tAgent.m_iSock, (char*)&iRemoteVer, sizeof(iRemoteVer) ); - if ( iRes!=sizeof(iRemoteVer) ) - { - if ( iRes<0 ) - { - int iErr = sphSockGetErrno(); - tAgent.Fail ( eNetworkErrors, "handshake failure (errno=%d, msg=%s)", iErr, sphSockError(iErr) ); - } else if ( iRes>0 ) - { - // incomplete reply - tAgent.Fail ( eWrongReplies, "handshake failure (exp=%d, recv=%d)", (int)sizeof(iRemoteVer), iRes ); - } else - { - // agent closed the connection - // this might happen in out-of-sync connect-accept case; so let's retry - tAgent.Fail ( eUnexpectedClose, "handshake failure (connection was closed)" ); - tAgent.m_eState = AGENT_RETRY; - } - continue; - } + void SetWorksCount ( int iWorkers ) + { + m_iWorksCount.SetValue (iWorkers); + } - iRemoteVer = ntohl ( iRemoteVer ); - if (!( iRemoteVer==SPHINX_SEARCHD_PROTO || iRemoteVer==0x01000000UL ) ) // workaround for all the revisions that sent it in host order... - { - tAgent.Fail ( eWrongReplies, "handshake failure (unexpected protocol version=%d)", iRemoteVer ); - continue; - } + void AddWorksCount ( int iWorkers ) + { + m_iWorksCount.Add ( iWorkers ); + } - NetOutputBuffer_c tOut ( tAgent.m_iSock ); - // check if we need to reset the persistent connection - if ( tAgent.m_bFresh && tAgent.m_bPersistent ) - { - tOut.SendWord ( SEARCHD_COMMAND_PERSIST ); - tOut.SendWord ( 0 ); // dummy version - tOut.SendInt ( 4 ); // request body length - tOut.SendInt ( 1 ); // set persistent to 1. - tOut.Flush (); - tAgent.m_bFresh = false; - } + static void PoolThreadFunc ( void * pArg ); +}; - tAgent.m_eState = AGENT_ESTABLISHED; - continue; - } +ThdWorkPool_c::ThdWorkPool_c ( int iLen ) +{ + m_tCrashQuery = SphCrashLogger_c::GetQuery(); // transfer query info for crash logger to new thread - if ( tAgent.m_eState==AGENT_ESTABLISHED && bWriteable ) - { - // send request - NetOutputBuffer_c tOut ( tAgent.m_iSock ); - pCtx->m_pBuilder->BuildRequest ( tAgent, tOut ); - tOut.Flush (); // FIXME! handle flush failure? - tAgent.m_eState = AGENT_QUERYED; - iAgents++; - continue; - } + m_iLen = iLen+1; + m_iTail = m_iHead = 0; + m_iWorksCount.SetValue (0); + m_iAgentsDone = m_iAgentsReported = 0; + m_bIsDestroying = false; - // check if queried agent replied while we were querying others - if ( tAgent.m_eState==AGENT_QUERYED && bReadable ) - { - // do not account agent wall time from here; agent is probably ready - tAgent.m_iWall += sphMicroTimer(); - tAgent.m_eState = AGENT_PREREPLY; - continue; - } - } - } + m_dData = new AgentWorkContext_t[m_iLen]; +#ifndef NDEBUG + for ( int i=0; im_iAgentCount; i++ ) - { - AgentConn_t & tAgent = pCtx->m_pAgents[i]; - if ( tAgent.m_eState!=AGENT_QUERYED && tAgent.m_eState!=AGENT_UNUSED && tAgent.m_eState!=AGENT_RETRY && tAgent.m_eState!=AGENT_PREREPLY - && tAgent.m_eState!=AGENT_REPLY ) - { - // technically, we can end up here via two different routes - // a) connect() never finishes in given time frame - // b) agent actually accept()s the connection but keeps silence - // however, there's no way to tell the two from each other - // so we just account both cases as connect() failure - tAgent.Fail ( eTimeoutsConnect, "connect() timed out" ); - tAgent.m_eState = AGENT_RETRY; // do retry on connect() failures - } - } +ThdWorkPool_c::~ThdWorkPool_c () +{ - return iAgents; + m_bIsDestroying = true; + while ( m_iActiveThreads>0 ) + sphSleepMsec ( 1 ); + m_tChanged.Done(); + SafeDeleteArray ( m_dData ); } -// processing states AGENT_QUERY, AGENT_PREREPLY and AGENT_REPLY -int RemoteWaitForAgents ( CSphVector & dAgents, int iTimeout, IReplyParser_t & tParser ) +AgentWorkContext_t ThdWorkPool_c::Pop() { - assert ( iTimeout>=0 ); + AgentWorkContext_t tRes; + if ( m_iTail==m_iHead ) // quick path for empty pool + return tRes; - int iAgents = 0; - int64_t tmMaxTimer = sphMicroTimer() + iTimeout*1000; // in microseconds - bool bTimeout = false; + CSphScopedLock tData ( m_tDataLock ); // lock on create, unlock on destroy - CSphVector dWorkingSet; - dWorkingSet.Reserve ( dAgents.GetLength() ); + if ( m_iTail==m_iHead ) // it might be empty now as another thread could steal work till that moment + return tRes; -#if HAVE_POLL - CSphVector fds; - fds.Reserve ( dAgents.GetLength() ); + tRes = m_dData[m_iHead]; + assert ( tRes.m_pfn ); +#ifndef NDEBUG + m_dData[m_iHead] = AgentWorkContext_t(); // to make sure that we don't rewrite valid elements #endif + m_iHead = ( m_iHead+1 ) % m_iLen; - for ( ;; ) - { - dWorkingSet.Reset(); - -#if HAVE_POLL - fds.Reset(); -#else - int iMax = 0; - fd_set fdsRead; - FD_ZERO ( &fdsRead ); -#endif - bool bDone = true; - ARRAY_FOREACH ( iAgent, dAgents ) - { - AgentConn_t & tAgent = dAgents[iAgent]; - if ( tAgent.m_bBlackhole ) - continue; + return tRes; +} - if ( tAgent.m_eState==AGENT_QUERYED || tAgent.m_eState==AGENT_REPLY || tAgent.m_eState==AGENT_PREREPLY ) +void ThdWorkPool_c::Push ( const AgentWorkContext_t & tElem ) +{ + if ( !tElem.m_pfn ) + return; + + CSphScopedLock tData ( m_tDataLock ); + RawPush ( tElem ); +} + +void ThdWorkPool_c::RawPush ( const AgentWorkContext_t & tElem ) +{ + assert ( !m_dData[m_iTail].m_pfn ); // to make sure that we don't rewrite valid elements + m_dData[m_iTail] = tElem; + m_iTail = ( m_iTail+1 ) % m_iLen; +} + +int ThdWorkPool_c::FetchReadyCount () +{ + // it could be better to lock here to get accurate value of m_iAgentsDone + // however that make lock contention of 1 sec on 1000 query ~ total 3.2sec vs 2.2 sec ( trunk ) + int iNowDone = GetReadyCount(); + m_iAgentsReported += iNowDone; + return iNowDone; +} + +int ThdWorkPool_c::GetReadyCount () const +{ + // it could be better to lock here to get accurate value of m_iAgentsDone + // however that make lock contention of 1 sec on 1000 query ~ total 3.2sec vs 2.2 sec ( trunk ) + return m_iAgentsDone - m_iAgentsReported; +} + +void ThdWorkPool_c::PoolThreadFunc ( void * pArg ) +{ + ThdWorkPool_c * pPool = (ThdWorkPool_c *)pArg; + assert (pPool); + ++pPool->m_iActiveThreads; + sphLogDebugv ("Thread func started for %p, now %d threads active", pArg, (DWORD)pPool->m_iActiveThreads); + SphCrashLogger_c::SetLastQuery ( pPool->m_tCrashQuery ); + + int iSpinCount = 0; + int iPopCount = 0; + AgentWorkContext_t tNext; + for ( ;; ) + { + if ( !tNext.m_pfn ) // pop new work if current is done + { + iSpinCount = 0; + ++iPopCount; + tNext = pPool->Pop(); + if ( !tNext.m_pfn ) // if there is no work at queue - worker done { - assert ( !tAgent.m_sPath.IsEmpty() || tAgent.m_iPort>0 ); - assert ( tAgent.m_iSock>0 ); - dWorkingSet.Add(iAgent); -#if HAVE_POLL - pollfd & pfd = fds.Add(); - pfd.fd = tAgent.m_iSock; - pfd.events = POLLIN; -#else - sphFDSet ( tAgent.m_iSock, &fdsRead ); - iMax = Max ( iMax, tAgent.m_iSock ); -#endif - bDone = false; + // this is last worker. Let us keep it while pool is alive. + if ( !pPool->m_bIsDestroying && pPool->m_iActiveThreads==1 ) + { + // this thread is 'virtually' inactive also + if ( !pPool->m_iWorksCount.GetValue () ) + { + sphSleepMsec ( 1 ); + continue; + } + } + break; } } - if ( bDone ) - break; + tNext.m_pfn ( &tNext ); + if ( tNext.m_iAgentsDone || !tNext.m_pfn ) + { + CSphScopedLock tStat ( pPool->m_tStatLock ); + pPool->m_iAgentsDone += tNext.m_iAgentsDone; + if (!tNext.m_pfn) + pPool->m_iWorksCount.Dec(); + pPool->m_tChanged.SetEvent(); + } - int64_t tmSelect = sphMicroTimer(); - int64_t tmMicroLeft = tmMaxTimer - tmSelect; - if ( tmMicroLeft<=0 ) // FIXME? what about iTimeout==0 case? + iSpinCount++; + if ( iSpinCount>1 && iSpinCount<4 ) // it could be better not to do the same work { - bTimeout = true; - break; + pPool->Push ( tNext ); + tNext = AgentWorkContext_t(); + } else if ( pPool->m_iWorksCount.GetValue ()>1 && iPopCount>pPool->m_iWorksCount.GetValue () ) // should sleep on queue wrap + { + iPopCount = 0; + sphSleepMsec ( 1 ); } -#if HAVE_POLL - int iSelected = ::poll ( fds.Begin(), fds.GetLength(), int( tmMicroLeft/1000 ) ); -#else - struct timeval tvTimeout; - tvTimeout.tv_sec = (int)( tmMicroLeft / 1000000 ); // full seconds - tvTimeout.tv_usec = (int)( tmMicroLeft % 1000000 ); // microseconds - int iSelected = ::select ( 1+iMax, &fdsRead, NULL, NULL, &tvTimeout ); -#endif + } + --pPool->m_iActiveThreads; + sphLogDebugv ( "Thread func finished for %p, now %d threads active", pArg, (DWORD)pPool->m_iActiveThreads ); +} - dAgents.Begin()->m_iWaited += sphMicroTimer() - tmSelect; +void ThdWorkParallel ( AgentWorkContext_t * ); +void ThdWorkWait ( AgentWorkContext_t * pCtx ) +{ + pCtx->m_pfn = ( pCtx->m_tmWaitm_pfn = NULL; + pCtx->m_pAgents->m_eState = AGENT_UNUSED; - ARRAY_FOREACH ( i, dWorkingSet ) - { - AgentConn_t & tAgent = dAgents[dWorkingSet[i]]; - if ( tAgent.m_bBlackhole ) - continue; - if (!( tAgent.m_eState==AGENT_QUERYED || tAgent.m_eState==AGENT_REPLY || tAgent.m_eState==AGENT_PREREPLY )) - continue; + if ( !pCtx->m_pAgents->m_iRetryLimit || !pCtx->m_iDelay || pCtx->m_pAgents->m_iRetries>pCtx->m_pAgents->m_iRetryLimit ) + return; -#if HAVE_POLL - if (!( fds[i].revents & POLLIN )) -#else - if ( !FD_ISSET ( tAgent.m_iSock, &fdsRead ) ) -#endif - continue; + int64_t tmNextTry = sphMicroTimer() + pCtx->m_iDelay*1000; + pCtx->m_pfn = ThdWorkWait; + ++pCtx->m_pAgents->m_iRetries; + pCtx->m_tmWait = tmNextTry; + pCtx->m_pAgents->m_eState = AGENT_RETRY; + pCtx->m_pAgents->SpecifyAndSelectMirror (); +} - // if there was no reply yet, read reply header - bool bFailure = true; - bool bWarnings = false; - for ( ;; ) +void ThdWorkParallel ( AgentWorkContext_t * pCtx ) +{ + RemoteConnectToAgent ( *pCtx->m_pAgents ); + if ( pCtx->m_pAgents->m_eState==AGENT_UNUSED ) + { + SetNextRetry ( pCtx ); + return; + } + + RemoteQueryAgents ( pCtx ); + if ( pCtx->m_pAgents->m_eState==AGENT_RETRY ) // next round of connect try + { + SetNextRetry ( pCtx ); + } else + { + pCtx->m_pfn = NULL; + pCtx->m_iAgentsDone = 1; + } +} + + +void ThdWorkSequental ( AgentWorkContext_t * pCtx ) +{ + if ( pCtx->m_pAgents->m_iRetries ) + sphSleepMsec ( pCtx->m_iDelay ); + + for ( int iAgent=0; iAgentm_iAgentCount; iAgent++ ) + { + AgentConn_t & tAgent = pCtx->m_pAgents[iAgent]; + if ( !tAgent.m_iRetries || tAgent.m_eState==AGENT_RETRY ) + RemoteConnectToAgent ( tAgent ); + } + + pCtx->m_iAgentsDone += RemoteQueryAgents ( pCtx ); + + bool bNeedRetry = false; + if ( pCtx->m_pAgents->m_iRetryLimit ) + { + for ( int i = 0; im_iAgentCount; i++ ) + if ( pCtx->m_pAgents[i].m_eState==AGENT_RETRY ) { - if ( tAgent.m_eState==AGENT_QUERYED || tAgent.m_eState==AGENT_PREREPLY ) + bNeedRetry = true; + break; + } + } + + pCtx->m_pfn = NULL; + if ( bNeedRetry ) + { + for ( int i = 0; im_iAgentCount; i++ ) + { + AgentConn_t & tAgent = pCtx->m_pAgents[i]; + if ( tAgent.m_eState==AGENT_RETRY ) + { + ++tAgent.m_iRetries; + if ( tAgent.m_iRetriesm_pfn = ThdWorkSequental; + tAgent.SpecifyAndSelectMirror (); + } else + tAgent.m_eState = AGENT_UNUSED; + } + } + } +} - // try to read - struct - { - WORD m_iStatus; - WORD m_iVer; - int m_iLength; - } tReplyHeader; - STATIC_SIZE_ASSERT ( tReplyHeader, 8 ); +class CSphRemoteAgentsController : public ISphRemoteAgentsController +{ +public: + CSphRemoteAgentsController ( int iThreads, CSphVector & dAgents, + const IRequestBuilder_t & tBuilder, int iTimeout, int iRetryMax=0, int iDelay=0 ); - if ( sphSockRecv ( tAgent.m_iSock, (char*)&tReplyHeader, sizeof(tReplyHeader) )!=sizeof(tReplyHeader) ) - { - // bail out if failed - tAgent.m_sFailure.SetSprintf ( "failed to receive reply header" ); - agent_stats_inc ( tAgent, eNetworkErrors ); - break; - } + virtual ~CSphRemoteAgentsController (); - tReplyHeader.m_iStatus = ntohs ( tReplyHeader.m_iStatus ); - tReplyHeader.m_iVer = ntohs ( tReplyHeader.m_iVer ); - tReplyHeader.m_iLength = ntohl ( tReplyHeader.m_iLength ); - // check the packet - if ( tReplyHeader.m_iLength<0 || tReplyHeader.m_iLength>g_iMaxPacketSize ) // FIXME! add reasonable max packet len too - { - tAgent.m_sFailure.SetSprintf ( "invalid packet size (status=%d, len=%d, max_packet_size=%d)", - tReplyHeader.m_iStatus, tReplyHeader.m_iLength, g_iMaxPacketSize ); - agent_stats_inc ( tAgent, eWrongReplies ); - break; - } + // check that there are no works to do + virtual bool IsDone () const override + { + return m_pWorkerPool->HasIncompleteWorks()==0; + } - // header received, switch the status - assert ( tAgent.m_pReplyBuf==NULL ); - tAgent.m_eState = AGENT_REPLY; - tAgent.m_pReplyBuf = new BYTE [ tReplyHeader.m_iLength ]; - tAgent.m_iReplySize = tReplyHeader.m_iLength; - tAgent.m_iReplyRead = 0; - tAgent.m_iReplyStatus = tReplyHeader.m_iStatus; + // block execution while there are works to do + virtual int Finish () override; - if ( !tAgent.m_pReplyBuf ) - { - // bail out if failed - tAgent.m_sFailure.SetSprintf ( "failed to alloc %d bytes for reply buffer", tAgent.m_iReplySize ); - break; - } - } + // check that some agents are done at this iteration + virtual bool HasReadyAgents () const override + { + return (m_pWorkerPool->GetReadyCount()>0 ); + } - // if we are reading reply, read another chunk - if ( tAgent.m_eState==AGENT_REPLY ) - { - // do read - assert ( tAgent.m_iReplyReadFetchReadyCount ()>0); + } - // bail out if read failed - if ( iRes<=0 ) - { - tAgent.m_sFailure.SetSprintf ( "failed to receive reply body: %s", sphSockError() ); - agent_stats_inc ( tAgent, eNetworkErrors ); - break; - } + virtual void WaitAgentsEvent () override; - assert ( iRes>0 ); - assert ( tAgent.m_iReplyRead+iRes<=tAgent.m_iReplySize ); - tAgent.m_iReplyRead += iRes; - } + virtual int RetryFailed () override; - // if reply was fully received, parse it - if ( tAgent.m_eState==AGENT_REPLY && tAgent.m_iReplyRead==tAgent.m_iReplySize ) - { - MemInputBuffer_c tReq ( tAgent.m_pReplyBuf, tAgent.m_iReplySize ); +private: + ThdWorkPool_c * m_pWorkerPool; + CSphVector m_dThds; - // absolve thy former sins - tAgent.m_sFailure = ""; + // stores params from ctr to work of RetryFailed + CSphVector * m_pAgents; + const IRequestBuilder_t * m_pBuilder; + int m_iTimeout; + int m_iDelay; - // check for general errors/warnings first - if ( tAgent.m_iReplyStatus==SEARCHD_WARNING ) - { - CSphString sAgentWarning = tReq.GetString (); - tAgent.m_sFailure.SetSprintf ( "remote warning: %s", sAgentWarning.cstr() ); - bWarnings = true; +}; - } else if ( tAgent.m_iReplyStatus==SEARCHD_RETRY ) - { - tAgent.m_eState = AGENT_RETRY; - CSphString sAgentError = tReq.GetString (); - tAgent.m_sFailure.SetSprintf ( "remote warning: %s", sAgentError.cstr() ); - break; +CSphRemoteAgentsController::CSphRemoteAgentsController ( int iThreads, + CSphVector & dAgents, const IRequestBuilder_t & tBuilder, + int iTimeout, int iRetryMax, int iDelay ) + : m_pWorkerPool ( new ThdWorkPool_c (dAgents.GetLength()) ) +{ + assert ( dAgents.GetLength() ); - } else if ( tAgent.m_iReplyStatus!=SEARCHD_OK ) - { - CSphString sAgentError = tReq.GetString (); - tAgent.m_sFailure.SetSprintf ( "remote error: %s", sAgentError.cstr() ); - break; - } + m_pAgents = &dAgents; + m_pBuilder = &tBuilder; + m_iTimeout = iTimeout; + m_iDelay = iDelay; - // call parser - if ( !tParser.ParseReply ( tReq, tAgent ) ) - break; + iThreads = Max ( 1, Min ( iThreads, dAgents.GetLength() ) ); + m_dThds.Resize ( iThreads ); - // check if there was enough data - if ( tReq.GetError() ) - { - tAgent.m_sFailure.SetSprintf ( "incomplete reply" ); - agent_stats_inc ( tAgent, eWrongReplies ); - break; - } + AgentWorkContext_t tCtx; + tCtx.m_pBuilder = &tBuilder; + tCtx.m_iAgentCount = 1; + tCtx.m_pfn = ThdWorkParallel; + tCtx.m_iDelay = iDelay; + tCtx.m_iTimeout = iTimeout; - // all is well - iAgents++; - tAgent.Close ( false ); - tAgent.m_bSuccess = true; - } + if ( iThreads>1 ) + { + m_pWorkerPool->SetWorksCount ( dAgents.GetLength() ); + ARRAY_FOREACH ( i, dAgents ) + { + tCtx.m_pAgents = dAgents.Begin()+i; + tCtx.m_pAgents->m_iRetryLimit = iRetryMax * tCtx.m_pAgents->NumOfMirrors (); + m_pWorkerPool->RawPush ( tCtx ); + } + } else + { + m_pWorkerPool->SetWorksCount ( 1 ); + tCtx.m_pAgents = dAgents.Begin(); + tCtx.m_iAgentCount = dAgents.GetLength(); + for ( auto& dAgent : dAgents ) + dAgent.m_iRetryLimit = iRetryMax * dAgent.NumOfMirrors (); + tCtx.m_pfn = ThdWorkSequental; + m_pWorkerPool->RawPush ( tCtx ); + } - bFailure = false; - break; - } + ARRAY_FOREACH ( i, m_dThds ) + SphCrashLogger_c::ThreadCreate ( m_dThds.Begin()+i, ThdWorkPool_c::PoolThreadFunc, m_pWorkerPool ); +} - if ( bFailure ) - { - tAgent.Close (); - tAgent.m_dResults.Reset (); - } else if ( tAgent.m_bSuccess ) +CSphRemoteAgentsController::~CSphRemoteAgentsController () +{ + SafeDelete (m_pWorkerPool); + ARRAY_FOREACH ( i, m_dThds ) + sphThreadJoin ( m_dThds.Begin()+i ); + + m_dThds.Resize ( 0 ); +} + +// block execution while there are works to do +int CSphRemoteAgentsController::Finish () +{ + while ( !IsDone() ) + WaitAgentsEvent(); + + return m_pWorkerPool->GetReadyTotal(); +} + +void CSphRemoteAgentsController::WaitAgentsEvent () +{ + m_pWorkerPool->m_tChanged.WaitEvent(); +} + +int CSphRemoteAgentsController::RetryFailed () +{ + AgentWorkContext_t tCtx; + tCtx.m_pBuilder = m_pBuilder; + tCtx.m_iDelay = m_iDelay; + tCtx.m_iTimeout = m_iTimeout; + + tCtx.m_pAgents = m_pAgents->Begin (); + tCtx.m_iAgentCount = m_pAgents->GetLength (); + int iRetries=0; + for ( auto &dAgent : *m_pAgents ) + { + if ( dAgent.m_eState==AGENT_RETRY ) + { + ++dAgent.m_iRetries; + if ( dAgent.m_iRetriesPush ( tCtx ); + m_pWorkerPool->AddWorksCount ( 1 ); + return iRetries; +} + +ISphRemoteAgentsController* GetAgentsController ( int iThreads, CSphVector & dAgents, + const IRequestBuilder_t & tBuilder, int iTimeout, int iRetryMax, int iDelay ) +{ + return new CSphRemoteAgentsController ( iThreads, dAgents, tBuilder, iTimeout, iRetryMax, iDelay ); +} + + +/// check if a non-blocked socket is still connected +bool sphNBSockEof ( int iSock ) +{ + if ( iSock<0 ) + return true; + + char cBuf; + // since socket is non-blocked, ::recv will not block anyway + int iRes = ::recv ( iSock, &cBuf, sizeof ( cBuf ), MSG_PEEK ); + if ( (!iRes) || (iRes<0 && sphSockGetErrno ()!=EWOULDBLOCK )) + return true; + return false; +} + +ISphNetEvents::~ISphNetEvents () = default; + +#if POLLING_EPOLL || POLLING_KQUEUE + +// in case of epoll/kqueue the full set of polled sockets are stored +// in a cache inside kernel, so once added, we can't iterate over all of the items. +// So, we store them in linked list for that purpose. + + +// store and iterate over the list of items +class IterableEvents_c : public ISphNetEvents +{ +protected: + + // wrap raw void* into ListNode_t to store it in List_t + class ListedData_t : public ListNode_t + { + const void * m_pData; + public: + explicit ListedData_t ( const void * pData ) : m_pData ( pData ) + { } + + const void * Data () const + { + return m_pData; + } + }; + + List_t m_tWork; + NetEventsIterator_t m_tIter; + ListedData_t * m_pIter; + +protected: + ListedData_t * AddNewEventData ( const void * pData ) + { + assert ( pData ); + auto pIntData = new ListedData_t ( pData ); + m_tWork.Add ( pIntData ); + return pIntData; + } + + void ResetIterator () + { + m_tIter.Reset(); + m_pIter = nullptr; + } + + void RemoveCurrentItem () + { + assert ( m_pIter ); + assert ( m_pIter->Data ()==m_tIter.m_pData ); + assert ( m_tIter.m_pData ); + + ListNode_t * pPrev = m_pIter->m_pPrev; + m_tWork.Remove ( (ListNode_t *) m_pIter ); + SafeDelete( m_pIter ); + m_pIter = (ListedData_t *) pPrev; + m_tIter.m_pData = m_pIter->Data (); + } + +public: + IterableEvents_c () + { + m_tIter.Reset(); + m_pIter = nullptr; } - // close timed-out agents - ARRAY_FOREACH ( iAgent, dAgents ) + ~IterableEvents_c () { - AgentConn_t & tAgent = dAgents[iAgent]; - if ( tAgent.m_bBlackhole ) - tAgent.Close (); - else if ( bTimeout && ( tAgent.m_eState==AGENT_QUERYED || tAgent.m_eState==AGENT_PREREPLY || - ( tAgent.m_eState==AGENT_REPLY && tAgent.m_iReplyRead!=tAgent.m_iReplySize ) ) ) + while (m_tWork.GetLength()) { - assert ( !tAgent.m_dResults.GetLength() ); - assert ( !tAgent.m_bSuccess ); - tAgent.Fail ( eTimeoutsQuery, "query timed out" ); + m_pIter = (ListedData_t *) m_tWork.Begin(); + m_tWork.Remove(m_pIter); + SafeDelete( m_pIter ); } } - return iAgents; -} - -#endif // HAVE_EPOLL + bool IterateNextAll () override + { + if ( !m_pIter ) + { + if ( m_tWork.Begin ()==m_tWork.End () ) + return false; + m_pIter = (ListedData_t *) m_tWork.Begin (); + m_tIter.m_pData = m_pIter->Data (); + return true; + } else + { + m_pIter = (ListedData_t *) m_pIter->m_pNext; + m_tIter.m_pData = m_pIter->Data (); + if ( m_pIter!=m_tWork.End () ) + return true; -struct AgentWorkContext_t; -typedef void ( *ThdWorker_fn ) ( AgentWorkContext_t * ); + m_tIter.m_pData = nullptr; + m_pIter = nullptr; + return false; + } + } + NetEventsIterator_t &IterateGet () override + { + return m_tIter; + } +}; -struct AgentWorkContext_t : public AgentConnectionContext_t -{ - ThdWorker_fn m_pfn; ///< work functor & flag of dummy element - int64_t m_tmWait; - int m_iAgentsDone; +#endif - AgentWorkContext_t () - : m_pfn ( NULL ) - , m_tmWait ( 0 ) - , m_iAgentsDone ( 0 ) - {} -}; -class ThdWorkPool_c : ISphNoncopyable +#if POLLING_EPOLL +class EpollEvents_c : public IterableEvents_c { private: - CSphMutex m_tDataLock; - CSphMutex m_tStatLock; -public: - CSphAutoEvent m_tChanged; - CSphAtomic m_iActiveThreads; + CSphVector m_dReady; + int m_iLastReportedErrno; + int m_iReady; + int m_iEFD; + int m_iIterEv; -private: - AgentWorkContext_t * m_dData; // works array - int m_iLen; +public: + explicit EpollEvents_c ( int iSizeHint ) + : m_iLastReportedErrno ( -1 ) + , m_iReady ( 0 ) + { + m_iEFD = epoll_create ( iSizeHint ); // 1000 is dummy, see man + if ( m_iEFD==-1 ) + sphDie ( "failed to create epoll main FD, errno=%d, %s", errno, strerror ( errno ) ); - int m_iHead; // ring buffer begin - int m_iTail; // ring buffer end + m_dReady.Reserve ( iSizeHint ); + m_iIterEv = -1; + } - CSphAtomic m_iWorksCount; // count of works to be done - volatile int m_iAgentsDone; // count of agents that finished their works - volatile int m_iAgentsReported; // count of agents that reported of their work done - volatile bool m_bIsDestroying; // help to keep at least 1 worker thread active + ~EpollEvents_c () + { + SafeClose ( m_iEFD ); + } - CrashQuery_t m_tCrashQuery; // query that got reported on crash + void SetupEvent ( int iSocket, PoolEvents_e eFlags, const void* pData ) override + { + assert ( pData && iSocket>=0 ); + assert ( eFlags==SPH_POLL_WR || eFlags==SPH_POLL_RD ); -public: - explicit ThdWorkPool_c ( int iLen ); - ~ThdWorkPool_c (); + auto pIntData = AddNewEventData(pData); + epoll_event tEv; + tEv.data.ptr = pIntData; + tEv.events = (eFlags==SPH_POLL_RD ? EPOLLIN : EPOLLOUT); - AgentWorkContext_t Pop(); + sphLogDebugv ( "%p epoll %d setup, ev=0x%u, sock=%d", pData, m_iEFD, tEv.events, iSocket ); - void Push ( const AgentWorkContext_t & tElem ); - void RawPush ( const AgentWorkContext_t & tElem ); - int GetReadyCount () const; - int FetchReadyCount (); // each call returns new portion of number + int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_ADD, iSocket, &tEv ); + if ( iRes==-1 ) + sphWarning ( "failed to setup epoll event for sock %d, errno=%d, %s", iSocket, errno, strerror ( errno ) ); + } - int GetReadyTotal () const + bool Wait ( int timeoutMs ) override { - return m_iAgentsDone; + m_dReady.Resize ( m_tWork.GetLength () ); + // need positive timeout for communicate threads back and shutdown + m_iReady = epoll_wait ( m_iEFD, m_dReady.Begin (), m_dReady.GetLength (), timeoutMs ); + + if ( m_iReady<0 ) + { + int iErrno = sphSockGetErrno (); + // common recoverable errors + if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) + return false; + + if ( m_iLastReportedErrno!=iErrno ) + { + sphWarning ( "poll tick failed: %s", sphSockError ( iErrno ) ); + m_iLastReportedErrno = iErrno; + } + return false; + } + + return ( m_iReady>0 ); } - bool HasIncompleteWorks () const + int IterateStart () override { - return ( m_iWorksCount.GetValue ()>0 ); + ResetIterator(); + m_iIterEv = -1; + return m_iReady; } - void SetWorksCount ( int iWorkers ) + bool IterateNextReady () override { - m_iWorksCount.SetValue (iWorkers); + ResetIterator (); + ++m_iIterEv; + if ( m_iReady<=0 || m_iIterEv>=m_iReady ) + return false; + + const epoll_event & tEv = m_dReady[m_iIterEv]; + + m_pIter = (ListedData_t *) tEv.data.ptr; + m_tIter.m_pData = m_pIter->Data (); + + if ( tEv.events & EPOLLIN ) + { + m_tIter.m_uEvents |= SPH_POLL_RD; + m_tIter.m_bReadable = true; + } + if ( tEv.events & EPOLLOUT ) + { + m_tIter.m_uEvents |= SPH_POLL_WR; + m_tIter.m_bWritable = true; + } + if ( tEv.events & EPOLLHUP ) + m_tIter.m_uEvents |= SPH_POLL_HUP; + if ( tEv.events & EPOLLERR ) + m_tIter.m_uEvents |= SPH_POLL_ERR; + if ( tEv.events & EPOLLPRI ) + m_tIter.m_uEvents |= SPH_POLL_PRI; + + return true; } - void AddWorksCount ( int iWorkers ) + void IterateChangeEvent ( int iSocket, PoolEvents_e eFlags ) override { - m_iWorksCount.Add ( iWorkers ); + epoll_event tEv; + tEv.data.ptr = (void*) m_pIter; + tEv.events = (eFlags==SPH_POLL_RD ? EPOLLIN : EPOLLOUT); ; + + sphLogDebugv ( "%p epoll change, ev=0x%u, sock=%d", m_tIter.m_pData, tEv.events, iSocket ); + + int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_MOD, iSocket, &tEv ); + if ( iRes==-1 ) + sphWarning ( "failed to modify epoll event for sock %d, errno=%d, %s", iSocket, errno, strerror ( errno ) ); } - static void PoolThreadFunc ( void * pArg ); -}; + void IterateRemove ( int iSocket ) override + { + assert ( m_pIter->Data()==m_tIter.m_pData ); -ThdWorkPool_c::ThdWorkPool_c ( int iLen ) -{ - m_tCrashQuery = SphCrashLogger_c::GetQuery(); // transfer query info for crash logger to new thread + sphLogDebugv ( "%p epoll remove, ev=0x%u, sock=%d", m_tIter.m_pData, m_tIter.m_uEvents, iSocket ); + assert ( m_tIter.m_pData ); - m_iLen = iLen+1; - m_iTail = m_iHead = 0; - m_iWorksCount.SetValue (0); - m_iAgentsDone = m_iAgentsReported = 0; - m_bIsDestroying = false; + epoll_event tEv; + int iRes = epoll_ctl ( m_iEFD, EPOLL_CTL_DEL, iSocket, &tEv ); - m_dData = new AgentWorkContext_t[m_iLen]; -#ifndef NDEBUG - for ( int i=0; i0 ) - sphSleepMsec ( 1 ); - m_tChanged.Done(); - SafeDeleteArray ( m_dData ); + return new EpollEvents_c ( iSizeHint ); } -AgentWorkContext_t ThdWorkPool_c::Pop() -{ - AgentWorkContext_t tRes; - if ( m_iTail==m_iHead ) // quick path for empty pool - return tRes; +#endif +#if POLLING_KQUEUE - CSphScopedLock tData ( m_tDataLock ); // lock on create, unlock on destroy +class KqueueEvents_c : public IterableEvents_c +{ - if ( m_iTail==m_iHead ) // it might be empty now as another thread could steal work till that moment - return tRes; +private: + CSphVector m_dReady; + int m_iLastReportedErrno; + int m_iReady; + int m_iKQ; + int m_iIterEv; - tRes = m_dData[m_iHead]; - assert ( tRes.m_pfn ); -#ifndef NDEBUG - m_dData[m_iHead] = AgentWorkContext_t(); // to make sure that we don't rewrite valid elements -#endif - m_iHead = ( m_iHead+1 ) % m_iLen; +public: + explicit KqueueEvents_c ( int iSizeHint ) + : m_iLastReportedErrno ( -1 ) + , m_iReady ( 0 ) + { + m_iKQ = kqueue (); // 1000 is dummy, see man + if ( m_iKQ==-1 ) + sphDie ( "failed to create kqueue main FD, errno=%d, %s", errno, strerror ( errno ) ); - return tRes; -} + m_dReady.Reserve ( iSizeHint ); + m_iIterEv = -1; + } -void ThdWorkPool_c::Push ( const AgentWorkContext_t & tElem ) -{ - if ( !tElem.m_pfn ) - return; + ~KqueueEvents_c () + { + SafeClose ( m_iKQ ); + } - CSphScopedLock tData ( m_tDataLock ); - RawPush ( tElem ); -} + void SetupEvent ( int iSocket, PoolEvents_e eFlags, const void* pData ) override + { + assert ( pData && iSocket>=0 ); + assert ( eFlags==SPH_POLL_WR || eFlags==SPH_POLL_RD ); -void ThdWorkPool_c::RawPush ( const AgentWorkContext_t & tElem ) -{ - assert ( !m_dData[m_iTail].m_pfn ); // to make sure that we don't rewrite valid elements - m_dData[m_iTail] = tElem; - m_iTail = ( m_iTail+1 ) % m_iLen; -} + auto pIntData = AddNewEventData(pData); -int ThdWorkPool_c::FetchReadyCount () -{ - // it could be better to lock here to get accurate value of m_iAgentsDone - // however that make lock contention of 1 sec on 1000 query ~ total 3.2sec vs 2.2 sec ( trunk ) - int iNowDone = GetReadyCount(); - m_iAgentsReported += iNowDone; - return iNowDone; -} + struct kevent tEv; + EV_SET(&tEv, iSocket, (eFlags==SPH_POLL_RD ? EVFILT_READ : EVFILT_WRITE), EV_ADD, 0, 0, pIntData); -int ThdWorkPool_c::GetReadyCount () const -{ - // it could be better to lock here to get accurate value of m_iAgentsDone - // however that make lock contention of 1 sec on 1000 query ~ total 3.2sec vs 2.2 sec ( trunk ) - return m_iAgentsDone - m_iAgentsReported; -} + sphLogDebugv ( "%p kqueue setup, ev=0x%u, sock=%d", pData, tEv.flags, iSocket ); -void ThdWorkPool_c::PoolThreadFunc ( void * pArg ) -{ - ThdWorkPool_c * pPool = (ThdWorkPool_c *)pArg; - assert (pPool); - ++pPool->m_iActiveThreads; - sphLogDebugv ("Thread func started for %p, now %d threads active", pArg, (DWORD)pPool->m_iActiveThreads); - SphCrashLogger_c::SetLastQuery ( pPool->m_tCrashQuery ); + int iRes = kevent (m_iKQ, &tEv, 1, NULL, 0, NULL); + if ( iRes==-1 ) + sphWarning ( "failed to setup kqueue event for sock %d, errno=%d, %s", iSocket, errno, strerror ( errno ) ); + } - int iSpinCount = 0; - int iPopCount = 0; - AgentWorkContext_t tNext; - for ( ;; ) + bool Wait ( int timeoutMs ) override { - if ( !tNext.m_pfn ) // pop new work if current is done + m_dReady.Resize ( m_tWork.GetLength () ); + + timespec ts; + timespec *pts = nullptr; + if ( timeoutMs ) { - iSpinCount = 0; - ++iPopCount; - tNext = pPool->Pop(); - if ( !tNext.m_pfn ) // if there is no work at queue - worker done - { - // this is last worker. Let us keep it while pool is alive. - if ( !pPool->m_bIsDestroying && pPool->m_iActiveThreads==1 ) - { - // this thread is 'virtually' inactive also - if ( !pPool->m_iWorksCount.GetValue () ) - { - sphSleepMsec ( 1 ); - continue; - } - } - break; - } + ts.tv_sec = timeoutMs/1000; + ts.tv_nsec = timeoutMs*1000000; + pts = &ts; } + // need positive timeout for communicate threads back and shutdown + m_iReady = kevent (m_iKQ, NULL, 0, m_dReady.begin(), m_dReady.GetLength(), pts); - tNext.m_pfn ( &tNext ); - if ( tNext.m_iAgentsDone || !tNext.m_pfn ) + if ( m_iReady<0 ) { - CSphScopedLock tStat ( pPool->m_tStatLock ); - pPool->m_iAgentsDone += tNext.m_iAgentsDone; - if (!tNext.m_pfn) - pPool->m_iWorksCount.Dec(); - pPool->m_tChanged.SetEvent(); + int iErrno = sphSockGetErrno (); + // common recoverable errors + if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) + return false; + + if ( m_iLastReportedErrno!=iErrno ) + { + sphWarning ( "kqueue tick failed: %s", sphSockError ( iErrno ) ); + m_iLastReportedErrno = iErrno; + } + return false; } - iSpinCount++; - if ( iSpinCount>1 && iSpinCount<4 ) // it could be better not to do the same work + return ( m_iReady>0 ); + } + + int IterateStart () override + { + ResetIterator(); + m_iIterEv = -1; + return m_iReady; + } + + bool IterateNextReady () override + { + ResetIterator(); + ++m_iIterEv; + if ( m_iReady<=0 || m_iIterEv>=m_iReady ) + return false; + + const struct kevent & tEv = m_dReady[m_iIterEv]; + + m_pIter = (ListedData_t *) tEv.data; + m_tIter.m_pData = m_pIter->Data (); + + if ( tEv.flags & EVFILT_READ ) { - pPool->Push ( tNext ); - tNext = AgentWorkContext_t(); - } else if ( pPool->m_iWorksCount.GetValue ()>1 && iPopCount>pPool->m_iWorksCount.GetValue () ) // should sleep on queue wrap + m_tIter.m_uEvents |= SPH_POLL_RD; + m_tIter.m_bReadable = true; + } + if ( tEv.flags & EVFILT_WRITE ) { - iPopCount = 0; - sphSleepMsec ( 1 ); + m_tIter.m_uEvents |= SPH_POLL_WR; + m_tIter.m_bWritable = true; } + return true; + } + + void IterateChangeEvent ( int iSocket, PoolEvents_e eFlags ) override + { + assert ( eFlags==SPH_POLL_WR || eFlags==SPH_POLL_RD ); + struct kevent tEv; + EV_SET(&tEv, iSocket, (eFlags==SPH_POLL_RD ? EVFILT_READ : EVFILT_WRITE), EV_ADD, 0, 0, (void*) m_pIter); + + int iRes = kevent (m_iKQ, &tEv, 1, NULL, 0, NULL); + if ( iRes==-1 ) + sphWarning ( "failed to setup kqueue event for sock %d, errno=%d, %s", iSocket, errno, strerror ( errno ) ); } - --pPool->m_iActiveThreads; - sphLogDebugv ( "Thread func finished for %p, now %d threads active", pArg, (DWORD)pPool->m_iActiveThreads ); -} -void ThdWorkParallel ( AgentWorkContext_t * ); -void ThdWorkWait ( AgentWorkContext_t * pCtx ) -{ - pCtx->m_pfn = ( pCtx->m_tmWaitm_pfn = NULL; - pCtx->m_pAgents->m_eState = AGENT_UNUSED; + // might be already closed by worker from thread pool + if ( iRes==-1 ) + sphLogDebugv ( "failed to remove kqueue event for sock %d(%p), errno=%d, %s", iSocket, m_tIter.m_pData, errno, strerror ( errno ) ); - if ( !pCtx->m_pAgents->m_iRetryLimit || !pCtx->m_iDelay || pCtx->m_pAgents->m_iRetries>pCtx->m_pAgents->m_iRetryLimit ) - return; + RemoveCurrentItem (); + } +}; - int64_t tmNextTry = sphMicroTimer() + pCtx->m_iDelay*1000; - pCtx->m_pfn = ThdWorkWait; - ++pCtx->m_pAgents->m_iRetries; - pCtx->m_tmWait = tmNextTry; - pCtx->m_pAgents->m_eState = AGENT_RETRY; - pCtx->m_pAgents->SpecifyAndSelectMirror (); +ISphNetEvents* sphCreatePoll ( int iSizeHint, bool ) +{ + return new KqueueEvents_c ( iSizeHint ); } -void ThdWorkParallel ( AgentWorkContext_t * pCtx ) +#endif +#if POLLING_POLL +class PollEvents_c : public ISphNetEvents { - RemoteConnectToAgent ( *pCtx->m_pAgents ); - if ( pCtx->m_pAgents->m_eState==AGENT_UNUSED ) +private: + CSphVector m_dWork; + CSphVector m_dEvents; + int m_iLastReportedErrno; + int m_iReady; + NetEventsIterator_t m_tIter; + int m_iIter; + +public: + explicit PollEvents_c ( int iSizeHint ) + : m_iLastReportedErrno ( -1 ) + , m_iReady ( 0 ) + , m_iIter ( -1) { - SetNextRetry ( pCtx ); - return; + m_dWork.Reserve ( iSizeHint ); + m_tIter.Reset(); } - RemoteQueryAgents ( pCtx ); - if ( pCtx->m_pAgents->m_eState==AGENT_RETRY ) // next round of connect try - { - SetNextRetry ( pCtx ); - } else + ~PollEvents_c () = default; + + void SetupEvent ( int iSocket, PoolEvents_e eFlags, const void* pData ) override { - pCtx->m_pfn = NULL; - pCtx->m_iAgentsDone = 1; - } -} + assert ( pData && iSocket>=0 ); + assert ( eFlags==SPH_POLL_WR || eFlags==SPH_POLL_RD ); + pollfd tEvent; + tEvent.fd = iSocket; + tEvent.events = (eFlags==SPH_POLL_RD ? POLLIN : POLLOUT); -void ThdWorkSequental ( AgentWorkContext_t * pCtx ) -{ - if ( pCtx->m_pAgents->m_iRetries ) - sphSleepMsec ( pCtx->m_iDelay ); + assert ( m_dEvents.GetLength() == m_dWork.GetLength() ); + m_dEvents.Add ( tEvent ); + m_dWork.Add ( pData ); + } - for ( int iAgent=0; iAgentm_iAgentCount; iAgent++ ) + bool Wait ( int timeoutMs ) override { - AgentConn_t & tAgent = pCtx->m_pAgents[iAgent]; - if ( !tAgent.m_iRetries || tAgent.m_eState==AGENT_RETRY ) - RemoteConnectToAgent ( tAgent ); - } + // need positive timeout for communicate threads back and shutdown + m_iReady = ::poll ( m_dEvents.Begin (), m_dEvents.GetLength (), timeoutMs ); - pCtx->m_iAgentsDone += RemoteQueryAgents ( pCtx ); + if ( m_iReady<0 ) + { + int iErrno = sphSockGetErrno (); + // common recoverable errors + if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) + return false; - bool bNeedRetry = false; - if ( pCtx->m_pAgents->m_iRetryLimit ) - { - for ( int i = 0; im_iAgentCount; i++ ) - if ( pCtx->m_pAgents[i].m_eState==AGENT_RETRY ) + if ( m_iLastReportedErrno!=iErrno ) { - bNeedRetry = true; - break; + sphWarning ( "poll tick failed: %s", sphSockError ( iErrno ) ); + m_iLastReportedErrno = iErrno; } + return false; + } + + return ( m_iReady>0 ); } - pCtx->m_pfn = NULL; - if ( bNeedRetry ) + bool IterateNextAll () override { - for ( int i = 0; im_iAgentCount; i++ ) + assert ( m_dEvents.GetLength ()==m_dWork.GetLength () ); + + ++m_iIter; + m_tIter.m_pData = ( m_iIter=m_dEvents.GetLength () ) + return false; + + for ( ;; ) { - AgentConn_t & tAgent = pCtx->m_pAgents[i]; - if ( tAgent.m_eState==AGENT_RETRY ) + ++m_iIter; + if ( m_iIter>=m_dEvents.GetLength () ) + return false; + + if ( m_dEvents[m_iIter].revents==0 ) + continue; + + --m_iReady; + + m_tIter.m_pData = m_dWork[m_iIter]; + pollfd & tEv = m_dEvents[m_iIter]; + if ( tEv.revents & POLLIN ) { - ++tAgent.m_iRetries; - if ( tAgent.m_iRetriesm_pfn = ThdWorkSequental; - tAgent.SpecifyAndSelectMirror (); - } else - tAgent.m_eState = AGENT_UNUSED; + m_tIter.m_uEvents |= SPH_POLL_RD; + m_tIter.m_bReadable = true; + } + if ( tEv.revents & POLLOUT ) + { + m_tIter.m_uEvents |= SPH_POLL_WR; + m_tIter.m_bWritable = true; } + if ( tEv.revents & POLLHUP ) + m_tIter.m_uEvents |= SPH_POLL_HUP; + if ( tEv.revents & POLLERR ) + m_tIter.m_uEvents |= SPH_POLL_ERR; + + tEv.revents = 0; + return true; } } -} -class CSphRemoteAgentsController : public ISphRemoteAgentsController -{ -public: - CSphRemoteAgentsController ( int iThreads, CSphVector & dAgents, - const IRequestBuilder_t & tBuilder, int iTimeout, int iRetryMax=0, int iDelay=0 ); + void IterateChangeEvent ( int iSocket, PoolEvents_e eFlags ) override + { + assert ( m_iIter>=0 && m_iIter=0 && m_iIterHasIncompleteWorks()==0; + --m_iIter; + m_tIter.m_pData = nullptr; } - // block execution while there are works to do - virtual int Finish () override; - - // check that some agents are done at this iteration - virtual bool HasReadyAgents () const override + int IterateStart () override { - return (m_pWorkerPool->GetReadyCount()>0 ); + m_iIter = -1; + m_tIter.Reset(); + + return m_iReady; } - virtual bool FetchReadyAgents () override + NetEventsIterator_t & IterateGet () override { - return (m_pWorkerPool->FetchReadyCount ()>0); + assert ( m_iIter>=0 && m_iIter m_dThds; + CSphVector m_dWork; + CSphVector m_dSockets; + fd_set m_fdsRead; + fd_set m_fdsReadResult; + fd_set m_fdsWrite; + fd_set m_fdsWriteResult; + int m_iMaxSocket; + int m_iLastReportedErrno; + int m_iReady; + NetEventsIterator_t m_tIter; + int m_iIter; - // stores params from ctr to work of RetryFailed - CSphVector * m_pAgents; - const IRequestBuilder_t * m_pBuilder; - int m_iTimeout; - int m_iDelay; +public: + explicit SelectEvents_c ( int iSizeHint ) + : m_iMaxSocket ( 0 ), + m_iLastReportedErrno ( -1 ), + m_iReady ( 0 ), + m_iIter ( -1 ) + { + m_dWork.Reserve ( iSizeHint ); -}; + FD_ZERO ( &m_fdsRead ); + FD_ZERO ( &m_fdsWrite ); -CSphRemoteAgentsController::CSphRemoteAgentsController ( int iThreads, - CSphVector & dAgents, const IRequestBuilder_t & tBuilder, - int iTimeout, int iRetryMax, int iDelay ) - : m_pWorkerPool ( new ThdWorkPool_c (dAgents.GetLength()) ) -{ - assert ( dAgents.GetLength() ); + m_tIter.Reset(); + } - m_pAgents = &dAgents; - m_pBuilder = &tBuilder; - m_iTimeout = iTimeout; - m_iDelay = iDelay; + ~SelectEvents_c () = default; - iThreads = Max ( 1, Min ( iThreads, dAgents.GetLength() ) ); - m_dThds.Resize ( iThreads ); + void SetupEvent ( int iSocket, PoolEvents_e eFlags, const void * pData ) override + { + assert ( pData && iSocket>=0 ); + assert ( eFlags==SPH_POLL_WR || eFlags==SPH_POLL_RD ); - AgentWorkContext_t tCtx; - tCtx.m_pBuilder = &tBuilder; - tCtx.m_iAgentCount = 1; - tCtx.m_pfn = ThdWorkParallel; - tCtx.m_iDelay = iDelay; - tCtx.m_iTimeout = iTimeout; + sphFDSet ( iSocket, (eFlags==SPH_POLL_RD ? &m_fdsRead : &m_fdsWrite)); + m_iMaxSocket = Max ( m_iMaxSocket, iSocket ); - if ( iThreads>1 ) + assert ( m_dSockets.GetLength ()==m_dWork.GetLength () ); + m_dWork.Add ( pData ); + m_dSockets.Add (iSocket); + } + + bool Wait ( int timeoutMs ) override { - m_pWorkerPool->SetWorksCount ( dAgents.GetLength() ); - ARRAY_FOREACH ( i, dAgents ) + struct timeval tvTimeout; + + tvTimeout.tv_sec = (int) (timeoutMs / 1000); // full seconds + tvTimeout.tv_usec = (int) (timeoutMs % 1000) * 1000; // microseconds + m_fdsReadResult = m_fdsRead; + m_fdsWriteResult = m_fdsWrite; + m_iReady = ::select ( 1 + m_iMaxSocket, &m_fdsReadResult, &m_fdsWriteResult, NULL, &tvTimeout ); + + if ( m_iReady<0 ) { - tCtx.m_pAgents = dAgents.Begin()+i; - tCtx.m_pAgents->m_iRetryLimit = iRetryMax * tCtx.m_pAgents->NumOfMirrors (); - m_pWorkerPool->RawPush ( tCtx ); + int iErrno = sphSockGetErrno (); + // common recoverable errors + if ( iErrno==EINTR || iErrno==EAGAIN || iErrno==EWOULDBLOCK ) + return false; + + if ( m_iLastReportedErrno!=iErrno ) + { + sphWarning ( "poll (select version) tick failed: %s", sphSockError ( iErrno ) ); + m_iLastReportedErrno = iErrno; + } + return false; } - } else + + return (m_iReady>0); + } + + bool IterateNextAll () override { - m_pWorkerPool->SetWorksCount ( 1 ); - tCtx.m_pAgents = dAgents.Begin(); - tCtx.m_iAgentCount = dAgents.GetLength(); - for ( auto& dAgent : dAgents ) - dAgent.m_iRetryLimit = iRetryMax * dAgent.NumOfMirrors (); - tCtx.m_pfn = ThdWorkSequental; - m_pWorkerPool->RawPush ( tCtx ); + assert ( m_dSockets.GetLength ()==m_dWork.GetLength () ); + + ++m_iIter; + m_tIter.m_pData = (m_iIter=m_dWork.GetLength () ) + return false; -CSphRemoteAgentsController::~CSphRemoteAgentsController () -{ - SafeDelete (m_pWorkerPool); - ARRAY_FOREACH ( i, m_dThds ) - sphThreadJoin ( m_dThds.Begin()+i ); + for ( ;; ) + { + ++m_iIter; + if ( m_iIter>=m_dWork.GetLength () ) + return false; - m_dThds.Resize ( 0 ); -} + bool bReadable = FD_ISSET ( m_dSockets[m_iIter], &m_fdsReadResult ); + bool bWritable = FD_ISSET ( m_dSockets[m_iIter], &m_fdsWriteResult ); -// block execution while there are works to do -int CSphRemoteAgentsController::Finish () -{ - while ( !IsDone() ) - WaitAgentsEvent(); + if ( !(bReadable || bWritable)) + continue; - return m_pWorkerPool->GetReadyTotal(); -} + --m_iReady; -void CSphRemoteAgentsController::WaitAgentsEvent () -{ - m_pWorkerPool->m_tChanged.WaitEvent(); -} + m_tIter.m_pData = m_dWork[m_iIter]; -int CSphRemoteAgentsController::RetryFailed () -{ - AgentWorkContext_t tCtx; - tCtx.m_pBuilder = m_pBuilder; - tCtx.m_iDelay = m_iDelay; - tCtx.m_iTimeout = m_iTimeout; + if ( bReadable ) + m_tIter.m_uEvents |= SPH_POLL_RD; + if ( bWritable ) + m_tIter.m_uEvents |= SPH_POLL_WR; + m_tIter.m_bReadable = bReadable; + m_tIter.m_bWritable = bWritable; - tCtx.m_pAgents = m_pAgents->Begin (); - tCtx.m_iAgentCount = m_pAgents->GetLength (); - int iRetries=0; - for ( auto &dAgent : *m_pAgents ) - { - if ( dAgent.m_eState==AGENT_RETRY ) - { - ++dAgent.m_iRetries; - if ( dAgent.m_iRetriesPush ( tCtx ); - m_pWorkerPool->AddWorksCount ( 1 ); - return iRetries; -} -ISphRemoteAgentsController* GetAgentsController ( int iThreads, CSphVector & dAgents, - const IRequestBuilder_t & tBuilder, int iTimeout, int iRetryMax, int iDelay ) + void IterateChangeEvent ( int iSocket, PoolEvents_e eFlags ) override + { + assert ( m_iIter>=0 && m_iIter=0 && m_iIter=0 && m_iIter +#if HAVE_KQUEUE +#include +#include +#include +#endif + ///////////////////////////////////////////////////////////////////////////// // SOME SHARED GLOBAL VARIABLES ///////////////////////////////////////////////////////////////////////////// @@ -538,6 +544,69 @@ ISphRemoteAgentsController* GetAgentsController ( int iThreads, CSphVectorversion = 1; + m_xsqlda->sqln = DEF_SQLVARS; +} + +CSphSource_FBSQL::~CSphSource_FBSQL() +{ + delete[](char *) m_xsqlda; + delete[] m_record; + delete[] m_blob; +} + +bool CSphSource_FBSQL::Setup(const CSphSourceParams_FBSQL & pParams) +{ + if (!CSphSource_SQL::Setup(pParams)) + return false; + + m_sCharset = pParams.m_sCharset; + m_sRole = pParams.m_sRole; + + return true; +} + +void CSphSource_FBSQL::SqlDismissResult() +{ + if (!m_statement) + return; + + if (m_selectable) { + sph_isc_dsql_free_statement(m_status, &m_statement, DSQL_close); + } +} + +bool CSphSource_FBSQL::SqlQuery(const char * sQuery) +{ + if (!m_statement) { + if (sph_isc_dsql_allocate_statement(m_status, &m_database, &m_statement)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + } + + if (sph_isc_dsql_prepare(m_status, &m_transaction, &m_statement, 0, sQuery, + SQL_DIALECT_CURRENT, NULL)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + + m_selectable = false; + + // get statement type + const char stmt_info[] = { isc_info_sql_stmt_type }; + char info_buff[16]; + if (sph_isc_dsql_sql_info(m_status, &m_statement, sizeof(stmt_info), stmt_info, + sizeof(info_buff), info_buff)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + if (info_buff[0] != stmt_info[0]) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + + { + const int len = sph_isc_vax_integer(&info_buff[1], 2); + const int stmt_type = sph_isc_vax_integer(&info_buff[3], (short)len); + + m_selectable = (stmt_type == isc_info_sql_stmt_select || + stmt_type == isc_info_sql_stmt_select_for_upd); + } + + + if (m_selectable) + { + if (sph_isc_dsql_describe(m_status, &m_statement, SQLDA_VERSION1, m_xsqlda)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + + if (m_xsqlda->sqld > m_xsqlda->sqln) + { + const short len = m_xsqlda->sqld; + delete[](char*) m_xsqlda; + + m_xsqlda = (XSQLDA*) new char[XSQLDA_LENGTH(len)]; + m_xsqlda->sqln = len; + m_xsqlda->version = 1; + + if (sph_isc_dsql_describe(m_status, &m_statement, SQLDA_VERSION1, m_xsqlda)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + } + + const size_t recsize = parseSQLDA(m_xsqlda, NULL); + if (recsize > m_recsize) + { + delete[] m_record; + + m_recsize = recsize; + m_record = new char[m_recsize]; + } + parseSQLDA(m_xsqlda, m_record); + } + else { + m_xsqlda->sqld = 0; + } + + if (sph_isc_dsql_execute(m_status, &m_transaction, &m_statement, SQLDA_VERSION1, NULL)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: FAIL\n", sQuery ); + return false; + } + + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-QUERY: %s: ok\n", sQuery ); + return true; +} + +bool CSphSource_FBSQL::SqlIsError() +{ + return (m_status[1] != 0); +} + +const char * CSphSource_FBSQL::SqlError() +{ + char * p = m_error, *const end = m_error + sizeof(m_error); + const ISC_STATUS * s = m_status; + while (sph_fb_interpret(p, end - p, &s)) + { + p += strlen(p); + if (p < end - 1) + *p++ = '\n'; + *p = 0; + } + + return m_error; +} + +bool CSphSource_FBSQL::SqlConnect() +{ + if_const ( !InitDynamicFirebird() ) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-CONNECT: FAIL (NO FIREBIRD CLIENT LIB)\n" ); + return false; + } + + char dpb[256]; + char *p = dpb; + + *p++ = isc_dpb_version1; + p = putSphStringInDPB(p, isc_dpb_user_name, m_tParams.m_sUser); + p = putSphStringInDPB(p, isc_dpb_password, m_tParams.m_sPass); + p = putSphStringInDPB(p, isc_dpb_sql_role_name, m_sRole); + p = putSphStringInDPB(p, isc_dpb_lc_ctype, m_sCharset); + + if (sph_isc_attach_database(m_status, 0, m_tParams.m_sDB.cstr(), &m_database, + (short)(p - dpb), dpb)) + { + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-CONNECT: FAIL\n" ); + return false; + } + + //char tpb[] = {isc_tpb_version3, isc_tpb_read, isc_tpb_read_committed, + // isc_tpb_rec_version, isc_tpb_wait}; + + char tpb[] = { isc_tpb_version3, isc_tpb_write, isc_tpb_concurrency, isc_tpb_wait }; + + if (sph_isc_start_transaction(m_status, &m_transaction, 1, &m_database, + sizeof(tpb), tpb)) + { + ISC_STATUS_ARRAY temp = { 0 }; + sph_isc_detach_database(temp, &m_database); + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-CONNECT: FAIL\n" ); + return false; + } + + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-CONNECT: ok\n" ); + + return true; +} + +void CSphSource_FBSQL::SqlDisconnect() +{ + if ( m_tParams.m_bPrintQueries ) + fprintf ( stdout, "SQL-DISCONNECT\n" ); + + if (m_transaction) { + if (sph_isc_commit_transaction(m_status, &m_transaction)) + { + ISC_STATUS_ARRAY temp = { 0 }; + sph_isc_rollback_transaction(temp, &m_transaction); + } + } + + if (m_statement) + sph_isc_dsql_free_statement(m_status, &m_statement, DSQL_drop); + + if (m_database) + sph_isc_detach_database(m_status, &m_database); +} + +int CSphSource_FBSQL::SqlNumFields() +{ + if (!m_xsqlda) + return 0; + + return m_xsqlda->sqld; +} + +bool CSphSource_FBSQL::SqlFetchRow() +{ + if (!m_statement) + return false; + + if (sph_isc_dsql_fetch(m_status, &m_statement, SQLDA_VERSION1, m_xsqlda) == 100) + return false; + + if (m_status[1]) + return false; + + // make returned strings NULL-terminated + XSQLVAR *var = m_xsqlda->sqlvar; + for (int i = 0; i < m_xsqlda->sqld; var++, i++) + { + if (*var->sqlind == 0 && (var->sqltype & (~1)) != SQL_BLOB) + { + short len = *(short *)var->sqldata; + var->sqldata[len + sizeof(short)] = 0; + } + } + + return true; +} + +const char * CSphSource_FBSQL::SqlColumn(int iIndex) +{ + if (!m_xsqlda) + return 0; + + XSQLVAR &var = m_xsqlda->sqlvar[iIndex]; + if (*var.sqlind != 0) + return NULL; + + if ((var.sqltype & (~1)) != SQL_BLOB) + return var.sqldata + sizeof(short); + + + // blob to string + ISC_QUAD * blob_id = (ISC_QUAD *)var.sqldata; + + // empty (NULL) blob + if (!blob_id->gds_quad_high && !blob_id->gds_quad_low) + return NULL; + + isc_blob_handle blob = { 0 }; + if (sph_isc_open_blob(m_status, &m_database, &m_transaction, &blob, blob_id)) + return NULL; + + const char info[] = { isc_info_blob_total_length }; + char resp[16]; + if (sph_isc_blob_info(m_status, &blob, sizeof(info), info, sizeof(resp), resp)) + return NULL; + + if (info[0] != resp[0]) + return NULL; + + int len = sph_isc_vax_integer(&resp[1], 2); + len = sph_isc_vax_integer(&resp[3], (short)len); + + if (len + 4 > m_blobsize) + { + delete[] m_blob; + + m_blobsize = len + 4; + m_blob = new char[m_blobsize]; + } + + char *p = m_blob; + while (true) + { + const unsigned short MAX_SEGMENT = 64 * 1024 - 3; + unsigned short reads = 0; + + const ISC_STATUS ret = sph_isc_get_segment(m_status, &blob, &reads, + (unsigned short)Min(len, MAX_SEGMENT), p); + + if (ret == 0 || ret == isc_segment) + { + p += reads; + len -= reads; + } + else + { + if (ret != isc_segstr_eof) + p = NULL; + + break; + } + } + sph_isc_close_blob(m_status, &blob); + + if (!p) + return NULL; + + // make string NULL-terminated for multi-byte encodings too + const char * end = p + 4; + while (p < end && p < m_blob + m_blobsize) + *p++ = 0; + + return m_blob; +} + +DWORD CSphSource_FBSQL::SqlColumnLength(int iIndex) +{ + if (!m_xsqlda) + return 0; + + XSQLVAR &var = m_xsqlda->sqlvar[iIndex]; + if (*var.sqlind != 0) + return 0; + + return var.sqllen; +} + +const char * CSphSource_FBSQL::SqlFieldName(int iIndex) +{ + if (!m_xsqlda) + return 0; + + return m_xsqlda->sqlvar[iIndex].aliasname; +} + +char * CSphSource_FBSQL::putSphStringInDPB(char * dpb, char clump, CSphString &str) +{ + if (str.IsEmpty()) + return dpb; + + *dpb++ = clump; + + const size_t len = strlen(str.cstr()); + *dpb++ = (char)len; + + memcpy(dpb, str.cstr(), len); + + return dpb + len; +} + +size_t CSphSource_FBSQL::parseSQLDA(XSQLDA * xsqlda, char * buff) +{ + // on the first pass (buff == NULL) convert all SQL_xxx into + // SQL_VARYING and make room for NULL-terminated string + + size_t offset = 0; + int i = 0; + XSQLVAR* var = xsqlda->sqlvar; + for (; i < xsqlda->sqld; var++, i++) + { + // round up to sizeof(short) + offset = (offset + 1) & ~(1); + + short length = var->sqllen; + const int type = var->sqltype & (~1); + switch (type) + { + case SQL_TEXT: + case SQL_VARYING: + if (!buff) + length += sizeof(short) + 1; + break; + + case SQL_SHORT: + case SQL_LONG: + case SQL_INT64: + case SQL_FLOAT: + case SQL_DOUBLE: + length = 26; + break; + + case SQL_TIMESTAMP: + case SQL_TYPE_TIME: + case SQL_TYPE_DATE: + length = 34; + break; + + case SQL_BLOB: + { + // round up to sizeof(ISC_QUAD) + const int quad_size = sizeof(ISC_QUAD) - 1; + offset = (offset + quad_size) & ~(quad_size); + } + break; + + default: + break; + } + + if (type != SQL_BLOB) + { + var->sqltype = SQL_VARYING | 1; + var->sqllen = length; + } + else + { + var->sqltype = SQL_BLOB | 1; + } + if (buff) { + var->sqldata = &buff[offset]; + } + offset += length; + } + + // round up to sizeof(short) + offset = (offset + 1) & ~(1); + + // room for null-indicators (short's) + if (buff) { + for (i = 0, var = xsqlda->sqlvar; i < xsqlda->sqld; var++, i++) + { + var->sqlind = (short*)(&buff[offset]); + offset += sizeof(short); + } + } + else { + offset += sizeof(short) * xsqlda->sqld; + } + + return offset; +} + +#endif // USE_FIREBIRD + ///////////////////////////////////////////////////////////////////////////// // XMLPIPE (v2) ///////////////////////////////////////////////////////////////////////////// diff --git a/src/sphinx.h b/src/sphinx.h index da4d6e570..0d91eb8bc 100644 --- a/src/sphinx.h +++ b/src/sphinx.h @@ -21,6 +21,7 @@ #ifdef _WIN32 #define USE_MYSQL 1 /// whether to compile MySQL support #define USE_PGSQL 0 /// whether to compile PgSQL support + #define USE_FIREBIRD 0 /// whether to compile Firebird support #define USE_ODBC 1 /// whether to compile ODBC support #define USE_LIBEXPAT 1 /// whether to compile libexpat support #define USE_LIBICONV 1 /// whether to compile iconv support @@ -55,6 +56,10 @@ #include #endif +#if USE_FIREBIRD +#include +#endif + #if USE_WINDOWS #include #else @@ -2363,6 +2368,67 @@ struct CSphSource_PgSQL : CSphSource_SQL }; #endif // USE_PGSQL + +#if USE_FIREBIRD +/// Firebird specific source params +struct CSphSourceParams_FBSQL : CSphSourceParams_SQL +{ + CSphString m_sCharset; + CSphString m_sRole; + CSphSourceParams_FBSQL (); +}; + + +/// Firebird source implementation +/// multi-field plain-text documents fetched from given query +struct CSphSource_FBSQL : CSphSource_SQL +{ + explicit CSphSource_FBSQL ( const char * sName ); + virtual ~CSphSource_FBSQL (); + bool Setup ( const CSphSourceParams_FBSQL & pParams ); + +protected: + /// config values + CSphString m_sCharset; + CSphString m_sRole; + + /// API handles + ISC_STATUS_ARRAY m_status; + isc_db_handle m_database; + isc_tr_handle m_transaction; + isc_stmt_handle m_statement; + + // error and record holders + static const int ERROR_BUFFER_SIZE = 1024; + char m_error[ERROR_BUFFER_SIZE]; + + XSQLDA * m_xsqlda; + char * m_record; + size_t m_recsize; + bool m_selectable; + char * m_blob; + size_t m_blobsize; + +protected: + virtual void SqlDismissResult (); + virtual bool SqlQuery ( const char * sQuery ); + virtual bool SqlIsError (); + virtual const char * SqlError (); + virtual bool SqlConnect (); + virtual void SqlDisconnect (); + virtual int SqlNumFields (); + virtual bool SqlFetchRow (); + virtual DWORD SqlColumnLength ( int iIndex ); + virtual const char * SqlColumn ( int iIndex ); + virtual const char * SqlFieldName ( int iIndex ); + +private: + char * putSphStringInDPB ( char * dpb, char clump, CSphString &str ); + static size_t parseSQLDA ( XSQLDA * xsqlda, char * buff ); +}; + +#endif // USE_FIREBIRD + #if USE_ODBC struct CSphSourceParams_ODBC: CSphSourceParams_SQL { diff --git a/src/sphinxrt.cpp b/src/sphinxrt.cpp index 0672729ae..5084b1ddd 100644 --- a/src/sphinxrt.cpp +++ b/src/sphinxrt.cpp @@ -3235,7 +3235,8 @@ void RtIndex_t::CommitReplayable ( RtSegment_t * pNewSeg, CSphVector // update stats m_tStats.m_iTotalDocuments += iNewDocs - iTotalKilled; - if ( m_tSchema.GetAttrId_FirstFieldLen()>=0 ) + // lengths might be empty on delete + if ( m_tSchema.GetAttrId_FirstFieldLen()>=0 && dLens.GetLength()>0 ) ARRAY_FOREACH ( i, m_tSchema.m_dFields ) { m_dFieldLensRam[i] += dLens[i]; diff --git a/src/sphinxsearch.cpp b/src/sphinxsearch.cpp index bdc0d6cb8..e205da62b 100644 --- a/src/sphinxsearch.cpp +++ b/src/sphinxsearch.cpp @@ -2940,22 +2940,22 @@ const ExtHit_t * ExtAndZonespanned_c::GetHitsChunk ( const ExtDoc_t * pDocs ) } } - // our special GetDocsChunk made the things so simply, that we doesn't need to care about tail hits at all. - // copy tail, while possible, unless the other child is at the end of a hit block - if ( pCurL && pCurL->m_uDocid==m_uMatchedDocid && !( pCurR && pCurR->m_uDocid==DOCID_MAX ) ) + // our special GetDocsChunk made the things so simply, that we doesn't need to care about tail hits at all. + // copy tail, while possible, unless the other child is at the end of a hit block + if ( pCurL && pCurL->m_uDocid==m_uMatchedDocid && !( pCurR && pCurR->m_uDocid==DOCID_MAX ) ) + { + while ( pCurL->m_uDocid==m_uMatchedDocid && iHitm_uDocid==m_uMatchedDocid && iHitm_uDocid==m_uMatchedDocid && !( pCurL && pCurL->m_uDocid==DOCID_MAX ) ) + } + if ( pCurR && pCurR->m_uDocid==m_uMatchedDocid && !( pCurL && pCurL->m_uDocid==DOCID_MAX ) ) + { + while ( pCurR->m_uDocid==m_uMatchedDocid && iHitm_uDocid==m_uMatchedDocid && iHitm_pNext && !pNode->m_pPrev ); + pNode->m_pNext = m_tStub.m_pNext; + pNode->m_pPrev = &m_tStub; + m_tStub.m_pNext->m_pPrev = pNode; + m_tStub.m_pNext = pNode; + + m_iCount++; + } + + void Remove ( ListNode_t * pNode ) + { + assert ( pNode->m_pNext && pNode->m_pPrev ); + pNode->m_pNext->m_pPrev = pNode->m_pPrev; + pNode->m_pPrev->m_pNext = pNode->m_pNext; + pNode->m_pNext = NULL; + pNode->m_pPrev = NULL; + + m_iCount--; + } + + int GetLength () const + { + return m_iCount; + } + + const ListNode_t * Begin () const + { + return m_tStub.m_pNext; + } + + const ListNode_t * End () const + { + return &m_tStub; + } + +private: + ListNode_t m_tStub; // stub node + int m_iCount; // elements counter +}; + #endif // _sphinxstd_ diff --git a/src/sphinxutils.cpp b/src/sphinxutils.cpp index 883eaa614..9ddda5db2 100644 --- a/src/sphinxutils.cpp +++ b/src/sphinxutils.cpp @@ -195,21 +195,21 @@ static bool sphWildcardMatchRec ( const T1 * sString, const T2 * sPattern ) return p[1]=='\0'; } - // short-circuit trailing star - if ( !*p ) - return true; + // short-circuit trailing star + if ( !*p ) + return true; - // so our wildcard expects a real character - // scan forward for its occurrences and recurse - for ( ;; ) - { - if ( !*s ) - return false; - if ( *s==*p && sphWildcardMatchRec ( s+1, p+1 ) ) - return true; - s++; - } - break; + // so our wildcard expects a real character + // scan forward for its occurrences and recurse + for ( ;; ) + { + if ( !*s ) + return false; + if ( *s==*p && sphWildcardMatchRec ( s+1, p+1 ) ) + return true; + s++; + } + break; default: // default case, strict match @@ -511,6 +511,10 @@ static KeyDesc_t g_dKeysSource[] = { "csvpipe_attr_json", KEY_LIST, NULL }, { "csvpipe_field_string", KEY_LIST, NULL }, { "csvpipe_delimiter", 0, NULL }, +#if USE_FIREBIRD + { "sql_charset", 0, NULL }, + { "sql_role", 0, NULL }, +#endif { NULL, 0, NULL } }; @@ -1191,7 +1195,11 @@ bool CSphConfigParser::Parse ( const char * sFileName, const char * pBuffer ) if ( !sToken[0]&&!sphIsAlpha(*p)) { LOC_ERROR2 ( "named section: expected name, got '%c'", *p ); } if ( !sToken[0] ) { LOC_PUSH ( S_TOK ); LOC_BACK(); continue; } - if ( !AddSection ( m_sSectionType.cstr(), sToken ) ) break; sToken[0] = '\0'; + + if ( !AddSection ( m_sSectionType.cstr(), sToken ) ) + break; + sToken[0] = '\0'; + if ( *p==':' ) { eState = S_SECBASE; continue; } if ( *p=='{' ) { eState = S_SEC; continue; } LOC_ERROR2 ( "named section: expected ':' or '{', got '%c'", *p ); @@ -1907,18 +1915,18 @@ static void UItoA ( char** ppOutput, Uint uVal, int iBase=10, int iWidth=0, int iWidth--; } - if ( iPrec ) + if ( iPrec ) + { + while ( uLen < iPrec ) { - while ( uLen < iPrec ) - { - *pOutput++=cDigits[0]; - iPrec--; - } - iPrec = uLen-iPrec; + *pOutput++=cDigits[0]; + iPrec--; } + iPrec = uLen-iPrec; + } - while ( pRes < CBuf+uMaxIndex-iPrec ) - *pOutput++ = *++pRes; + while ( pRes < CBuf+uMaxIndex-iPrec ) + *pOutput++ = *++pRes; }