if [ "${FSX_SHUTILS+x}" = "x" ]; then
    return 0
fi
FSX_SHUTILS=''

FSX_SHUTILS_REL_PATH="${BASH_SOURCE[0]}"
if [ "x${FSX_SHUTILS_REL_PATH}" = "x" ]; then
    FSX_SHUTILS_REL_PATH="${(%):-%N}"
fi
FSX_SHUTILS_PATH="$(realpath -e "${FSX_SHUTILS_REL_PATH}")"

deploy_dir="$(dirname "$(dirname "$(cd -- "$(dirname "${FSX_SHUTILS_PATH}")" > /dev/null 2>&1; pwd)")")"
. "${deploy_dir}/tools/third/shflags"

unset FSX_SHUTILS_REL_PATH FSX_SHUTILS_PATH
script_name="$(basename "${FLAGS_PARENT:-$0}")"
supported_instances=("vepfs" "tos" "nas" "efs" "cfs")

mount_lock_fd=6001

bfutils_common_socket_poller_num() {
    echo "${FLAGS_common_socket_poller_num:-0}"
}

bfutils_vhost_user_fast_fail_on_umount_enabled() {
    [[ "${FSX_VHOST_USER_FAST_FAIL_ON_UMOUNT:-}" =~ ^(1|[Tt][Rr][Uu][Ee])$ ]]
}

bfutils_append_fusedaemon_runtime_args() {
    local __var_name=$1
    local __socket_poller_num="$(bfutils_common_socket_poller_num)"
    if [ -n "${FSX_AGENT_NAME:-}" ]; then
        eval "${__var_name}+=(\"--fusedaemon_metrics_agent_name=${FSX_AGENT_NAME}\")"
    fi
    if [ -n "${FSX_AGENT_GROUP:-}" ]; then
        eval "${__var_name}+=(\"--fusedaemon_metrics_agent_group=${FSX_AGENT_GROUP}\")"
    fi
    if [ -n "${FSX_AGENT_RUNTIME_TAG:-}" ]; then
        eval "${__var_name}+=(\"--fusedaemon_metrics_agent_runtime_tag=${FSX_AGENT_RUNTIME_TAG}\")"
    fi
    if [ "${__socket_poller_num}" -gt 0 ]; then
        eval "${__var_name}+=(\"--common_socket_poller_num=${__socket_poller_num}\")"
    fi
    if bfutils_vhost_user_fast_fail_on_umount_enabled; then
        eval "${__var_name}+=(\"--fusedaemon_vhost_user_fast_fail_on_umount\")"
    fi
    if [ -n "${FSX_COMMON_METRICS_OVERRIDE_TAGS:-}" ]; then
        eval "${__var_name}+=(\"--common_metrics_override_tags=${FSX_COMMON_METRICS_OVERRIDE_TAGS}\")"
    fi
    if [[ "${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS:-0}" =~ ^[0-9]+$ ]] && [ "${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS:-0}" -gt 0 ]; then
        eval "${__var_name}+=(\"--common_metrics_purge_metrics_timeout_ms=${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS}\")"
    fi
}

bfutils_set_exit_status() {
    return "$1"
}

declare -A export_error_codes=(
    [CHECK_SERVICE_CGROUP_BASE_PATH_FAIL]="CHECK_SERVICE_CGROUP_BASE_PATH_FAIL"
    [CHECK_SERVICE_CGROUP_PATH_FAIL]="CHECK_SERVICE_CGROUP_PATH_FAIL"
    [CHECK_SERVICE_FAIL]="CHECK_SERVICE_FAIL"
    [CHECK_SERVICE_FAIL_FREQUENT_RESTART]="CHECK_SERVICE_FAIL_FREQUENT_RESTART"
    [CONF_DIR_EMPTY]="CONF_DIR_EMPTY"
    [CONF_DIR_INVALID]="CONF_DIR_INVALID"
    [CONF_DIR_MISMATCH_RUNNING]="CONF_DIR_MISMATCH_RUNNING"
    [CONF_DIR_NOT_EMPTY]="CONF_DIR_NOT_EMPTY"
    [CONF_LOG_PATH_EMPTY]="CONF_LOG_PATH_EMPTY"
    [CONF_PATH_EMPTY]="CONF_PATH_EMPTY"
    [CREATE_CONF_DIR_FAILED]="CREATE_CONF_DIR_FAILED"
    [CREATE_LOG_DIR_FAILED]="CREATE_LOG_DIR_FAILED"
    [CREATE_LOG_DIR_FAILED_WHEN_INIT]="CREATE_LOG_DIR_FAILED_WHEN_INIT"
    [CREATE_PREFIX_DIR_FAILED]="CREATE_PREFIX_DIR_FAILED"
    [CREATE_WORK_DIR_FAILED]="CREATE_WORK_DIR_FAILED"
    [DEPLOY_MODE_MISMATCH_RUNNING]="DEPLOY_MODE_MISMATCH_RUNNING"
    [GET_SERVICE_CGROUP_FAIL]="GET_SERVICE_CGROUP_FAIL"
    [HPVS_INSTALL_FAILED]="HPVS_INSTALL_FAILED"
    [HPVS_MAY_LOAD_FAILED]="HPVS_MAY_LOAD_FAILED"
    [HPVS_UNINSTALL_FAILED]="HPVS_UNINSTALL_FAILED"
    [LOG_DIR_INVALID]="LOG_DIR_INVALID"
    [LOG_DIR_MISMATCH_RUNNING]="LOG_DIR_MISMATCH_RUNNING"
    [MODULE_STILL_RUNNING]="MODULE_STILL_RUNNING"
    [MOUNTPOINTS_STILL_ALIVE]="MOUNTPOINTS_STILL_ALIVE"
    [MULTI_FSX_PROCESS]="MULTI_FSX_PROCESS"
    [NO_DEPLOY_MODE]="NO_DEPLOY_MODE"
    [NO_RUNNING_CONF_DIR]="NO_RUNNING_CONF_DIR"
    [NO_RUNNING_LOG_DIR]="NO_RUNNING_LOG_DIR"
    [NO_RUNNING_WORK_DIR]="NO_RUNNING_WORK_DIR"
    [PREFIX_DIR_NOT_EMPTY]="PREFIX_DIR_NOT_EMPTY"
    [PREFIX_INVALID]="PREFIX_INVALID"
    [PREFIX_MISMATCH_RUNNING]="PREFIX_MISMATCH_RUNNING"
    [RESTART_SERVICE_FAILED]="RESTART_SERVICE_FAILED"
    [SERVICE_INACTIVE]="SERVICE_INACTIVE"
    [STOP_FUSEDAEMON_FAILED]="STOP_FUSEDAEMON_FAILED"
    [STOP_SERVICE_FAILED]="STOP_SERVICE_FAILED"
    [WORK_DIR_INVALID]="WORK_DIR_INVALID"
    [WORK_DIR_MISMATCH_RUNNING]="WORK_DIR_MISMATCH_RUNNING"
    [FUSEDAEMON_STILL_ALIVE]="FUSEDAEMON_STILL_ALIVE"
    [FUSEMONITOR_NOT_RUNNING]="FUSEMONITOR_NOT_RUNNING"
    [FUSEDAEMON_NOT_RUNNING]="FUSEDAEMON_NOT_RUNNING"
    [FUSEMONITOR_UDS_ERROR]="FUSEMONITOR_UDS_ERROR"
    [FUSEDAEMON_UDS_ERROR]="FUSEDAEMON_UDS_ERROR"
)

run_fusetool() {
    local _run_args="$*"
    # fuse tool bin path
    local _deploy_dir=${FLAGS_prefix}
    # fuse tool conf path
    local _fsx_conf_dir=${FLAGS_conf_dir}
    # fuse tool work path
    local _fsx_work_dir=${FLAGS_work_dir}

    pushd ${_deploy_dir}/bin > /dev/null 2>&1
    __bfutils_return=$(./fusetool ${_run_args} -- --common_conf_dir=${_fsx_conf_dir} --common_work_dir=${_fsx_work_dir} 2>&1)
    local _error_code=$?
    popd > /dev/null 2>&1

    if [ ${_error_code} -ne 0 ]; then
        bfutils_log WARN "Failed to run fusetool, error: ${_error_code}, error_msg: ${__bfutils_return}"
    fi

    return ${_error_code}
}

#
# Golang-like defer functions.
#
# Ugly but do work
#
# Example: bfutils_defer command [args]
#
# Notice: unlike golang, bfutils_defer previously set in the same scope would also be executed
#

_bfutils_run_deferred() {
    if [ "${__bfutils_defer_skip+x}" = "x" ]; then
        unset __bfutils_defer_skip
        return
    fi

    local _bfutils_deferred_depth="BFUTILS_DEFERS_${$}_${#FUNCNAME[@]}_${FUNCNAME[1]}"
    eval "local _bfutils_deferred_count=\"\${#${_bfutils_deferred_depth}[@]}\""
    if [ ${_bfutils_deferred_count} -eq 0 ]; then
        unset "${_bfutils_deferred_depth}"
        return
    fi

    local _bfutils_run_deferred_opt=$-
    set +e
    for (( i=${_bfutils_deferred_count} - 1; i >= 0; i-- )); do
        eval "eval \"\$(echo -e \${${_bfutils_deferred_depth}[i]})\""
    done
    [[ "$_bfutils_run_deferred_opt" == *e* ]] && set -e

    return
}
_bfutils_defer() {
    local _bfutils_deferred_depth="BFUTILS_DEFERS_${$}_${#FUNCNAME[@]}_${FUNCNAME[1]}"
    eval "local _bfutils_deferred_count=\"\${#${_bfutils_deferred_depth}[@]}\""
    if [ ${_bfutils_deferred_count} -eq 0 ]; then
        eval "${_bfutils_deferred_depth}=()"
    fi
    eval "${_bfutils_deferred_depth}+=(\"\$(printf '%q ' \"\$@\")\")"
    return
}
# This has to be an alias so that the `trap ... RETURN` runs appropriately.
shopt -s expand_aliases
alias bfutils_defer='
trap _bfutils_run_deferred EXIT RETURN;
_bfutils_defer'

bfutils_defer_skip() {
    local _bfutils_orig_err_code=${1:-$?}
    __bfutils_defer_skip=''
    return ${_bfutils_orig_err_code}
}
bfutils_defer_clean() {
    local _bfutils_deferred_depth="BFUTILS_DEFERS_${$}_${#FUNCNAME[@]}_${FUNCNAME[1]}"
    eval "${_bfutils_deferred_depth}=()"
    return
}

#
# Logging functions.
#

__bfutils_log_level_default=${FLAGS_LEVEL_INFO}
__bfutils_log_level=${__bfutils_log_level_default} # Current logging level.
__bfutils_console_log_level=${FLAGS_LEVEL_ERROR}
DEFINE_string 'log_dir' '/var/log/fsx' 'The target directory for fsx log'
DEFINE_integer 'minloglevel' ${__bfutils_log_level_default} 'The log level for this script'
DEFINE_boolean 'pool_mode' ${FLAGS_FALSE} 'Enable fixed daemon pools (FAAS only); also FSX_POOL_MODE=true'
. "${deploy_dir}/tools/shpack/pool.sh"

# Return valid getopt options using currently defined list of long options.
#
# This function builds a proper getopt option string for short (and long)
# options, using the current list of long options for reference.
#
# Args:
#   _bfutils_log_severity_name: string: desired logging level
# Output:
#   string:
# Returns:
#   nothing
bfutils_log() {
    local _bfutils_log_severity_name="$1"
    eval "local _bfutils_log_severity=\"\${FLAGS_LEVEL_${_bfutils_log_severity_name}:-}\""
    if [ "${_bfutils_log_severity}" -lt "${__bfutils_log_level}" ]; then
        return
    fi

    shift
    local _bfutils_log_file="${FLAGS_log_dir}/${script_name}"
    local _bfutils_log_caller="$(caller | awk '{system("basename -z " $2); print ":"$1}' | tr -d '\0')"
    local _bfutils_log_title="[$(date '+%F %T.%N') ${script_name}($$) ${_bfutils_log_caller}] ${_bfutils_log_severity_name}: "
    local _bfutils_log_content="$*"
    local _bfutils_log_tail=" "

    flock -ew 1 "${_bfutils_log_file}" echo "$(echo -ne "${_bfutils_log_title}" && echo -nE "${_bfutils_log_content}" && echo -ne "${_bfutils_log_tail}"; echo)" 2>/dev/null >> "${_bfutils_log_file}"
    if [ $? -ne 0 ]; then
        _bfutils_log_tail=" (logging to \"$(printf '%q' "${_bfutils_log_file}")\" failed)"
    fi
    { echo -ne "${_bfutils_log_title}" && echo -nE "$@" && echo -ne "${_bfutils_log_tail}"; echo; } | tee /dev/kmsg > /dev/null 2>&1 # native timestamp not accurate so

    if [ "${_bfutils_log_severity}" -ge "${__bfutils_console_log_level}" ]; then
        >&2 echo -nE "$@"; >&2 echo
    fi

    if [ "${_bfutils_log_severity}" -eq "${FLAGS_LEVEL_FATAL}" ]; then
        exit ${__bfutils_fatal_log_errno:-${FLAGS_ERROR}}
    fi
    return
}

# Get the logging level.
bfutils_logging_level() {
    return "${__bfutils_log_level}"
}

# Set the logging level by overriding the `__bfutils_log_level` variable.
#
# Args:
#   _bfutils_log_severity: integer: new logging level
# Returns:
#   nothing
bfutils_set_logging_level() {
    if [ $# -ne 1 ]; then
        bfutils_log FATAL "${FUNCNAME}($*): logging level missing"
    fi
    local _bfutils_log_severity="$1"
    if ! [ "${_bfutils_log_severity}" -ge "${FLAGS_LEVEL_DEBUG}" -a "${_bfutils_log_severity}" -le "${FLAGS_LEVEL_FATAL}" ]; then
        bfutils_log FATAL "Invalid logging level \"${_bfutils_log_severity}\" specified."
    fi
    __bfutils_log_level="${_bfutils_log_severity}"
}

alias bfutils_exit_if_err='__bfutils_ole=$?; [ ${__bfutils_ole} -eq 0 ] || { __bfutils_fatal_log_errno=${__bfutils_ole}; bfutils_log WARN "${__bfutils_ole}"; }'
alias bfutils_exit='bfutils_exit_if_err; exit ${__bfutils_ole}'
alias bfutils_return_if_err='__bfutils_ole=$?; [ ${__bfutils_ole} -eq 0 ] || { bfutils_log WARN "${__bfutils_ole}"; return ${__bfutils_ole}; }'
alias bfutils_return='bfutils_return_if_err; return ${__bfutils_ole}'

bfutils_log_shrink() {
    local last_errno=$1
    local _bfutils_log_file="${FLAGS_log_dir}/${script_name}"
    if ! [ -f "${_bfutils_log_file}" ]; then
        return $last_errno
    fi
    local _bfutils_log_tmp_file="${_bfutils_log_file}.tmp.${BASHPID:-$$}"
    # Log shrinking is best effort: it must not change the command result when
    # concurrent fsx invocations replace the same log file.
    if ! tail -n 100000 "${_bfutils_log_file}" 2>/dev/null > "${_bfutils_log_tmp_file}"; then
        rm -f "${_bfutils_log_tmp_file}" 2>/dev/null
        return $last_errno
    fi
    if ! mv -f "${_bfutils_log_tmp_file}" "${_bfutils_log_file}" 2>/dev/null; then
        rm -f "${_bfutils_log_tmp_file}" 2>/dev/null
    fi
    return $last_errno
}

bfutils_get_mp_json() {
    local __instances=$1
    local __prefix=$2
    local __work_dir=$3

    local __json_files=""
    local instance
    local json_file
    local record_dir
    __bfutils_return=""
    for instance in ${__instances}; do
        json_file="${__work_dir}/conf/${instance}/${instance}.json"
        if [ -f "${json_file}" ]; then
            __json_files="${__json_files} ${json_file}"
        fi
        record_dir="${__work_dir}/conf/${instance}/mounts"
        if [ -d "${record_dir}" ]; then
            # Per-mount records end with .json. Incomplete .json.tmp files are intentionally
            # excluded so this offline view matches fusedaemon recovery behavior.
            for json_file in "${record_dir}/${instance}_"*.json; do
                if [ -f "${json_file}" ]; then
                    __json_files="${__json_files} ${json_file}"
                fi
            done
        fi
    done
    bfutils_log INFO "Running get_mp_json with instances: ${__instances} json files: ${__json_files}"
    if [ -z "${__json_files}" ]; then
        bfutils_log INFO "No ${__instances} json files found."
        __bfutils_return="[]"
        return 0
    fi
    if command -v python3 >/dev/null 2>&1; then
        __bfutils_return=$(python3 ${__prefix}/tools/merge_json_arrays.py ${__json_files}); bfutils_return_if_err
    elif perl -e 'use JSON::PP;' >/dev/null 2>&1; then
        __bfutils_return=$(perl ${__prefix}/tools/merge_json_arrays.pl ${__json_files}); bfutils_return_if_err
    else
        bfutils_log WARN "No python3 or perl JSON:PP found, try cat json files"
        __bfutils_return=$(cat ${__json_files}); bfutils_return_if_err
    fi
}

bfutils_check_mp_in_json() {
    # check if target mountpoint is in mp_json
    local __mountpoint=$1
    local __prefix=$2
    local __work_dir=$3

    local _check_result="not found"
    for instance in "${supported_instances[@]}"; do
        bfutils_get_mp_json "${instance}" "${__prefix}" "${__work_dir}"; bfutils_return_if_err
        if echo "${__bfutils_return}" | grep -F "\\\"mountpoint\\\":\\\"${__mountpoint}\\\"" > /dev/null; then
            _check_result="found"
            break
        fi
    done

    __bfutils_return="${_check_result}"
    bfutils_log INFO "Check mountpoint ${__mountpoint} in ${supported_instances[*]} json files, result: ${_check_result}"

    return 0
}

bfutils_check_umount_process() {
    # 0: found, 1: not found
    local _mountpoint="${1}"
    local _escaped_mountpoint

    if [ -z "${_mountpoint}" ]; then
        return 1
    fi

    _escaped_mountpoint=$(printf '%q' "${_mountpoint}")
    pgrep -f "^umount ${_escaped_mountpoint}$" >/dev/null 2>&1

    return $?
}

bfutils_get_log_last_error_msg() {
    local _bfutils_log_file="${FLAGS_log_dir}/${script_name}"
    local _seek_count=20
    local _lines
    local _last_match=""
    local _error_code

    mapfile -t _lines < <(tail -n "${_seek_count}" "${_bfutils_log_file}" | grep "ERROR:")

    for (( idx=${#_lines[@]}-1 ; idx>=0 ; idx-- )); do
        local line="${_lines[idx]}"
        for _error_code in "${export_error_codes[@]}"; do
            if [[ "$line" == *"${_error_code}"* ]]; then
                echo "${_error_code}"
                return 0
            fi
        done
    done

    echo "unknown"
}

bfutils_get_mountpoints_json() {
    local prefix_path=$1
    local instance=$2
    local work_path=$3

    # Keep all control-plane checks compatible with both the legacy per-instance
    # array and the per-mount JSON records used by newer fusedaemon versions.
    bfutils_get_mp_json "${instance}" "${prefix_path}" "${work_path}"; bfutils_return_if_err

    return 0
}

bfutils_check_mountpoints() {
    local prefix_path=$1
    local instance=$2
    local work_path=$3

    bfutils_get_mountpoints_json "${prefix_path}" "${instance}" "${work_path}"; bfutils_return_if_err
    if [ -n "${__bfutils_return}" ] && [ "x${__bfutils_return}" != "x[]" ]; then
        bfutils_log WARN fsx still has mounted $instance mountpoints, json: ${__bfutils_return}
        return 17 # EEXIST
    fi

    bfutils_log INFO fsx has no ${instance} mountpoints
    return 0
}


bfutils_check_native_fuse_mountpoints() {
    local prefix_path=$1
    local instance=$2
    local work_path=$3

    bfutils_get_mountpoints_json "${prefix_path}" "${instance}" "${work_path}"; bfutils_return_if_err
    if echo "$__bfutils_return" | grep -E 'native_fuse=(1|true)' > /dev/null; then
        bfutils_log ERROR fsx has mounted native_fuse $instance mountpoints
        return 17 # EEXIST
    fi

    bfutils_log INFO fsx has no native_fuse ${instance} mountpoints
    return 0
}

bfutils_get_error_code() {
    local error_type=$1

    case "$error_type" in
        lock_error)
            echo 255
            ;;
        backup_conf_error)
            echo 254
            ;;
        update_inner_prepare)
            echo 253
            ;;
        check_mount_error)
            echo 252
            ;;
        *)
            echo "Unknown error type"
            return 22 # ENIVAL
            ;;
    esac
}

# ${FLAGS_prefix}/*/xxx
# 进程名后必须为空格，避免误判
bfutils_running_processes() {
    __bfutils_return="$(ps ax | grep -E "/fuse(daemon|monitor)(_run)?[[:space:]]" | awk '{print $1}')"
    bfutils_return
}

bfutils_running_daemons() {
    local instance=$1
    if [ -n "${instance}" ]; then
        __bfutils_return="$(ps ax | grep -E "/fuse(daemon)(_run)?[[:space:]]" | grep "${instance}" | awk '{print $1}')"
    else
        __bfutils_return="$(ps ax | grep -E "/fuse(daemon)(_run)?[[:space:]]" | awk '{print $1}')"
    fi
    bfutils_return
}

bfutils_running_monitor() {
    __bfutils_return="$(ps ax | grep -E "/fusemonitor[[:space:]]" | awk '{print $1}')"
    bfutils_return
}

bfutils_running_path() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found unified fsx deployment path

    bfutils_running_processes; bfutils_return_if_err
    local _bfutils_running_path_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_path_return=''
    local _bfutils_running_path_errno=0
    while read -r _bfutils_running_path_pid; do
        [ -n "${_bfutils_running_path_pid}" ] || continue
        local _bfutils_running_path_exe="$(readlink "/proc/${_bfutils_running_path_pid}/exe" 2>/dev/null)" || continue
        _bfutils_running_path_exe="${_bfutils_running_path_exe% (deleted)}"
        local _bfutils_running_path_binary="${_bfutils_running_path_exe##*/}"
        case "${_bfutils_running_path_binary}" in
            fusedaemon|fusemonitor)
                ;;
            *)
                # Ignore shell wrappers such as fusemonitor_run and fusedaemon_run.
                continue
                ;;
        esac
        local _bfutils_running_path_current="$(dirname "$(dirname "${_bfutils_running_path_exe}")")"
        bfutils_log INFO "Found PID ${_bfutils_running_path_pid} binary @\"${_bfutils_running_path_exe}\", deployed @\"${_bfutils_running_path_current}\""
        if [ -z "${_bfutils_running_path_return}" ]; then
            _bfutils_running_path_return="${_bfutils_running_path_current}"
            continue
        fi
        if [ "${_bfutils_running_path_return}" != "${_bfutils_running_path_current}" ]; then
            _bfutils_running_path_errno=95 # ENOTSUP
            bfutils_log ERROR "PID ${_bfutils_running_path_pid} has a different deployment path between ${_bfutils_running_path_return} and ${_bfutils_running_path_current}."
            continue
        fi
    done <<< "${_bfutils_running_path_process_list}"

    bfutils_set_exit_status "${_bfutils_running_path_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_path_return}" ]; then
        bfutils_log INFO "No running fsx found"
    else
        bfutils_log INFO "Found running fsx deployed @\"${_bfutils_running_path_return}\""
    fi
    __bfutils_return="${_bfutils_running_path_return}"
    bfutils_return
}

bfutils_running_bin_path() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found fsx bin path

    bfutils_running_monitor; bfutils_return_if_err
    local _bfutils_running_bin_path_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_bin_path_return=''
    local _bfutils_running_bin_path_errno=0
    while read -r _bfutils_running_bin_path_pid; do
        [ -n "${_bfutils_running_bin_path_pid}" ] || continue
        while IFS="" read -r _bfutils_running_bin_path_cmdline_arg; do
            [ -n "${_bfutils_running_bin_path_cmdline_arg}" ] || continue
            # FIXME:
            local _bfutils_running_bin_path_path="$(awk '{print substr($0, index($0, "=") + 1)}' <<< "${_bfutils_running_bin_path_cmdline_arg}")"
            bfutils_log DEBUG "Found PID ${_bfutils_running_bin_path_pid} conf @\"${_bfutils_running_bin_path_path}\""
            if [ -z "${_bfutils_running_bin_path_return}" ]; then
                _bfutils_running_bin_path_return="${_bfutils_running_bin_path_path}"
                continue
            fi
            if [ "${_bfutils_running_bin_path_return}" != "${_bfutils_running_bin_path_path}" ]; then
                _bfutils_running_bin_path_errno=95 # ENOTSUP
                bfutils_log ERROR "PID ${_bfutils_running_bin_path_pid} has a different conf path than the first found one."
                break
            fi
        done <<< "$(cat "/proc/${_bfutils_running_bin_path_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep common_deploy_dir)"
        if [ ${_bfutils_running_bin_path_errno} -ne 0 ]; then
            break
        fi
    done <<< "${_bfutils_running_bin_path_process_list}"

    bfutils_set_exit_status "${_bfutils_running_bin_path_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_bin_path_return}" ]; then
        bfutils_log INFO "No running fsx found"
    else
        bfutils_log INFO "Found running fsx prefix @\"${_bfutils_running_bin_path_return}\""
    fi
    __bfutils_return="${_bfutils_running_bin_path_return}"
    bfutils_return
}


bfutils_running_conf_path() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found fsx conf path

    bfutils_running_monitor; bfutils_return_if_err
    local _bfutils_running_conf_path_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_conf_path_return=''
    local _bfutils_running_conf_path_errno=0
    while read -r _bfutils_running_conf_path_pid; do
        [ -n "${_bfutils_running_conf_path_pid}" ] || continue
        while IFS="" read -r _bfutils_running_conf_path_cmdline_arg; do
            [ -n "${_bfutils_running_conf_path_cmdline_arg}" ] || continue
            # FIXME:
            local _bfutils_running_conf_path_path="$(awk '{print substr($0, index($0, "=") + 1)}' <<< "${_bfutils_running_conf_path_cmdline_arg}")"
            bfutils_log DEBUG "Found PID ${_bfutils_running_conf_path_pid} conf @\"${_bfutils_running_conf_path_path}\""
            if [ -z "${_bfutils_running_conf_path_return}" ]; then
                _bfutils_running_conf_path_return="${_bfutils_running_conf_path_path}"
                continue
            fi
            if [ "${_bfutils_running_conf_path_return}" != "${_bfutils_running_conf_path_path}" ]; then
                _bfutils_running_conf_path_errno=95 # ENOTSUP
                bfutils_log ERROR "PID ${_bfutils_running_conf_path_pid} has a different conf path than the first found one."
                break
            fi
        done <<< "$(cat "/proc/${_bfutils_running_conf_path_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep conf_dir)"
        if [ ${_bfutils_running_conf_path_errno} -ne 0 ]; then
            break
        fi
    done <<< "${_bfutils_running_conf_path_process_list}"

    bfutils_set_exit_status "${_bfutils_running_conf_path_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_conf_path_return}" ]; then
        bfutils_log INFO "No running fsx found"
    else
        bfutils_log INFO "Found running fsx conf @\"${_bfutils_running_conf_path_return}\""
    fi
    __bfutils_return="${_bfutils_running_conf_path_return}"
    bfutils_return
}


bfutils_running_work_path() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found fsx work path

    bfutils_running_monitor; bfutils_return_if_err
    local _bfutils_running_work_path_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_work_path_return=''
    local _bfutils_running_work_path_errno=0
    while read -r _bfutils_running_work_path_pid; do
        [ -n "${_bfutils_running_work_path_pid}" ] || continue
        while IFS="" read -r _bfutils_running_work_path_cmdline_arg; do
            [ -n "${_bfutils_running_work_path_cmdline_arg}" ] || continue
            # FIXME:
            local _bfutils_running_work_path_path="$(awk '{print substr($0, index($0, "=") + 1)}' <<< "${_bfutils_running_work_path_cmdline_arg}")"
            bfutils_log DEBUG "Found PID ${_bfutils_running_work_path_pid} work @\"${_bfutils_running_work_path_path}\""
            if [ -z "${_bfutils_running_work_path_return}" ]; then
                _bfutils_running_work_path_return="${_bfutils_running_work_path_path}"
                continue
            fi
            if [ "${_bfutils_running_work_path_return}" != "${_bfutils_running_work_path_path}" ]; then
                _bfutils_running_work_path_errno=95 # ENOTSUP
                bfutils_log ERROR "PID ${_bfutils_running_work_path_pid} has a different work path than the first found one."
                break
            fi
        done <<< "$(cat "/proc/${_bfutils_running_work_path_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep work_dir)"
        if [ ${_bfutils_running_work_path_errno} -ne 0 ]; then
            break
        fi
    done <<< "${_bfutils_running_work_path_process_list}"

    bfutils_set_exit_status "${_bfutils_running_work_path_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_work_path_return}" ]; then
        bfutils_log INFO "No running fsx work dir found, is old version"
    else
        bfutils_log INFO "Found running fsx work @\"${_bfutils_running_work_path_return}\""
    fi
    __bfutils_return="${_bfutils_running_work_path_return}"
    bfutils_return
}

bfutils_running_log_path() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found fsx log path

    bfutils_running_monitor; bfutils_return_if_err
    local _bfutils_running_log_path_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_log_path_return=''
    local _bfutils_running_log_path_errno=0
    while read -r _bfutils_running_log_path_pid; do
        [ -n "${_bfutils_running_log_path_pid}" ] || continue
        while IFS="" read -r _bfutils_running_log_path_cmdline_arg; do
            [ -n "${_bfutils_running_log_path_cmdline_arg}" ] || continue
            # FIXME:
            local _bfutils_running_log_path_path="$(awk '{print substr($0, index($0, "=") + 1)}' <<< "${_bfutils_running_log_path_cmdline_arg}")"
            bfutils_log DEBUG "Found PID ${_bfutils_running_log_path_pid} log @\"${_bfutils_running_log_path_path}\""
            if [ -z "${_bfutils_running_log_path_return}" ]; then
                _bfutils_running_log_path_return="${_bfutils_running_log_path_path}"
                continue
            fi
            if [ "${_bfutils_running_log_path_return}" != "${_bfutils_running_log_path_path}" ]; then
                _bfutils_running_log_path_errno=95 # ENOTSUP
                bfutils_log ERROR "PID ${_bfutils_running_log_path_pid} has a different log path than the first found one."
                break
            fi
        done <<< "$(cat "/proc/${_bfutils_running_log_path_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep common_log_dir)"
        if [ ${_bfutils_running_log_path_errno} -ne 0 ]; then
            break
        fi
    done <<< "${_bfutils_running_log_path_process_list}"

    bfutils_set_exit_status "${_bfutils_running_log_path_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_log_path_return}" ]; then
        bfutils_log DEBUG "No running fsx log path found, use default @\"${FLAGS_log_dir}\""
    else
        bfutils_log DEBUG "Found running fsx log @\"${_bfutils_running_log_path_return}\""
    fi
    __bfutils_return="${_bfutils_running_log_path_return}"
    bfutils_return
}


bfutils_running_deploy_mode() {
    bfutils_defer_clean
    bfutils_defer __bfutils_return='/error/in/fsx/path/parse'
    bfutils_defer bfutils_log ERROR Failed to found fsx deploy mode

    bfutils_running_monitor; bfutils_return_if_err
    local _bfutils_running_deploy_mode_process_list="${__bfutils_return}" # Operate on empty list to simplify
    local _bfutils_running_deploy_mode_return=''
    local _bfutils_running_deploy_mode_errno=0
    while read -r _bfutils_running_deploy_mode_pid; do
        [ -n "${_bfutils_running_deploy_mode_pid}" ] || continue
        while IFS="" read -r _bfutils_running_deploy_mode_cmdline_arg; do
            [ -n "${_bfutils_running_deploy_mode_cmdline_arg}" ] || continue
            # FIXME:
            local _bfutils_running_deploy_mode_path="$(awk '{print substr($0, index($0, "=") + 1)}' <<< "${_bfutils_running_deploy_mode_cmdline_arg}")"
            bfutils_log DEBUG "Found PID ${_bfutils_running_deploy_mode_pid} deploy mode @\"${_bfutils_running_deploy_mode_path}\""
            if [ -z "${_bfutils_running_deploy_mode_return}" ]; then
                _bfutils_running_deploy_mode_return="${_bfutils_running_deploy_mode_path}"
                continue
            fi
            if [ "${_bfutils_running_deploy_mode_return}" != "${_bfutils_running_deploy_mode_path}" ]; then
                _bfutils_running_deploy_mode_errno=95 # ENOTSUP
                bfutils_log ERROR "PID ${_bfutils_running_deploy_mode_pid} has a different deploy mode than the first found one."
                break
            fi
        done <<< "$(cat "/proc/${_bfutils_running_deploy_mode_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep common_deploy_mode)"
        if [ ${_bfutils_running_deploy_mode_errno} -ne 0 ]; then
            break
        fi
    done <<< "${_bfutils_running_deploy_mode_process_list}"

    bfutils_set_exit_status "${_bfutils_running_deploy_mode_errno}"; bfutils_return_if_err
    bfutils_defer_clean
    if [ -z "${_bfutils_running_deploy_mode_return}" ]; then
        bfutils_log INFO "No running fsx deploy mode found, use default"
        _bfutils_running_deploy_mode_return="${__flags_deploy_mode_default}"
    else
        bfutils_log INFO "Found running fsx deploy mode @\"${_bfutils_running_deploy_mode_return}\""
    fi
    __bfutils_return="${_bfutils_running_deploy_mode_return}"
    bfutils_return
}

bfutils_binary_copy() {
    local _deploy_dir=$2
    bfutils_defer_clean

    bfutils_log INFO Copying binary files from "\"${_deploy_dir}\"" to "\"${1}\""

    bfutils_defer bfutils_log ERROR Failed to copy fsx from @${_deploy_dir}@ to @${1}@

    while read -r _bfutils_binary_copy_entry_path; do
        [ -d "${1}/${_bfutils_binary_copy_entry_path}" ] || mkdir -p "${1}/${_bfutils_binary_copy_entry_path}"; bfutils_return_if_err
    done <<< "$(find "${_deploy_dir}" -mindepth 1 -type d | perl -pe s{"^${_deploy_dir}"}{'.'})"; bfutils_return_if_err
    find "${_deploy_dir}" -not -type d | perl -pe s{"^${_deploy_dir}"}{'.'} | xargs -I{} cp -a --remove-destination "${_deploy_dir}/{}" "${1}/{}"; bfutils_return_if_err

    bfutils_defer_clean
    return 0
}

bfutils_entry_remove() {
    local _bfutils_entry_remove_option="$1";
    shift
    local _bfutils_entry_remove_err_msg="$(rm "-${_bfutils_entry_remove_option}" "${@}" 2>&1)"
    local _bfutils_entry_remove_errno=$?
    if [ ${_bfutils_entry_remove_errno} -eq 0 ]; then
        bfutils_log DEBUG "$(printf '"%s" ' "$@")"removed.
    else
        bfutils_log ERROR "${_bfutils_entry_remove_err_msg}"
    fi
    return ${_bfutils_entry_remove_errno}
}

bfutils_binary_remove() {
    bfutils_defer_clean

    bfutils_log INFO Removing binary files from "\"${*}\""

    bfutils_defer bfutils_log ERROR Failed to remove fsx from @${*}@

    # Before install, make sure there is no extra data in the directory.
    # When uninstall, clear the data in the fsx directory.
    bfutils_entry_remove vrf "${1}"; bfutils_return_if_err

    bfutils_defer_clean
    return 0
}

bfutils_detect_os_type() {
    __bfutils_return="unknown"

    if command -v apt >/dev/null 2>&1; then
        __bfutils_return="debian"
    elif command -v yum >/dev/null 2>&1 || command -v dnf >/dev/null 2>&1; then
        __bfutils_return="centos"
    fi

    bfutils_return
}

bfutils_get_ca_bundle_path() {
    local -a _search_paths=()
    local _ca_path=

    bfutils_defer_clean
    bfutils_defer "bfutils_log ERROR \"Failed to find CA bundle"

    if [ -n "${FSX_CA_BUNDLE_INFO}" ]; then
        bfutils_log INFO "Using existing FSX_CA_BUNDLE_INFO: ${FSX_CA_BUNDLE_INFO}"
        _ca_path="${FSX_CA_BUNDLE_INFO}"
    else
        bfutils_detect_os_type
        local _os_type="${__bfutils_return}"

        case "${_os_type}" in
            debian)
                _ca_path="/etc/ssl/certs/ca-certificates.crt"
                ;;
            centos)
                _ca_path="/etc/pki/tls/certs/ca-bundle.crt"
                ;;
            *)
                _search_paths=(
                    "/etc/ssl/certs/ca-certificates.crt"
                    "/etc/pki/tls/certs/ca-bundle.crt"
                    "/usr/share/ssl/certs/ca-bundle.crt"
                    "/usr/local/share/certs/ca-root-nss.crt"
                    "/etc/ssl/cert.pem"
                )
                for _path in "${_search_paths[@]}"; do
                    if [ -f "${_path}" ]; then
                        _ca_path="${_path}"
                        bfutils_log INFO "Auto-detected CA path: ${_ca_path}"
                        break
                    fi
                done
                ;;
        esac
    fi
    if [ -z "${_ca_path}" ] || [ ! -f "${_ca_path}" ]; then
        bfutils_log ERROR "CA bundle path \"${_ca_path}\" not existed, ensure that the ca-certificates package is installed"
        return 2 # ENOENT
    fi

    __bfutils_return="${_ca_path}"
    bfutils_log INFO "Found CA bundle path \"${_ca_path}\""
    bfutils_defer_clean
    bfutils_return
}

bfutils_conf_copy() {
    local _conf_dir=$1
    local _deploy_dir=$2
    local _deploy_mode=$3
    local _pool_enabled=false
    local _pool_process_counts=
    local _pool_affinity_soft_limits= _pool_daemon_memory_max_bytes= _pool_memory_max_bytes=
    bfutils_pool_enabled && _pool_enabled=true
    if ${_pool_enabled}; then
        _pool_process_counts=$(bfutils_pool_process_counts) || return $?
        _pool_affinity_soft_limits=$(bfutils_pool_affinity_soft_limits) || return $?
        _pool_daemon_memory_max_bytes=$(bfutils_pool_daemon_memory_max_bytes) || return $?
        _pool_memory_max_bytes=$(bfutils_pool_memory_max_bytes) || return $?
    fi
    bfutils_defer_clean

    bfutils_log INFO Creating conf files in "\"${_conf_dir}\""
    bfutils_defer bfutils_log ERROR Failed to create fsx conf files in "\"${_conf_dir}\""
    mkdir -p "${_conf_dir}/svc" 2>/dev/null; bfutils_return_if_err

    bfutils_get_ca_bundle_path; bfutils_return_if_err
    local _ca_path="${__bfutils_return}"

    while read -r _bfutils_conf_copy_entry_path; do
        [ -d "${_conf_dir}/${_bfutils_conf_copy_entry_path}" ] || mkdir "${_conf_dir}/${_bfutils_conf_copy_entry_path}"; bfutils_return_if_err
    done <<< "$(find "${_deploy_dir}/conf" -type d | perl -pe s{"^${_deploy_dir}"}{'.'})"; bfutils_return_if_err
    while read -r _bfutils_conf_copy_entry_path; do
        cat "${_deploy_dir}/${_bfutils_conf_copy_entry_path}" |
        {
            if [ "x${FLAGS_monitor_secret_path}" != "x" ]; then
                perl -pe s{'^--fusemonitor_heartbeat_credential_path=(.*?)$'}{"--fusemonitor_heartbeat_credential_path=${FLAGS_monitor_secret_path}"}
            else
                cat
            fi
        } |
        perl -pe s{'^--auth_ca_bundle_path=(.*?)$'}{"--auth_ca_bundle_path=${_ca_path}"} |
        perl -pe s{'^--(\w+)=(.*?)\$\{FLAGS_conf_dir\}(.*)$'}{"--\$1=\$2${FLAGS_conf_dir}\$3"} |
        perl -pe s{'^--(\w+)=(.*?)\$\{FLAGS_log_dir\}(.*)$'}{"--\$1=\$2${FLAGS_log_dir}\$3"} |
        perl -pe s{'^--(\w+)=(.*?)\$\{FLAGS_prefix\}(.*)$'}{"--\$1=\$2${FLAGS_prefix}\$3"} > "${_conf_dir}/${_bfutils_conf_copy_entry_path}"; bfutils_return_if_err
        if  [[ "${_bfutils_conf_copy_entry_path}" == *fsx_global.conf ]]; then
            if [ "x${_deploy_mode}" = "xfaas" ]; then
                printf '%s\n' '--fusemonitor_enable_heartbeat=false' >> "${_conf_dir}/${_bfutils_conf_copy_entry_path}"
            fi
            if ${_pool_enabled}; then
                printf '\n--fusemonitor_enable_pool_mode=%s\n--fuseclient_pool_enabled=%s\n' "${_pool_enabled}" "${_pool_enabled}"
                printf '%s\n' "--fusemonitor_pool_process_counts=${_pool_process_counts}"
                printf '%s\n' "--fusemonitor_pool_affinity_soft_limits=${_pool_affinity_soft_limits}"
                printf '%s\n' "--fusemonitor_pool_daemon_memory_max_bytes=${_pool_daemon_memory_max_bytes}"
                printf '%s\n' "--fusemonitor_pool_memory_max_bytes=${_pool_memory_max_bytes}"
                printf '%s\n' "--common_fusemonitor_uds_path=${FSX_MONITOR_UDS:-${FLAGS_work_dir}/socket/fusemonitor.socket}"
                printf '%s\n' "--fusecontroller_uds_path=${FSX_CONTROLLER_SOCKET:-/var/run/fsx-process-monitor/fsx-process-monitor.sock}"
            fi >> "${_conf_dir}/${_bfutils_conf_copy_entry_path}"
            if [ "x${FSX_PROMETHEUS_ENDPOINT}" != "x" ]; then
                echo "--common_metrics_prometheus_listening_endpoint=${FSX_PROMETHEUS_ENDPOINT}" >> "${_conf_dir}/${_bfutils_conf_copy_entry_path}"
            fi
        fi
        if [[ "${_bfutils_conf_copy_entry_path}" == *_${_deploy_mode}.conf ]] && [[ "${_bfutils_conf_copy_entry_path}" != *eic_${_deploy_mode}.conf ]] \
            || [[ "${_bfutils_conf_copy_entry_path}" == *fsx_global.conf ]]; then
            local _conf_file="${_conf_dir}/${_bfutils_conf_copy_entry_path}"
            if [ -s ${_conf_file} ] && [ "$(tail -c1 ${_conf_file})" != "" ]; then
                # ensure conf end with \n
                echo >> "${_conf_file}"
            fi
            if [ "x${FLAGS_region_id}" != "x" ]; then
                echo "--common_region_id=${FLAGS_region_id}" >> "${_conf_file}"
            fi
            if [ "x${FLAGS_instance_id}" != "x" ]; then
                echo "--common_instance_id=${FLAGS_instance_id}" >> "${_conf_file}"
            fi
            if [ "x${FLAGS_owner_account_id}" != "x" ]; then
                echo "--common_owner_account_id=${FLAGS_owner_account_id}" >> "${_conf_file}"
            fi
            if [ ${BFUTILS_TOB_MODE} -eq ${FLAGS_FALSE} ]; then
                echo "--common_tob_mode=false" >> "${_conf_file}"
            fi
        fi
        bfutils_log INFO conf file "\"${_conf_dir}/${_bfutils_conf_copy_entry_path}\"" generated.
    done <<< "$(find "${_deploy_dir}/conf" -not -type d | perl -pe s{"^${_deploy_dir}"}{'.'})"; bfutils_return_if_err

    bfutils_defer_clean
}

bfutils_backup_clean() {
    local _work_dir=$1
    bfutils_log INFO Clean backup files in "\"${_work_dir}\""
    rm -rf "${_work_dir}/binary_backup"
    rm -rf "${_work_dir}/misc_backup"
    rm -rf "${_work_dir}/conf_backup"
}

bfutils_binary_backup() {
    local _target_path=$1
    local _work_dir=$2
    local _backup_path="${_work_dir}/binary_backup"

    bfutils_log INFO "Backup fsx binary before update, from \"${_target_path}\" to \"${_backup_path}\""
    bfutils_defer bfutils_log ERROR Failed to backup fsx binary in "\"${_backup_path}\""

    rm -rf "${_backup_path}"
    mkdir "${_backup_path}" 2>/dev/null

    bfutils_binary_copy "${_backup_path}" "${_target_path}" ; bfutils_return_if_err

    bfutils_defer_clean
}

bfutils_backup_binary_recover() {
    local _target_path=$1
    local _work_dir=$2
    local _backup_path="${_work_dir}/binary_backup"

    bfutils_log WARN "Rollback fsx binary to old version"
    bfutils_defer bfutils_log ERROR Failed to recover fsx binary in "\"${_backup_path}\""

    bfutils_binary_copy "${_target_path}" "${_backup_path}" ; bfutils_return_if_err

    bfutils_log INFO "Rollback fsx binary succeed from \"${_backup_path}\" to \"${_target_path}\""
    rm -rf "${_backup_path}"

    bfutils_defer_clean
}

bfutils_misc_backup() {
    local _conf_dir=$1
    local _work_dir=$2

    local _backup_path="${_work_dir}/misc_backup"

    bfutils_log INFO "Backup fsx misc before update to \"${_backup_path}\""
    bfutils_defer bfutils_log ERROR Failed to backup fsx binary in "\"${_backup_path}\""

    rm -rf "${_backup_path}"
    mkdir "${_backup_path}" 2>/dev/null

    # copy fusemonitor.service
    cp -rf "${_conf_dir}/svc/fusemonitor.service" "${_backup_path}"

    # copy mount.fsx
    cp -rf "/sbin/mount.fsx" "${_backup_path}"

    bfutils_defer_clean
}

bfutils_backup_misc_recover() {
    local _conf_dir=$1
    local _work_dir=$2
    local _backup_path="${_work_dir}/misc_backup"

    bfutils_log WARN "Rollback fsx misc to old version"
    bfutils_defer bfutils_log ERROR Failed to recover fsx misc from "\"${_backup_path}\""

    if [ -e "${_backup_path}/fusemonitor.service" ]; then
        cp -rf "${_backup_path}/fusemonitor.service" "${_conf_dir}/svc/"
        systemctl daemon-reload
        systemctl restart fusemonitor
    fi

    # copy mount.fsx
    if [ -e "${_backup_path}/mount.fsx" ]; then
        cp -rf "${_backup_path}/mount.fsx" "/sbin/"
    fi

    bfutils_log INFO "Rollback fsx misc succeed from \"${_backup_path}\""
    rm -rf "${_backup_path}"

    bfutils_defer_clean
}

bfutils_conf_backup() {
    local _conf_dir=$1
    local _work_dir=${2:-${_conf_dir}}
    if [ ! -d "${_conf_dir}" ]; then
        echo "Directory ${_conf_dir} does not exist."
        return $(bfutils_get_error_code "backup_conf_error")
    fi

    bfutils_log INFO Backup conf files in "\"${_work_dir}\""
    bfutils_defer bfutils_log ERROR Failed to backup fsx conf files in "\"${_work_dir}\""

    rm -rf "${_work_dir}/conf_backup"
    mkdir "${_work_dir}/conf_backup" 2>/dev/null

    # only copy conf dir's subdirs which contains file who's filename is end with ".conf"
    find "${_conf_dir}/conf" -type f -name "*.conf" | while read conf_file; do
        conf_dir=$(dirname "$conf_file")
        new_conf_dir=$(echo $conf_dir | sed 's/conf/conf_backup/')
        mkdir -p "${new_conf_dir}"
        cp -rf "$conf_file" "${new_conf_dir}/"
    done

    # copy quota file
    if [ -f "${_conf_dir}/fsx_quota.json" ]; then
        cp -rf "${_conf_dir}/fsx_quota.json" "${_conf_dir}/conf_backup/"
    fi

    bfutils_defer_clean
}

bfutils_backup_conf_recover() {
    local _conf_dir=$1
    local _work_dir=${2:-${_conf_dir}}

    bfutils_log INFO Recover backup conf files in "\"${_work_dir}\""
    bfutils_defer bfutils_log ERROR Failed to recover backup fsx conf files in "\"${_work_dir}\""

   find "${_work_dir}/conf_backup" -type f -name "*.conf" | while read conf_file; do
        conf_dir=$(dirname "$conf_file")
        old_conf_dir=$(echo $conf_dir | sed 's/conf_backup/conf/')
        cp -rf "$conf_file" "${old_conf_dir}/"
    done

    # copy quota file
    if [ -f "${_conf_dir}/conf_backup/fsx_quota.json" ]; then
        cp -rf "${_conf_dir}/conf_backup/fsx_quota.json" "${_conf_dir}/"
    fi

    rm -rf "${_conf_dir}/conf_backup"
    bfutils_defer_clean
}

bfutils_backup_conf_clean() {
    local _conf_dir=$1
    bfutils_log INFO Clean backup conf files in "\"${_conf_dir}\""
    rm -rf "${_conf_dir}/conf_backup"
}

bfutils_conf_remove() {
    bfutils_defer_clean

    bfutils_log INFO Removing conf files from "\"${1}\""
    bfutils_defer bfutils_log ERROR Failed to remove fsx conf files from @${1}@

    # Before install, make sure there is no extra data in the directory.
    # When uninstall, clear the data in the fsx directory.
    bfutils_entry_remove vrf "${1}"; bfutils_return_if_err
    bfutils_defer_clean
}

bfutils_work_remove() {
    bfutils_defer_clean

    bfutils_log INFO Removing work files from "\"${1}\""
    bfutils_defer bfutils_log ERROR Failed to remove fsx work files from @${1}@

    # Before install, make sure there is no extra data in the directory.
    # When uninstall, clear the data in the fsx directory.
    bfutils_entry_remove vrf "${1}"; bfutils_return_if_err
    bfutils_defer_clean
}

bfutils_log_remove() {
    bfutils_defer_clean

    bfutils_log INFO Removing log files from "\"${1}\""
    bfutils_defer bfutils_log ERROR Failed to remove fsx log files from @${1}@
    bfutils_entry_remove rf "${1}/fuseclient" "${1}/fusedaemon" "${1}/fusemonitor"; bfutils_return_if_err
    bfutils_entry_remove f "${1}/fsx" "${1}/install.sh" "${1}/uninstall.sh" "${1}/update.sh"; bfutils_return_if_err
    bfutils_entry_remove df "${1}"; bfutils_return_if_err
    bfutils_defer_clean
}

hpvs_rm_src_tar() {
    rm -rf $1/module/*.tar.gz
}

bfutils_component_policy_init() {
    local _deploy_mode=$1
    local _fsx_no_tob_mode="${FSX_NO_TOB_MODE:-}"

    BFUTILS_TOB_MODE=${FLAGS_TRUE}
    BFUTILS_DEPLOY_ONEAGENT=${FLAGS_TRUE}
    # install hpvs by default
    BFUTILS_INSTALL_HPVS=${FLAGS_TRUE}
    # unless user set `disable_hpvs` argument
    if [ "${FLAGS_disable_hpvs:-${FLAGS_FALSE}}" -eq "${FLAGS_TRUE}" ]; then
        BFUTILS_INSTALL_HPVS=${FLAGS_FALSE}
    fi
    # Treat hpvs install failure as fatal by default so deployment fails loudly.
    BFUTILS_IGNORE_HPVS_ERROR=${FLAGS_FALSE}
    BFUTILS_HPVS_INSTALL_PARAMS=""

    if [ "x${_deploy_mode}" = "xfaas" ]; then
        if bfutils_pool_enabled; then
            # FAAS pools run on the host; the guest image owns its HPVS driver.
            BFUTILS_INSTALL_HPVS=${FLAGS_FALSE}
            # Pool daemons and the controller publish host-side metrics through OneAgent.
            BFUTILS_DEPLOY_ONEAGENT=${FLAGS_TRUE}
        else
            # Legacy FAAS deployment installs FSX and HPVS inside the guest.
            BFUTILS_INSTALL_HPVS=${FLAGS_TRUE}
            BFUTILS_HPVS_INSTALL_PARAMS="-k"
            BFUTILS_DEPLOY_ONEAGENT=${FLAGS_FALSE}
        fi
    elif [ "x${_deploy_mode}" = "xvci" ]; then
        # In vci mode, hpvs is compiled and installed separately (without fsx),
        # so the fsx install here must skip hpvs.
        BFUTILS_INSTALL_HPVS=${FLAGS_FALSE}
        BFUTILS_DEPLOY_ONEAGENT=${FLAGS_FALSE}
    fi

    # Backward compatibility for external deployment flows.
    if [ -n "${FSX_NO_DEPLOY_ONEAGENT}" ]; then
        BFUTILS_DEPLOY_ONEAGENT=${FLAGS_FALSE}
    fi
    if [ -n "${_fsx_no_tob_mode}" ]; then
        BFUTILS_TOB_MODE=${FLAGS_FALSE}
        BFUTILS_DEPLOY_ONEAGENT=${FLAGS_FALSE}
        BFUTILS_INSTALL_HPVS=${FLAGS_FALSE}
    fi

    bfutils_log INFO "component policy resolved: deploy_mode=${_deploy_mode}, tob_mode=${BFUTILS_TOB_MODE}, deploy_oneagent=${BFUTILS_DEPLOY_ONEAGENT}, install_hpvs=${BFUTILS_INSTALL_HPVS}"
}

hpvs_install() {
    output=$(./hpvs.sh -m install $1)
    _last_errno=$?
    if [ $_last_errno -ne 0 ]; then
        bfutils_log ERROR "hpvs install failed, error: $output, fsx will try to use kernel builtin fuse.ko"
    else
        bfutils_log INFO "hpvs install succeed, output: $output"
    fi
    return $_last_errno
}

hpvs_reinstall() {
    output=$(./hpvs.sh -m reinstall $1)
    _last_errno=$?
    if [ $_last_errno -ne 0 ]; then
        bfutils_log ERROR "${export_error_codes['HPVS_INSTALL_FAILED']} hpvs install failed, error: $output, fsx will try to use kernel builtin fuse.ko"
    else
        bfutils_log INFO "hpvs install succeed, output: $output"
    fi
    return $_last_errno
}

hpvs_uninstall() {
    bfutils_running_monitor
    if [ ! -z "${__bfutils_return}" ]; then
        run_fusetool inner stop_kmod_tracer; bfutils_return_if_err
        sleep 3
    fi
    output=$(./hpvs.sh -m uninstall)
    _last_errno=$?
    if [ $_last_errno -ne 0 ]; then
        bfutils_log ERROR "${export_error_codes['HPVS_UNINSTALL_FAILED']} hpvs uninstall failed, error: $output"
    else
        bfutils_log INFO "hpvs uninstall succeed, output: $output"
    fi
    return $_last_errno
}

hpvs_upgrade() {
    local _deploy_dir=$1
    local _work_dir=$2
    # frist, install the new kernel module.
    hpvs_install; bfutils_return_if_err

    # try to get current hpvs module name
    hpvs_version=$(awk -F'"' '/^version=/ {print $2}' hpvs.sh)

    IFS='.' read -ra version_array <<< "$hpvs_version"

    major=$(printf "%02d" "${version_array[0]}")
    minor=$(printf "%02d" "${version_array[1]}")
    patch=$(printf "%02d" "${version_array[2]}")

    hpvs_lversion="$major$minor$patch"

    upgrade_timeout=30
    # If tos mountpoints exist, hpvs upgrade may take longer.
    if ! bfutils_check_mountpoints "${_deploy_dir}" "tos" "${_work_dir}"; then
        upgrade_timeout=300
    fi
    if [[ -n $FSX_UPGRADE_TIMEOUT ]]; then
        upgrade_timeout=$FSX_UPGRADE_TIMEOUT
    fi

    is_curr=$(cat /sys/module/hpvs_proxy/parameters/status  | grep "curr vsm" | grep $hpvs_lversion | wc -l)
    is_new=$(cat /sys/module/hpvs_proxy/parameters/status  | grep "new vsm" | grep $hpvs_lversion | wc -l)
    if [[ $is_curr -eq 1 ]]; then
        bfutils_log INFO "hpvs kernel module needn't upgrade"
    elif [[ $is_new -eq 1 ]]; then
        bfutils_log INFO "hpvs kernel module start upgrade"
        ./hpvs.sh -m upgrade -t ${upgrade_timeout};
        exit_code=$?
        if [[ $exit_code == 62 ]]; then
            bfutils_log WARN "upgrade hpvs kernel module timeout, try again setting FSX_UPGRADE_TIMEOUT env value"
            bfutils_set_exit_status 62; bfutils_return
        elif [[ $exit_code != 0 ]]; then
            bfutils_log WARN "upgrade hpvs kernel module failed"
            bfutils_set_exit_status $exit_code; bfutils_return
        fi
        bfutils_log INFO "hpvs kernel module successfully upgrade"
    else
        bfutils_log ERROR "${export_error_codes['HPVS_MAY_LOAD_FAILED']} hpvs kernel module may load failed, is_curr=$is_curr, is_new=$is_new, hpvs_version=$hpvs_version"
        bfutils_log ERROR "hpvs_proxy status: $(cat /sys/module/hpvs_proxy/parameters/status)"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
}

hpvs_clean_old() {
    # rmmod old module
    modules=$(lsmod | grep -E '^hpvs_')
    while IFS= read -r line; do
        if [[ $line == "" ]]; then
            continue;
        fi
        name=$(echo "$line" | awk '{print $1}')
        refcnt=$(echo "$line" | awk '{print $3}')
        if [[ $name != "hpvs_proxy" && $refcnt -eq 0 ]]; then
            bfutils_log INFO "remove old kernel module "$name
            rmmod $name
        fi
    done <<< "$modules"
}

monitor_install() {
    local _install_params=""
    local _ignore_monitor_error=${FLAGS_TRUE}
    if [ "x${FLAGS_deploy_mode}" = "xvke" ]; then
        _ignore_monitor_error=${FLAGS_FALSE}
        _install_params+=" -a KubernetesNodeRoleForECS -e vke"
        if [ "x${FLAGS_region_id}" != "x" ]; then
            _install_params+=" -r ${FLAGS_region_id}"
        fi
        if [ "x${FLAGS_monitor_auth_strategy}" != "x" ]; then
            _install_params+=" -s ${FLAGS_monitor_auth_strategy}"
        fi
        if [ "x${FLAGS_monitor_secret_path}" != "x" ]; then
            _install_params+=" -p ${FLAGS_monitor_secret_path}"
        fi
        bfutils_log INFO "Oneagent deploy in VKE scene, install params ${_install_params}"
    fi

    if [ "x${FLAGS_log_dir}" != "x" ]; then
        _install_params+=" -l ${FLAGS_log_dir}/o11yagent_fsx"
    fi

    output=$(./install.sh ${_install_params})
    _last_errno=$?
    if [ $_last_errno -ne 0 ]; then
        # 254 means skip to install o11yagent_fsx agent
        if [ $_last_errno -eq 254 ] && [ ${_ignore_monitor_error} -eq ${FLAGS_TRUE} ]; then
            bfutils_log ERROR "monitor install skipped, output: $output"
            _last_errno=0
        else
            bfutils_log ERROR "monitor install failed, error: $output"
        fi
    else
        bfutils_log INFO "monitor install succeed, output: $output"
    fi
    return $_last_errno
}

monitor_uninstall() {
    output=$(./uninstall.sh ${1})
    _last_errno=$?
    if [ $_last_errno -ne 0 ]; then
        bfutils_log ERROR "monitor uninstall failed, error: $output"
    else
        bfutils_log INFO "monitor uninstall succeed, output: $output"
    fi
    return $_last_errno
}

dependency_install() {

    if [ ${BFUTILS_DEPLOY_ONEAGENT} -eq ${FLAGS_FALSE} ]; then
        bfutils_log INFO "No oneagent deploy, remove monitor directory"
        rm -rf ${1}/monitor
    fi

    if [ -d "${1}/module" ]; then
        if [ ${BFUTILS_INSTALL_HPVS} -eq ${FLAGS_FALSE} ]; then
            bfutils_log INFO "hpvs install is disabled, skip hpvs_reinstall"
        else
            pushd ${1}/module > /dev/null 2>&1
            hpvs_reinstall ${BFUTILS_HPVS_INSTALL_PARAMS}
            _last_errno=$?
            popd > /dev/null 2>&1
            if [ $_last_errno -ne 0 ] && [ ${BFUTILS_IGNORE_HPVS_ERROR} -eq ${FLAGS_FALSE} ]; then
                bfutils_log ERROR "hpvs install failed, error: $output"
                return $_last_errno
            fi
        fi
    fi

    if [ ${BFUTILS_DEPLOY_ONEAGENT} -eq ${FLAGS_TRUE} ] && [ -d "${1}/monitor" ]; then
        pushd ${1}/monitor > /dev/null 2>&1
        monitor_install; bfutils_return_if_err
        popd > /dev/null 2>&1
    fi
}

dependency_update() {
    local _deploy_dir=$1
    local _work_dir=$2

    if [ ${BFUTILS_DEPLOY_ONEAGENT} -eq ${FLAGS_FALSE} ]; then
        bfutils_log INFO "No oneagent deploy, remove monitor directory"
        rm -rf "${_deploy_dir}/monitor"
    fi

    if [ -d "${_deploy_dir}/module" ]; then
        if [ ${BFUTILS_INSTALL_HPVS} -eq ${FLAGS_FALSE} ]; then
            bfutils_log INFO "hpvs upgrade is disabled, skip hpvs_upgrade"
        else
            pushd "${_deploy_dir}/module" > /dev/null 2>&1
            hpvs_upgrade "${_deploy_dir}" "${_work_dir}"; bfutils_return_if_err
            popd > /dev/null 2>&1
        fi
    fi

    if [ ${BFUTILS_DEPLOY_ONEAGENT} -eq ${FLAGS_TRUE} ] && [ -d "${_deploy_dir}/monitor" ]; then
        pushd "${_deploy_dir}/monitor" > /dev/null 2>&1
        monitor_uninstall "${_deploy_dir}"; bfutils_return_if_err
        monitor_install; bfutils_return_if_err
        popd > /dev/null 2>&1
    fi
}

dependency_uninstall() {
    if [ -d "${1}/monitor" ]; then
        pushd ${1}/monitor > /dev/null 2>&1
        monitor_uninstall ${1}; bfutils_return_if_err
        popd > /dev/null 2>&1
    fi

    if ! [ -d "${1}/module" ]; then
        bfutils_log WARN "No dependency directory \"${1}/module\" found, ignore this message if integrity has been confirmed"; bfutils_return
    else
        pushd ${1}/module > /dev/null 2>&1
        hpvs_uninstall; bfutils_return_if_err
        popd > /dev/null 2>&1
    fi
}

bfutils_lock_ctl() {
    lock_file="${1}/fsx_ctl.lock"

    exec {ctl_lock_fd}>"${lock_file}" || return $?
    if flock -n $ctl_lock_fd; then
        bfutils_log INFO "Acquired fsx ctl lock successfully, lock file: ${lock_file}"
    else
        bfutils_log ERROR "Fail to acquire fsx ctl lock, lock file: ${lock_file}"
        return $(bfutils_get_error_code "lock_error")
    fi
}

bfutils_lock_mounts() {
    lock_file="${1}/fsx_mount.lock"

    exec {mount_lock_fd}>$lock_file
    if flock -n $mount_lock_fd; then
        bfutils_log INFO "Acquired mount lock successfully, lock file: ${lock_file}"
    else
        bfutils_log ERROR "Fail to acquire mount lock, lock file: ${lock_file}"
        return $(bfutils_get_error_code "lock_error")
    fi
}

bfutils_unlock_mounts() {
    flock -u $mount_lock_fd
    exec $mount_lock_fd>&-
}

# get deploy mode from service file
bfutils_get_service_deploy_mode() {
    __bfutils_return=""
    local _update_conf_path=$1
    local _fusemonitor_file_path="${_update_conf_path}/svc/fusemonitor.service"

    if [ ! -f "${_fusemonitor_file_path}" ]; then
        bfutils_log ERROR "Fusemonitor service file not found"
        return 2 # ENOENT
    fi

    local deploy_mode=$(grep 'ExecStart=' "${_fusemonitor_file_path}" | grep --only-matching 'common_deploy_mode=[^[:space:]]*' | cut -d'=' -f2 | sed "s/^['\"]//; s/['\"]$//")
    if [ -n "${deploy_mode}" ]; then
        __bfutils_return="${deploy_mode}"
    else
        __bfutils_return="${__flags_deploy_mode_default}"
    fi
    bfutils_log INFO "Fusemonitor deploy mode: ${__bfutils_return}"
}

# get work dir from service file
bfutils_get_service_work_dir() {
    __bfutils_return=""
    local _update_conf_path=$1
    local _fusemonitor_file_path="${_update_conf_path}/svc/fusemonitor.service"

    if [ ! -f "${_fusemonitor_file_path}" ]; then
        bfutils_log ERROR "Fusemonitor service file not found"
        return 2 # ENOENT
    fi

    local work_dir=$(grep 'ExecStart=' "${_fusemonitor_file_path}" | grep --only-matching 'common_work_dir=[^[:space:]]*' | cut -d'=' -f2 | sed "s/^['\"]//; s/['\"]$//")
    if [ -n "${work_dir}" ]; then
        __bfutils_return="${work_dir}"
    else
        __bfutils_return="${_update_conf_path}"
    fi
    bfutils_log INFO "Fusemonitor service work dir: ${__bfutils_return}"
}

bfutils_precheck_mount() {
    local _deploy_dir=$1
    local _conf_dir=$2
    local _deploy_mode=$3

    pushd ${_deploy_dir}/bin > /dev/null 2>&1
    local tags="vepfs:${_deploy_mode},tos:${_deploy_mode},nas:${_deploy_mode},efs:${_deploy_mode}"
    ./fusetool resource check --combination_tags="${tags} -- --common_conf_dir=${_conf_dir}"
    _errno=$?
    popd > /dev/null 2>&1

    if [ ${_errno} -ne 0 ]; then
        _errno=$(bfutils_get_error_code "check_mount_error")
        bfutils_log ERROR "Failed to precheck mount fsx ${_errno}"
        return ${_errno}
    fi
}

mount_scripts_install() {
    local __deploy_path=$1
    local __conf_path=$2
    local __work_dir=$3
    local __log_path=$4
    local _deploy_mode=$5
    cat "${__deploy_path}/tools/mount.fsx" |
    perl -pe s{'^DEPLOY_DIR=.*$'}{"DEPLOY_DIR='$(printf '%q' "${__deploy_path}")'"} |
    perl -pe s{'^CONF_DIR=.*$'}{"CONF_DIR='$(printf '%q' "${__conf_path}")'"} |
    perl -pe s{'^WORK_DIR=.*$'}{"WORK_DIR='$(printf '%q' "${__work_dir}")'"} |
    perl -pe s{'^LOG_DIR=.*$'}{"LOG_DIR='$(printf '%q' "${__log_path}")'"} |
    perl -pe s{'^DEPLOY_MODE=.*$'}{"DEPLOY_MODE='$(printf '%q' "${_deploy_mode}")'"} \
    > /sbin/mount.fsx; bfutils_return_if_err
    chmod +x /sbin/mount.fsx; bfutils_return
}
# TODO: More accurate install/uninstall
mount_scripts_uninstall() {
    bfutils_entry_remove f /sbin/mount.fsx; bfutils_return
}

env_install() {
    # ln -sfn makes this safe to repeat after a partial recovery. It replaces
    # a stale symlink but still refuses to overwrite a directory.
    ln -sfn "${1}/fsx" /usr/bin/fsx
}

env_uninstall() {
    rm -f /usr/bin/fsx
}

bfservice_enable_fsx_umount() {
    local __deploy_path=$1
    local __conf_path=$2
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfutils_enable_fsx_umount_service_path="${__conf_path}/svc/fsx_umount.service"
    local _bfutils_enable_fsx_umount_service_execstop="ExecStop='"${__deploy_path}"/fsx' '-m' 'umount'"
    perl -pe s{'^ExecStop=.*$'}{"$(printf '%q' "${_bfutils_enable_fsx_umount_service_execstop}")"}g "${__deploy_path}/templates/fsx_umount.service" \
     > "${_bfutils_enable_fsx_umount_service_path}"; bfutils_return_if_err

    systemctl enable "${_bfutils_enable_fsx_umount_service_path}" 2>/dev/null; bfutils_return_if_err

    bfutils_log INFO "fsx_umount is successfully enabled"
    bfutils_return
}

bfservice_disable_fsx_umount() {
    local __conf_path=$1
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_DIR_EMPTY']} conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _bfutils_disable_fsx_umount_service_status=$(systemctl show fsx_umount -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfutils_disable_fsx_umount_service_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to disable \"fsx_umount\", due to service not exist."
        return 0
    fi

    local _bfutils_disable_fsx_umount_service_path="${__conf_path}/svc/fsx_umount.service"
    systemctl disable fsx_umount.service 2>/dev/null; bfutils_return_if_err
    rm -f "${_bfutils_disable_fsx_umount_service_path}"; bfutils_return_if_err

    bfutils_log INFO "fsx_umount is successfully disabled"
    bfutils_return
}

bfservice_enable_fsx_target() {
    local __deploy_path=$1
    local __conf_path=$2
    local __deploy_mode=$3
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfutils_enable_fsx_target_path="${__conf_path}/svc/fsx.target"

    cp "${__deploy_path}/templates/fsx.target" "${_bfutils_enable_fsx_target_path}"; bfutils_return_if_err

    if [[ ! "x${__deploy_mode}" =~ ^x(faas|vci)$ ]]; then
        systemctl enable "${_bfutils_enable_fsx_target_path}" 2>/dev/null; bfutils_return_if_err
        bfutils_log INFO "fsx.target is successfully enabled"
    else
        systemctl link "${_bfutils_enable_fsx_target_path}" 2>/dev/null; bfutils_return_if_err
        bfutils_log INFO "fsx.target is successfully linked"
    fi

    bfutils_return
}

bfservice_disable_fsx_target() {
    local __conf_path=$1
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _bfutils_disable_fsx_target_status=$(systemctl show fsx.target -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfutils_disable_fsx_target_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to disable \"fsx.target\", due to service not exist."
        return 0
    fi

    local _bfutils_disable_fsx_target_path="${__conf_path}/svc/fsx.target"
    systemctl disable fsx.target 2>/dev/null; bfutils_return_if_err
    rm -f "${_bfutils_disable_fsx_target_path}"; bfutils_return_if_err

    bfutils_log INFO "fsx.target is successfully disabled"
    bfutils_return
}

bfservice_enable_fusemonitor() {
    local __deploy_path=$1
    local __conf_path=$2
    local __work_dir=$3
    local __log_path=$4
    local __deploy_mode=$5
    if [ -z "${__conf_path}" ] || [ -z "${__log_path}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_LOG_PATH_EMPTY']} conf path or log path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfutils_enable_fusemonitor_service_path="${__conf_path}/svc/fusemonitor.service"

    local _bfutils_enable_fusemonitor_service_execstart_pre="ExecStartPre='"${__deploy_path}"/tools/fsx_prepare.sh' '--type=fusemonitor' '--work_dir=${__work_dir}' '--conf_dir=${__conf_path}'"
    local _bfutils_enable_fusemonitor_service_execstart="ExecStart='"${__deploy_path}"/bin/fusemonitor_run' '--common_conf_dir=${__conf_path}' '--common_work_dir=${__work_dir}' '--common_log_dir=${__log_path}' '--common_deploy_mode=${__deploy_mode}'"
    local _common_socket_poller_num="$(bfutils_common_socket_poller_num)"
    if [ "${_common_socket_poller_num}" -gt 0 ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--common_socket_poller_num=${_common_socket_poller_num}'"
    fi
    if bfutils_vhost_user_fast_fail_on_umount_enabled; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_fusedaemon_vhost_user_fast_fail_on_umount'"
    fi
    if [ -n "${FSX_AGENT_NAME:-}" ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_metrics_agent_name=${FSX_AGENT_NAME}'"
    fi
    if [ -n "${FSX_AGENT_GROUP:-}" ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_metrics_agent_group=${FSX_AGENT_GROUP}'"
    fi
    if [ -n "${FSX_AGENT_RUNTIME_TAG:-}" ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_metrics_agent_runtime_tag=${FSX_AGENT_RUNTIME_TAG}'"
    fi
    if [ -n "${FSX_COMMON_METRICS_OVERRIDE_TAGS:-}" ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_fusedaemon_common_metrics_override_tags=${FSX_COMMON_METRICS_OVERRIDE_TAGS}'"
    fi
    if [[ "${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS:-0}" =~ ^[0-9]+$ ]] && [ "${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS:-0}" -gt 0 ]; then
        _bfutils_enable_fusemonitor_service_execstart+=" '--fusemonitor_fusedaemon_common_metrics_purge_metrics_timeout_ms=${FSX_COMMON_METRICS_PURGE_METRICS_TIMEOUT_MS}'"
    fi
    local _bfutils_enable_fusemonitor_service_execstart_post="ExecStartPost='"${__deploy_path}"/tools/check_uds_ready.sh' '--instance=fusemonitor' '--common_conf_dir=${__conf_path}' '--common_work_dir=${__work_dir}' '--common_log_dir=${__log_path}'"

    perl -pe s{'^ExecStart=.*$'}{"$(printf '%q' "${_bfutils_enable_fusemonitor_service_execstart}")"}g "${__deploy_path}/templates/fusemonitor.service" |
    perl -pe s{'^ExecStartPre=.*$'}{"$(printf '%q' "${_bfutils_enable_fusemonitor_service_execstart_pre}")"}g |
    perl -pe s{'^ExecStartPost=.*$'}{"$(printf '%q' "${_bfutils_enable_fusemonitor_service_execstart_post}")"}g |
    perl -pe s{'^EnvironmentFile=.*$'}{"EnvironmentFile="${__conf_path}"/conf/fsx_monitor.env"}g > "${_bfutils_enable_fusemonitor_service_path}"; bfutils_return_if_err

    systemctl enable "${_bfutils_enable_fusemonitor_service_path}" 2>/dev/null; bfutils_return_if_err

    bfutils_log INFO "fusemonitor is successfully enabled"
    bfutils_return
}

bfservice_disable_fusemonitor() {
    local __conf_path=$1
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_DIR_EMPTY']} conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _bfutils_disable_fusemonitor_service_status=$(systemctl show fusemonitor -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfutils_disable_fusemonitor_service_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to disable \"fusemonitor\", due to service not exist."
        return 0
    fi

    local _bfutils_disable_fusemonitor_service_path="${__conf_path}/svc/fusemonitor.service"
    systemctl disable fusemonitor.service 2>/dev/null; bfutils_return_if_err
    rm -f "${_bfutils_disable_fusemonitor_service_path}"; bfutils_return_if_err

    bfutils_log INFO "fusemonitor is successfully disabled"
    bfutils_return
}

# enable one global fusedaemon service, for faas/vci mode
bfservice_enable_fusedaemon() {
    local __deploy_path=$1
    local __conf_path=$2
    local __work_dir=$3
    local __log_path=$4
    local __deploy_mode=$5
    local __combination_name=$6
    if [ -z "${__conf_path}" ] || [ -z "${__log_path}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_LOG_PATH_EMPTY']} conf path or log path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _common_name="fusedaemon_${__combination_name}"

    local _target_name="${_common_name}.target"
    local _target_path="${__conf_path}/svc/${_target_name}"
    cat "${__deploy_path}/templates/fusedaemon_template.target" > "${_target_path}"
    systemctl enable "${_target_path}" 2>/dev/null; bfutils_return_if_err

    local fusedaemon_exec_start_pre="ExecStartPre='"${__deploy_path}"/tools/fsx_prepare.sh' '--type=fusedaemon' '--instance=${__combination_name}' '--work_dir=${__work_dir}'"
    fusedaemon_run_opts=(
        "--fusedaemon_combination_name=${__combination_name}"
        "--fusedaemon_support_instance_list=${__combination_name}"
        "--common_conf_dir=${__conf_path}"
        "--common_work_dir=${__work_dir}"
        "--common_log_dir=${__log_path}"
        "--common_deploy_mode=${__deploy_mode}"
    )
    bfutils_append_fusedaemon_runtime_args fusedaemon_run_opts

    local _service_name="${_common_name}.service"
    local _service_path="${__conf_path}/svc/${_service_name}"

    cat "${__deploy_path}/templates/fusedaemon_template.service" |
    perl -pe s{"^ExecStart=.*$"}{"$(printf '%q' "ExecStart='${__deploy_path}/bin/fusedaemon_run' ${fusedaemon_run_opts[*]}")"} |
    perl -pe s{"^ExecStartPre=.*$"}{"$(printf '%q' "${fusedaemon_exec_start_pre}")"} |
    perl -pe s{"^PartOf=.*$"}{"PartOf=${_target_name}"} |
    perl -pe s{"^WantedBy=.*$"}{"WantedBy=${_target_name}"} |
    perl -pe s{'^EnvironmentFile=.*$'}{"EnvironmentFile="${__conf_path}"/conf/fsx_global_daemon.env"}g \
    > "${_service_path}"

    systemctl enable "${_service_path}" 2>/dev/null; bfutils_return_if_err
    bfutils_log INFO "${__combination_name} fusedaemon is successfully enabled"
    bfutils_return
}

bfservice_disable_fusedaemon() {
    local __conf_path=$1
    local __combination_name=$2
    if [ -z "${__conf_path}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_PATH_EMPTY']} conf path is empty"
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _bfutils_disable_fusedaemon_target_status=$(systemctl show "fusedaemon_${__combination_name}.target" -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfutils_disable_fusedaemon_target_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to disable \"fusedaemon_${__combination_name}.target\", due to service not exist."
    else
        local _bfutils_disable_fusedaemon_target_path="${__conf_path}/svc/fusedaemon_${__combination_name}.target"
        systemctl disable fusedaemon_${__combination_name}.target 2>/dev/null; bfutils_return_if_err
        rm -f "${_bfutils_disable_fusedaemon_target_path}"; bfutils_return_if_err
    fi

    local _bfutils_disable_fusedaemon_service_status=$(systemctl show "fusedaemon_${__combination_name}.service" -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfutils_disable_fusedaemon_service_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to disable \"fusedaemon_${__combination_name}.service\", due to service not exist."
    else
        local _bfutils_disable_fusedaemon_service_path="${__conf_path}/svc/fusedaemon_${__combination_name}.service"
        systemctl disable fusedaemon_${__combination_name}.service 2>/dev/null; bfutils_return_if_err
        rm -f "${_bfutils_disable_fusedaemon_service_path}"; bfutils_return_if_err
    fi

    bfutils_log INFO "fusedaemon_${__combination_name} is successfully disabled"
    bfutils_return
}

bfservice_start_service() {
    if [ -z "${1}" ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfservice_start_service_name="${1}"

    systemctl start "${_bfservice_start_service_name}" 2>/dev/null; bfutils_return_if_err

    local _bfservice_start_service_status=$(systemctl show "${_bfservice_start_service_name}" -p ActiveState | awk -F= '{print $2}')
    if [ "x${_bfservice_start_service_name}" == "xinactive" ]; then
        bfutils_log ERROR "${export_error_codes['SERVICE_INACTIVE']} Failed to check \"${_bfservice_start_service_name}\", because it is inactive, status: ${_bfservice_start_service_status}"
        return 3 # ESRCH
    fi

    local _bfservice_start_service_cgroup=$(systemctl show "${_bfservice_start_service_name}" -p ControlGroup | awk -F= '{print $2}')
    if [ -z "${_bfservice_start_service_cgroup:+x}" ]; then
        bfutils_log ERROR "${export_error_codes['GET_SERVICE_CGROUP_FAIL']} Failed to get \"${_bfservice_start_service_name}\" cgroup."
        return 3 # ESRCH
    fi

    # get cgroup full path
    local _result=$(stat -fc %T /sys/fs/cgroup/)
    if [ "x${_result}" == "xtmpfs" ]; then
        local _cgroup_base_path="/sys/fs/cgroup/systemd"
    elif [ "x${_result}" == "xcgroup2fs" ]; then
        local _cgroup_base_path="/sys/fs/cgroup"
    else
        bfutils_log ERROR "${export_error_codes['CHECK_SERVICE_CGROUP_BASE_PATH_FAIL']} Failed to check \"${_bfservice_start_service_name}\" cgroup base path"
        return 3 # ESRCH
    fi

    local _bfservice_start_service_cgroup_procs="${_cgroup_base_path}/${_bfservice_start_service_cgroup}/cgroup.procs"
    if ! [ -f "${_bfservice_start_service_cgroup_procs}" ]; then
        bfutils_log ERROR "${export_error_codes['CHECK_SERVICE_CGROUP_PATH_FAIL']} Failed to check \"${_bfservice_start_service_name}\" cgroup path"
        return 3 # ESRCH
    fi

    local _bfservice_start_service_started=${FLAGS_FALSE}
    _bfservice_start_service_start_time=$(date +'%s')
    while [ ${_bfservice_start_service_started} -eq ${FLAGS_FALSE} ] && [ $(($(date +'%s') - ${_bfservice_start_service_start_time})) -lt 5 ]; do
        while read -r proc_line; do
            while read -r param_line; do
                if [[ "x${param_line}" =~ ^.*/${_bfservice_start_service_name}$ ]]; then
                    _bfservice_start_service_started=${FLAGS_TRUE}
                    break
                fi
            done <<< "$(cat "/proc/${proc_line}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g)"
            if [ ${_bfservice_start_service_started} -ne ${FLAGS_FALSE} ]; then
                break
            fi
        done <<< "$(cat "${_bfservice_start_service_cgroup_procs}" 2>/dev/null)"
    done
    if [ ${_bfservice_start_service_started} -ne ${FLAGS_TRUE} ]; then
        bfutils_log ERROR "${export_error_codes['CHECK_SERVICE_FAIL']} Failed to check \"${_bfservice_start_service_name}\", which cannot be detected"
        return 3 # ESRCH
    fi

    bfutils_log INFO "\"${_bfservice_start_service_name}\" started successfully."
    bfutils_return
}

bfservice_restart_service() {
    if [ -z "${1}" ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfservice_restart_service_errno=0
    local _bfservice_restart_service_name="${1}"

    local _bfservice_restart_service_status=$(systemctl show "${_bfservice_restart_service_name}" -p LoadState | awk -F= '{print $2}')
    if [[ "x${_bfservice_restart_service_status}" =~ ^x(|not-found)$ ]]; then
        bfutils_log WARN "No need to restart \"${_bfservice_restart_service_name}\", due to service not exist."
        return 0
    fi

    systemctl restart "${_bfservice_restart_service_name}" 2>/dev/null
    _bfservice_restart_service_errno=$?
    if [ ${_bfservice_restart_service_errno} -ne 0 ]; then
        bfutils_log ERROR "${export_error_codes['RESTART_SERVICE_FAILED']} Failed to restart service \"${_bfservice_restart_service_name}\", errno: ${_bfservice_restart_service_errno}"
        return ${_bfservice_restart_service_errno}
    fi

    bfutils_log INFO "\"${_bfservice_restart_service_name}\" restarted successfully."
    bfutils_return
}

bfservice_stop_service() {
    if [ -z "${1}" ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _bfservice_stop_service_errno=0
    local _bfservice_stop_service_name="${1}"

    local _bfservice_stop_service_active_status=$(systemctl show "${_bfservice_stop_service_name}" -p ActiveState | awk -F= '{print $2}')
    if [[ ! "x${_bfservice_stop_service_active_status}" =~ ^x(|active)$ ]]; then
        bfutils_log WARN "No need to stop \"${_bfservice_stop_service_name}\", due to service not active."
        return 0
    fi

    systemctl stop "${_bfservice_stop_service_name}" 2>/dev/null
    _bfservice_stop_service_errno=$?
    if [ ${_bfservice_stop_service_errno} -ne 0 ]; then
        bfutils_log ERROR "${export_error_codes['STOP_SERVICE_FAILED']} Failed to stop service \"${_bfservice_stop_service_name}\", errno: ${_bfservice_stop_service_errno}"
        return ${_bfservice_stop_service_errno}
    fi

    bfutils_log INFO "\"${_bfservice_stop_service_name}\" stopped successfully."
    bfutils_return
}

# for ECS update
bfservice_update_daemons() {
    local _deploy_dir=$1
    local _conf_dir=$2
    local _work_dir=$3
    local _log_dir=$4
    local _deploy_mode=$5

    while read -r _bfservice_update_daemons_pid; do
        [ -n "${_bfservice_update_daemons_pid}" ] || continue
        local _bfservice_update_daemons_combination_name="$(cat "/proc/${_bfservice_update_daemons_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | grep fusedaemon_combination_name | awk '{print substr($0, index($0, "=") + 1)}')"
        [ -n "${_bfservice_update_daemons_combination_name}" ] || continue
        local _bfservice_params="$(cat "/proc/${_bfservice_update_daemons_pid}/cmdline" 2>/dev/null | sed s/"\x00"/"\n"/g | awk 'NR>2 {print $0}' | tr '\n' ' ')"
        if [[ ! $_bfservice_params =~ --common_deploy_mode=[^[:space:]]+ ]]; then
            _bfservice_params+=" --common_deploy_mode=${_deploy_mode}"
        fi
        if [[ ! $_bfservice_params =~ --common_work_dir=[^[:space:]]+ ]]; then
            _bfservice_params+=" --common_work_dir=${_work_dir}"
        fi
        local _common_socket_poller_num="$(bfutils_common_socket_poller_num)"
        if [ "${_common_socket_poller_num}" -gt 0 ] && [[ ! $_bfservice_params =~ --common_socket_poller_num=[^[:space:]]+ ]]; then
            _bfservice_params+=" --common_socket_poller_num=${_common_socket_poller_num}"
        fi
        if bfutils_vhost_user_fast_fail_on_umount_enabled && [[ ! $_bfservice_params =~ --fusedaemon_vhost_user_fast_fail_on_umount([[:space:]]|$) ]]; then
            _bfservice_params+=" --fusedaemon_vhost_user_fast_fail_on_umount"
        fi
        bfutils_log INFO Updating fusedaemon "${_bfservice_params}"
        "${_deploy_dir}/tools/reload_fusedaemon.sh" ${_bfservice_params} >> "${_log_dir}/fusemonitor/reload_fusedaemon.log" 2>&1; bfutils_return_if_err
        tail -n -10000 "${_log_dir}/fusemonitor/reload_fusedaemon.log" > "${_log_dir}/fusemonitor/reload_fusedaemon.tmp"; mv -f "${_log_dir}/fusemonitor/reload_fusedaemon.tmp" "${_log_dir}/fusemonitor/reload_fusedaemon.log" ; bfutils_return_if_err
    done <<< "$(ps ax | grep fusedaemon_run | awk '{print $1}')"

    # ensure all mounted instance has been started
    for instance in "${supported_instances[@]}"; do
        if ! bfutils_check_mountpoints ${_deploy_dir} ${instance} ${_work_dir}; then
            local _start_fusedaemon_args=(
                "--fusedaemon_combination_name=${instance}"
                "--fusedaemon_support_instance_list=${instance}"
                "--common_deploy_mode=${_deploy_mode}"
                "--common_conf_dir=${_conf_dir}"
                "--common_work_dir=${_work_dir}"
                "--common_log_dir=${_log_dir}"
            )
            bfutils_append_fusedaemon_runtime_args _start_fusedaemon_args
            "${_deploy_dir}/tools/start_fusedaemon.sh" "${_start_fusedaemon_args[@]}" >> "${_log_dir}/fusemonitor/reload_fusedaemon.log" 2>&1; bfutils_return_if_err
            tail -n -10000 "${_log_dir}/fusemonitor/reload_fusedaemon.log" > "${_log_dir}/fusemonitor/reload_fusedaemon.tmp"; mv -f "${_log_dir}/fusemonitor/reload_fusedaemon.tmp" "${_log_dir}/fusemonitor/reload_fusedaemon.log"; bfutils_return_if_err
        fi
    done
}

# for VKE update and recovery
bfservice_mountpoint_recovery() {
    local _deploy_dir=$1
    local _conf_dir=$2
    local _work_dir=$3
    local _deploy_mode=$4

    for instance in "${supported_instances[@]}"; do
        if ! bfutils_check_mountpoints ${_deploy_dir} ${instance} ${_work_dir}; then
            # non-empty mountponts json, execute start_fusedaemon.sh
            local _start_fusedaemon_args=(
                "--fusedaemon_combination_name=${instance}"
                "--fusedaemon_support_instance_list=${instance}"
                "--common_deploy_mode=${_deploy_mode}"
                "--common_conf_dir=${_conf_dir}"
                "--common_work_dir=${_work_dir}"
                "--common_log_dir=${FLAGS_log_dir}"
                "--is_recovery_flow"
            )
            bfutils_append_fusedaemon_runtime_args _start_fusedaemon_args
            "${_deploy_dir}/tools/start_fusedaemon.sh" "${_start_fusedaemon_args[@]}" >> "${FLAGS_log_dir}/fusemonitor/recovery_mountpoint.log" 2>&1
            local _start_fusedaemon_errno=$?
            if [ ${_start_fusedaemon_errno} -ne 0 ]; then
                # Close the timeout-boundary race: if the daemon became healthy
                # just after start_fusedaemon.sh returned, recovery is allowed to
                # continue and install the runtime artifacts.
                if bfutils_check_fusedaemon_ready "${instance}" "${_deploy_dir}" "${_conf_dir}" "${_work_dir}" "${FLAGS_log_dir}"; then
                    bfutils_log WARN "fusedaemon_${instance} became ready after recovery command returned errno=${_start_fusedaemon_errno}; continue recovery"
                else
                    return ${_start_fusedaemon_errno}
                fi
            fi
            tail -n -10000 "${FLAGS_log_dir}/fusemonitor/recovery_mountpoint.log" > "${FLAGS_log_dir}/fusemonitor/recovery_mountpoint.tmp"; mv -f "${FLAGS_log_dir}/fusemonitor/recovery_mountpoint.tmp" "${FLAGS_log_dir}/fusemonitor/recovery_mountpoint.log"; bfutils_return_if_err
        fi
    done
}

bfutils_check_fusedaemon_ready() {
    local _instance=$1
    local _deploy_dir=$2
    local _conf_dir=$3
    local _work_dir=$4
    local _log_dir=$5
    local _probe_timeout_s=${6:-5}

    pushd "${_deploy_dir}/bin" > /dev/null 2>&1 || return $?
    # GETFLAGREQ waits for uds_ready_; GETVERSIONREQ intentionally does not.
    timeout "${_probe_timeout_s}s" ./fusetool flags get --flag v --instance "${_instance}" -- \
        --common_conf_dir="${_conf_dir}" \
        --common_work_dir="${_work_dir}" \
        --common_log_dir="${_log_dir}" > /dev/null 2>&1
    local _errno=$?
    popd > /dev/null 2>&1
    return ${_errno}
}

bfutils_wait_fusedaemon_ready() {
    local _instance=$1
    local _deploy_dir=$2
    local _conf_dir=$3
    local _work_dir=$4
    local _log_dir=$5
    local _wait_time_s=${6:-120}
    local _start_time
    _start_time=$(date +'%s')

    while [ $(($(date +'%s') - ${_start_time})) -lt ${_wait_time_s} ]; do
        if bfutils_check_fusedaemon_ready "${_instance}" "${_deploy_dir}" "${_conf_dir}" "${_work_dir}" "${_log_dir}"; then
            return 0
        fi
        sleep 1
    done
    bfutils_check_fusedaemon_ready "${_instance}" "${_deploy_dir}" "${_conf_dir}" "${_work_dir}" "${_log_dir}"
}

fsx_ensure_runtime_artifacts() {
    local _deploy_dir=$1
    local _conf_dir=$2
    local _work_dir=$3
    local _log_dir=$4
    local _deploy_mode=$5
    local _runtime_instance=$6
    local _wait_time_s=${7:-120}

    if [ -z "${_deploy_dir}" ] || [ -z "${_conf_dir}" ] || [ -z "${_work_dir}" ] || [ -z "${_log_dir}" ] || [ -z "${_deploy_mode}" ] || [ -z "${_runtime_instance}" ]; then
        bfutils_log ERROR "ensure runtime artifacts received an empty required argument"
        return 22 # EINVAL
    fi

    bfutils_running_daemons "${_runtime_instance}"
    if [ -z "${__bfutils_return}" ] && ! systemctl is-active --quiet "fusedaemon_${_runtime_instance}.target"; then
        bfutils_log INFO "No active fusedaemon_${_runtime_instance} runtime to repair"
        return 3 # ESRCH; caller may fall back to a full start
    fi

    # A target managed by Restart=always may temporarily have no process. Wait
    # for UDS health instead of racing it with another full FSX start.
    if ! bfutils_wait_fusedaemon_ready "${_runtime_instance}" "${_deploy_dir}" "${_conf_dir}" "${_work_dir}" "${_log_dir}" "${_wait_time_s}"; then
        bfutils_log ERROR "fusedaemon_${_runtime_instance} is running or restarting but did not become healthy within ${_wait_time_s}s"
        return 11 # EAGAIN; retry repair, do not run fsx -m start
    fi

    mount_scripts_install "${_deploy_dir}" "${_conf_dir}" "${_work_dir}" "${_log_dir}" "${_deploy_mode}"; bfutils_return_if_err
    env_install "${_deploy_dir}"; bfutils_return_if_err

    if [ ! -x /sbin/mount.fsx ] || [ ! -L /usr/bin/fsx ]; then
        bfutils_log ERROR "runtime artifact verification failed after repair"
        return 5 # EIO
    fi
    bfutils_log INFO "Runtime artifacts repaired for healthy fusedaemon_${_runtime_instance}"
    return 0
}

fsx_install() {
    if bfutils_pool_enabled; then
        [ "${FLAGS_deploy_mode}" = faas ] && [ "${FLAGS_disable_fusemonitor}" -eq "${FLAGS_FALSE}" ] || return 22
        bfutils_pool_validate_config || return $?
    fi
    local _install_errno=0

    bfutils_log INFO Calling "${FUNCNAME}" version: "${RELEASE_VERSION}" prefix: "\"${FLAGS_prefix}\"" conf_dir: "\"${FLAGS_conf_dir}\"" work_dir: "\"${FLAGS_work_dir}\"" log_dir: "\"${FLAGS_log_dir}\""

    bfutils_defer_clean
    bfutils_defer bfutils_log ERROR Failed to install fsx into "${FLAGS_prefix}"

    # NOTE: Do not check the log directory as a path not exist may be necessary
    if [ -z "${FLAGS_prefix}" ] || [ -z "${FLAGS_conf_dir}" ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _install_invalid_input=${FLAGS_FALSE}
    if ! [ "x${FLAGS_prefix}" == "x${__flags_prefix_default}" ] && ! [[ "x${FLAGS_prefix}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['PREFIX_INVALID']} prefix: "\"${FLAGS_prefix}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_conf_dir}" == "x${__flags_conf_dir_default}" ] && ! [[ "x${FLAGS_conf_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['CONF_DIR_INVALID']} conf_dir: "\"${FLAGS_conf_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_work_dir}" == "x${__flags_work_dir_default}" ] && ! [[ "x${FLAGS_work_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['WORK_DIR_INVALID']} work_dir: "\"${FLAGS_work_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_log_dir}" == "x${__flags_log_dir_default}" ] && ! [[ "x${FLAGS_log_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['LOG_DIR_INVALID']} log_dir: "\"${FLAGS_log_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if [ "${_install_invalid_input}" -ne ${FLAGS_FALSE} ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    local _install_need_copy=${FLAGS_FALSE}
    if [ "${deploy_dir}" -ef "${FLAGS_prefix}" ]; then
        bfutils_log DEBUG prefix is where we are, no need to copy
    else
        if ! [ -d "${FLAGS_prefix}" ]; then
            if [ -d "$(dirname "${FLAGS_prefix}")" ] && mkdir "${FLAGS_prefix}"; then
                bfutils_log DEBUG Try to install fsx @"'${FLAGS_prefix}'"
            else
                _install_errno=$?
                bfutils_log ERROR ${export_error_codes['CREATE_PREFIX_DIR_FAILED']} Failed to create fsx prefix dir "\"${FLAGS_prefix}\""
                return ${_install_errno}
            fi
        fi
        if [ -n "$(ls -A "${FLAGS_prefix}")" ]; then
            bfutils_log ERROR ${export_error_codes['PREFIX_DIR_NOT_EMPTY']} fsx prefix dir "\"${FLAGS_prefix}\"" is not empty, stop installation
            return 39 # ENOTEMPTY
        fi
        _install_need_copy=${FLAGS_TRUE}
    fi

    if [ "x${FLAGS_prefix}" == "x." ]; then
        FLAGS_prefix="${deploy_dir}"
    else
        local _install_real_prefix="$(realpath -e "${FLAGS_prefix}")"
        if [ "${FLAGS_prefix}" != "${_install_real_prefix}" ] || [[ "${_install_real_prefix}" =~ ^/$ ]]; then
            # bfutils_log ERROR ${export_error_codes['PREFIX_INVALID']} fsx prefix dir "\"${FLAGS_prefix}\"" is invalid, stop installation
            # return 22 # EINVAL
            bfutils_log INFO fsx prefix dir "\"${FLAGS_prefix}\"" is not equal with realpath "\"${_install_real_prefix}\""
        fi
    fi

    if ! [ -d "${FLAGS_conf_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_conf_dir}")" ] && mkdir "${FLAGS_conf_dir}"; then
            bfutils_log DEBUG Try to store fsx configurations into "${FLAGS_conf_dir}"
        elif ! [ -d "$(dirname "${FLAGS_conf_dir}")" ] && [ "${FLAGS_conf_dir}" == "${__flags_conf_dir_default}" ]; then # TODO: Deprecate this
            mkdir -p "${FLAGS_conf_dir}"; bfutils_return_if_err
        else
            bfutils_log ERROR ${export_error_codes['CREATE_CONF_DIR_FAILED']} Failed to create fsx conf dir "\"${FLAGS_conf_dir}\""
            return 22 # EINVAL
        fi
    fi
    if [ -n "$(ls -A "${FLAGS_conf_dir}")" ]; then
        bfutils_log ERROR ${export_error_codes['CONF_DIR_NOT_EMPTY']} fsx conf dir "\"${FLAGS_conf_dir}\"" is not empty, stop installation
        return 39 # ENOTEMPTY
    fi

    if ! [ -d "${FLAGS_work_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_work_dir}")" ] && mkdir "${FLAGS_work_dir}"; then
            bfutils_log DEBUG Try to store fsx configurations into "${FLAGS_work_dir}"
        elif ! [ -d "$(dirname "${FLAGS_work_dir}")" ] && [ "${FLAGS_work_dir}" == "${__flags_work_dir_default}" ]; then # TODO: Deprecate this
            mkdir -p "${FLAGS_work_dir}"; bfutils_return_if_err
        else
            bfutils_log ERROR ${export_error_codes['CREATE_WORK_DIR_FAILED']} Failed to create fsx work dir "\"${FLAGS_work_dir}\""
            return 22 # EINVAL
        fi
    fi
    if [ -n "$(ls -A "${FLAGS_work_dir}")" ]; then
        bfutils_log ERROR fsx work dir "\"${FLAGS_work_dir}\"" is not empty, stop installation
        return 39 # ENOTEMPTY
    fi

    if ! [ -d "${FLAGS_log_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_log_dir}")" ] && mkdir "${FLAGS_log_dir}"; then
            bfutils_log DEBUG Try to logging into "\"${FLAGS_log_dir}\""
        else
            bfutils_log ERROR ${export_error_codes['CREATE_LOG_DIR_FAILED']} Failed to create fsx log dir "\"${FLAGS_log_dir}\"" # NOTE: No more than error log
            return 22 # EINVAL
        fi
    fi
    # init global component policy var
    bfutils_component_policy_init "${FLAGS_deploy_mode}"; bfutils_return_if_err

    # lock fsx install process
    bfutils_defer bfutils_work_remove "${FLAGS_work_dir}"
    mkdir -p "${FLAGS_work_dir}/conf"; bfutils_return_if_err
    bfutils_lock_ctl "${FLAGS_work_dir}/conf"; bfutils_return_if_err

    if bfutils_running_processes && [ -n "${__bfutils_return}" ]; then
        bfutils_log ERROR ${export_error_codes['MODULE_STILL_RUNNING']} Some fsx module is running, abort installation
        return 17 # EEXIST
    fi

    if [ ${_install_need_copy} -eq ${FLAGS_TRUE} ]; then
        bfutils_defer bfutils_binary_remove "${FLAGS_prefix}"
        bfutils_binary_copy "${FLAGS_prefix}" "${deploy_dir}"; bfutils_return_if_err
    fi

    bfutils_defer dependency_uninstall "${FLAGS_prefix}"
    dependency_install "${FLAGS_prefix}"; bfutils_return_if_err

    bfutils_defer bfutils_conf_remove "${FLAGS_conf_dir}"
    bfutils_conf_copy "${FLAGS_conf_dir}" "${deploy_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

    # enable order: fsx.target -> fusemonitor -> fsx_umount
    bfutils_defer bfservice_disable_fsx_target "${FLAGS_conf_dir}"
    bfservice_enable_fsx_target "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

    if [ ${FLAGS_disable_fusemonitor} -eq ${FLAGS_TRUE} ]; then
        bfutils_log INFO "fusemonitor service enable is disabled by --disable_fusemonitor flag, skip bfservice_enable_fusemonitor in fsx_install"
    else
        bfutils_defer bfservice_disable_fusemonitor "${FLAGS_conf_dir}"
        bfservice_enable_fusemonitor "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err
    fi

    bfutils_defer bfservice_disable_fsx_umount "${FLAGS_conf_dir}"
    bfservice_enable_fsx_umount "${FLAGS_prefix}" "${FLAGS_conf_dir}"; bfutils_return_if_err

    if bfutils_pool_enabled; then
        fsx_pool_start || return $?
    elif [[ ! "x${FLAGS_deploy_mode}" =~ ^x(faas|vci)$ ]]; then
        # start order: fusemonitor -> fsx_umount -> fsx.target
        if [ ${FLAGS_disable_fusemonitor} -eq ${FLAGS_TRUE} ]; then
            bfutils_log INFO "fusemonitor.service start is disabled by --disable_fusemonitor flag, skip bfservice_start_service in fsx_install"
        else
            bfutils_defer bfservice_stop_service 'fusemonitor'
            bfservice_start_service 'fusemonitor'; bfutils_return_if_err
        fi

        bfutils_defer bfservice_stop_service 'fsx_umount'
        systemctl start fsx_umount 2>/dev/null; bfutils_return_if_err  # fsx_umount no running process

        bfutils_defer bfservice_stop_service 'fsx.target'
        systemctl start fsx.target 2>/dev/null; bfutils_return_if_err  # start fsx.target at last
    else
        instances_list=(${FLAGS_deploy_instances//,/ })
        for instance in "${instances_list[@]}"; do
            bfutils_defer bfservice_disable_fusedaemon "${FLAGS_conf_dir}" "${instance}"
            bfservice_enable_fusedaemon "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}" "${instance}"; bfutils_return_if_err
        done
    fi

    bfutils_defer mount_scripts_uninstall
    mount_scripts_install "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

    bfutils_defer env_uninstall "${FLAGS_prefix}"
    env_install "${FLAGS_prefix}"; bfutils_return_if_err

    bfutils_defer_clean
    bfutils_log INFO "${FUNCNAME}" succeed
    hpvs_rm_src_tar "${FLAGS_prefix}"
    return ${_install_errno}
}

update_inner_func() {
    local _deploy_dir=$1
    local __update_target_path=$2
    local __update_conf_path=$3
    local __update_work_path=$4
    local __update_log_path=$5
    local __update_deploy_mode=$6

    bfutils_log INFO "Updating fsx deployed @\"${__update_target_path}\""
    bfutils_lock_ctl "${__update_work_path}/conf"; bfutils_return_if_err

    # if exists native_fuse mountpoints, not allowed to update
    for instance in "${supported_instances[@]}"; do
        bfutils_check_native_fuse_mountpoints ${_deploy_dir} ${instance} ${__update_work_path}; bfutils_return_if_err
    done
    if ! [ "${_deploy_dir}" -ef "${__update_target_path}" ]; then
        # copy binaries / scripts
        bfutils_binary_copy "${__update_target_path}" "${_deploy_dir}"; bfutils_return_if_err
    fi

    # write to /sbin/mount.fsx
    mount_scripts_install "${__update_target_path}" "${__update_conf_path}" "${__update_work_path}" "${__update_log_path}" "${__update_deploy_mode}"; bfutils_return_if_err

    # upgrade o11yagent_fusemonitor / hpvs
    dependency_update "${__update_target_path}" "${__update_work_path}"; bfutils_return_if_err

    # fill conf and copy to deploy_dir
    bfutils_conf_copy "${__update_conf_path}" "${_deploy_dir}" "${__update_deploy_mode}"; bfutils_return_if_err

    bfservice_enable_fsx_target "${__update_target_path}" "${__update_conf_path}" "${__update_deploy_mode}"; bfutils_return_if_err

    if [ ${FLAGS_disable_fusemonitor} -eq ${FLAGS_TRUE} ]; then
        bfutils_log INFO "fusemonitor service enable/restart is disabled by --disable_fusemonitor flag, skip in fsx_update"
    else
        bfservice_enable_fusemonitor "${__update_target_path}" "${__update_conf_path}" "${__update_work_path}" "${__update_log_path}" "${__update_deploy_mode}"; bfutils_return_if_err

        bfservice_restart_service 'fusemonitor'; bfutils_return_if_err
    fi

    bfservice_enable_fsx_umount "${__update_target_path}" "${__update_conf_path}"; bfutils_return_if_err
    systemctl start fsx_umount 2>/dev/null; bfutils_return_if_err  # ensure fsx_umount is active

    systemctl start fsx.target 2>/dev/null; bfutils_return_if_err  # ensure fsx.target is active at last

    # update fusedaemon service
    bfservice_update_daemons "${__update_target_path}" "${__update_conf_path}" "${__update_work_path}" "${__update_log_path}" "${__update_deploy_mode}"; bfutils_return_if_err

    systemctl daemon-reload 2>/dev/null
    systemctl reset-failed 2>/dev/null

    hpvs_clean_old "${_deploy_dir}"; bfutils_return_if_err
    return 0
}

fsx_update() {
    if bfutils_pool_enabled; then
        fsx_pool_update
        return $?
    fi
    local _update_errno=0
    bfutils_log INFO Calling "${FUNCNAME}" version: "${RELEASE_VERSION}" prefix: "\"${FLAGS_prefix}\"" conf_dir: "\"${FLAGS_conf_dir}\"" work_dir: "\"${FLAGS_work_dir}\"" log_dir: "\"${FLAGS_log_dir}\""

    local _update_invalid_input=${FLAGS_FALSE}
    if ! [ "x${FLAGS_prefix}" == "x${__flags_prefix_default}" ] && ! [[ "x${FLAGS_prefix}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['PREFIX_INVALID']} prefix: "\"${FLAGS_prefix}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _update_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_conf_dir}" == "x${__flags_conf_dir_default}" ] && ! [[ "x${FLAGS_conf_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['CONF_DIR_INVALID']} conf_dir: "\"${FLAGS_conf_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _update_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_work_dir}" == "x${__flags_work_dir_default}" ] && ! [[ "x${FLAGS_work_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['WORK_DIR_INVALID']} work_dir: "\"${FLAGS_work_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _update_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_log_dir}" == "x${__flags_log_dir_default}" ] && ! [[ "x${FLAGS_log_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['LOG_DIR_INVALID']} log_dir: "\"${FLAGS_log_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _update_invalid_input=${FLAGS_TRUE}
    fi
    if [ "${_update_invalid_input}" -ne ${FLAGS_FALSE} ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    #TODO: simplify the logic of detecting running fsx process
    local _update_target_path=''
    if ! bfutils_running_bin_path; then
        _update_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['MULTI_FSX_PROCESS']} Multiple source fsx processes detected, abort update."
        return ${_update_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _update_target_path="${__bfutils_return}"
        FLAGS_prefix="${__bfutils_return}"
    else
        _update_target_path="${__flags_prefix_default}"
        FLAGS_prefix="${__flags_prefix_default}"
    fi
    if [ -n "${_update_target_path}" ] && [ "${FLAGS_prefix}" != "${__flags_prefix_default}" ] && ! [ "${_update_target_path}" -ef "${FLAGS_prefix}" ]; then
        bfutils_log ERROR "${export_error_codes['PREFIX_MISMATCH_RUNNING']} Specified prefix \"${FLAGS_prefix}\" different from running ones': \"${_update_target_path}\", abort updating."
        return 22 # EINVAL
    fi
    if [ -z "${_update_target_path}" ]; then
        _update_target_path="${FLAGS_prefix}"
    fi

    local _update_conf_path=''
    if ! bfutils_running_conf_path; then
        _update_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_CONF_DIR']} Cannot detect running fsx conf dir, abort updating."
        return ${_update_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _update_conf_path="${__bfutils_return}"
        FLAGS_conf_dir="${__bfutils_return}"
    else
        _update_conf_path="${__flags_conf_dir_default}"
        FLAGS_conf_dir="${__flags_conf_dir_default}"
    fi
    if [ -n "${_update_conf_path}" ] && [ "${FLAGS_conf_dir}" != "${__flags_conf_dir_default}" ] && ! [ "${_update_conf_path}" -ef "${FLAGS_conf_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_DIR_MISMATCH_RUNNING']} Specified conf dir \"${FLAGS_conf_dir}\" different from running ones': \"${_update_conf_path}\", abort updating."
        return 22 # EINVAL
    fi
    if [ -z "${_update_conf_path}" ]; then
        _update_conf_path="${FLAGS_conf_dir}"
        FLAGS_conf_dir="${_update_conf_path}"
    fi

    local _update_work_path=''
    if ! bfutils_running_work_path; then
        _update_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_WORK_DIR']} Cannot detect running fsx work dir, abort updating."
        return ${_update_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _update_work_path="${__bfutils_return}"
        FLAGS_work_dir="${__bfutils_return}"
    else
        # if not found, use conf path as work path
        _update_work_path="${_update_conf_path}"
        FLAGS_work_dir="${_update_conf_path}"
    fi
    if [ -n "${_update_work_path}" ] && [ "${FLAGS_work_dir}" != "${__flags_work_dir_default}" ] && ! [ "${_update_work_path}" -ef "${FLAGS_work_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['WORK_DIR_MISMATCH_RUNNING']} Specified work dir \"${FLAGS_work_dir}\" different from running ones': \"${_update_work_path}\", abort updating."
        return 22 # EINVAL
    fi

    local _update_log_path=''
    if ! bfutils_running_log_path; then
        _update_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_LOG_DIR']} Cannot detect running fsx log dir, abort updating."
        return ${_update_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _update_log_path="${__bfutils_return}"
    else
        _update_log_path="${__flags_log_dir_default}"
    fi
    if [ -n "${_update_log_path}" ] && [ "${FLAGS_log_dir}" != "${__flags_log_dir_default}" ] && ! [ "${_update_log_path}" -ef "${FLAGS_log_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['LOG_DIR_MISMATCH_RUNNING']} Specified log dir \"${FLAGS_log_dir}\" different from running ones': \"${_update_log_path}\", abort updating."
        return 22 # EINVAL
    fi
    if [ -z "${_update_log_path}" ]; then
        _update_log_path="${FLAGS_log_dir}"
    fi

    local _update_deploy_mode=''
    if ! bfutils_running_deploy_mode; then
        _update_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_DEPLOY_MODE']} Cannot detect running fsx deploy mode, abort updating."
        return ${_update_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _update_deploy_mode="${__bfutils_return}"
    else
        _update_deploy_mode="${__flags_deploy_mode_default}"
    fi
    if [ -n "${_update_deploy_mode}" ] && [ "${FLAGS_deploy_mode}" != "${__flags_deploy_mode_default}" ] && [ "${_update_deploy_mode}" != "${FLAGS_deploy_mode}" ]; then
        bfutils_log ERROR "${export_error_codes['DEPLOY_MODE_MISMATCH_RUNNING']} Specified deploy mode \"${FLAGS_deploy_mode}\" different from running ones': \"${_update_deploy_mode}\", abort updating."
        return 22 # EINVAL
    fi

    # init global component policy var
    bfutils_component_policy_init "${_update_deploy_mode}"; bfutils_return_if_err

    if ! [ "${deploy_dir}" -ef "${_update_target_path}" ]; then
        # non deb/rpm update, need to backup binary here
        bfutils_binary_backup "${_update_target_path}" "${_update_work_path}"
    fi
    bfutils_conf_backup "${_update_conf_path}" "${_update_work_path}"
    bfutils_misc_backup "${_update_conf_path}" "${_update_work_path}"

    update_inner_func "${deploy_dir}" "${_update_target_path}" "${_update_conf_path}" "${_update_work_path}" "${_update_log_path}" "${_update_deploy_mode}"
    _update_errno=$?
    if [ ${_update_errno} -ne 0 ]; then
        # try to rollback to old version
        bfutils_backup_binary_recover "${_update_target_path}" "${_update_work_path}"
        bfutils_backup_conf_recover "${_update_conf_path}" "${_update_work_path}"
        bfutils_backup_misc_recover "${_update_conf_path}" "${_update_work_path}"

        return ${_update_errno}
    fi

    bfutils_backup_clean "${_update_work_path}"

    bfutils_log INFO "${FUNCNAME}" succeed
    hpvs_rm_src_tar "${FLAGS_prefix}"
    return ${_update_errno}
}

fsx_uninstall() {
    local _uninstall_errno=0
    bfutils_log INFO Calling "${FUNCNAME}" version: "${RELEASE_VERSION}" prefix: "\"${FLAGS_prefix}\"" conf_dir: "\"${FLAGS_conf_dir}\"" work_dir: "\"${FLAGS_work_dir}\"" log_dir: "\"${FLAGS_log_dir}\""

    local _uninstall_invalid_input=${FLAGS_FALSE}
    if ! [ "x${FLAGS_prefix}" == "x${__flags_prefix_default}" ] && ! [[ "x${FLAGS_prefix}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['PREFIX_INVALID']} prefix: "\"${FLAGS_prefix}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _uninstall_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_conf_dir}" == "x${__flags_conf_dir_default}" ] && ! [[ "x${FLAGS_conf_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['CONF_DIR_INVALID']} conf_dir: "\"${FLAGS_conf_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _uninstall_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_work_dir}" == "x${__flags_work_dir_default}" ] && ! [[ "x${FLAGS_work_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['WORK_DIR_INVALID']} work_dir: "\"${FLAGS_work_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _uninstall_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_log_dir}" == "x${__flags_log_dir_default}" ] && ! [[ "x${FLAGS_log_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['LOG_DIR_INVALID']} log_dir: "\"${FLAGS_log_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _uninstall_invalid_input=${FLAGS_TRUE}
    fi
    if [ "${_uninstall_invalid_input}" -ne ${FLAGS_FALSE} ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _uninstall_target_path=''
    if ! bfutils_running_path; then
        _uninstall_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['MULTI_FSX_PROCESS']} Multiple source fsx processes detected, abort uninstall."
        return ${_uninstall_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _uninstall_target_path="${__bfutils_return}"
        FLAGS_prefix="${__bfutils_return}"
    else
        _uninstall_target_path="${__flags_prefix_default}"
        FLAGS_prefix="${__flags_prefix_default}"
    fi

    if [ -n "${_uninstall_target_path}" ] && [ "${FLAGS_prefix}" != "${__flags_prefix_default}" ] && ! [ "${_uninstall_target_path}" -ef "${FLAGS_prefix}" ]; then
        bfutils_log ERROR "${export_error_codes['PREFIX_MISMATCH_RUNNING']} Specified prefix \"${FLAGS_prefix}\" different from running ones': \"${_uninstall_target_path}\", abort uninstallation."
        return 22 # EINVAL
    fi
    if [ -z "${_uninstall_target_path}" ]; then
        _uninstall_target_path="${FLAGS_prefix}"
    fi

    local _uninstall_conf_path=''
    if ! bfutils_running_conf_path; then
        _uninstall_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_CONF_DIR']} Cannot detect running fsx conf dir, abort uninstallation."
        return ${_uninstall_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _uninstall_conf_path="${__bfutils_return}"
        FLAGS_conf_dir="${__bfutils_return}"
    else
        _uninstall_conf_path="${__flags_conf_dir_default}"
        FLAGS_conf_dir="${__flags_conf_dir_default}"
    fi
    if [ -n "${_uninstall_conf_path}" ] && [ "${FLAGS_conf_dir}" != "${__flags_conf_dir_default}" ] && ! [ "${_uninstall_conf_path}" -ef "${FLAGS_conf_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['CONF_DIR_MISMATCH_RUNNING']} Specified conf dir \"${FLAGS_conf_dir}\" different from running ones': \"${_uninstall_conf_path}\", abort uninstallation."
        return 22 # EINVAL
    fi
    if [ -z "${_uninstall_conf_path}" ]; then
        _uninstall_conf_path="${FLAGS_conf_dir}"
    fi


    local _uninstall_work_path=''
    if ! bfutils_running_work_path; then
        _uninstall_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_WORK_DIR']} Cannot detect running fsx work dir, abort uninstallation."
        return ${_uninstall_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _uninstall_work_path="${__bfutils_return}"
        FLAGS_work_dir="${__bfutils_return}"
    else
        # if not found, use conf path as work path
        _uninstall_work_path="${_uninstall_conf_path}"
        FLAGS_work_dir="${_uninstall_conf_path}"
    fi
    if [ -n "${_uninstall_work_path}" ] && [ "${FLAGS_work_dir}" != "${__flags_work_dir_default}" ] && ! [ "${_uninstall_work_path}" -ef "${FLAGS_work_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['WORK_DIR_MISMATCH_RUNNING']} Specified work dir \"${FLAGS_work_dir}\" different from running ones': \"${_uninstall_work_path}\", abort uninstallation."
        return 22 # EINVAL
    fi

    local _uninstall_log_path=''
    if ! bfutils_running_log_path; then
        _uninstall_errno=${__bfutils_ole}
        bfutils_log ERROR "${export_error_codes['NO_RUNNING_LOG_DIR']} Cannot detect running fsx log dir, abort uninstallation."
        return ${_uninstall_errno}
    fi
    if [ -n "${__bfutils_return}" ]; then
        _uninstall_log_path="${__bfutils_return}"
    else
        _uninstall_log_path="${__flags_log_dir_default}"
    fi
    if [ -n "${_uninstall_log_path}" ] && [ "${FLAGS_log_dir}" != "${__flags_log_dir_default}" ] && ! [ "${_uninstall_log_path}" -ef "${FLAGS_log_dir}" ]; then
        bfutils_log ERROR "${export_error_codes['LOG_DIR_MISMATCH_RUNNING']} Specified log dir \"${FLAGS_log_dir}\" different from running ones': \"${_uninstall_log_path}\", abort uninstallation."
        return 22 # EINVAL
    fi
    if [ -z "${_uninstall_log_path}" ]; then
        _uninstall_log_path="${FLAGS_log_dir}"
    fi

    # lock fsx uninstall process
    mkdir -p "${_uninstall_work_path}"/conf; bfutils_return_if_err
    bfutils_lock_ctl "${_uninstall_work_path}"/conf; bfutils_return_if_err

    for instance in "${supported_instances[@]}"; do
        if bfutils_check_mountpoints ${_uninstall_target_path} ${instance} ${_uninstall_work_path}; then
            # no mountpoint, try to stop fusedaemon
            ${_uninstall_target_path}/tools/stop_fusedaemon.sh ${instance} >> "${_uninstall_log_path}/fusemonitor/stop_fusedaemon.log" 2>&1
            if [ $? -ne 0 ]; then
                bfutils_log ERROR "${export_error_codes['STOP_FUSEDAEMON_FAILED']} Stop ${instance} fusedaemon failed, abort uninstallation."
                return 17 # EEXIST
            fi
        else
            bfutils_log ERROR "${export_error_codes['MOUNTPOINTS_STILL_ALIVE']} ${instance} mountpoint is still mounted, abort uninstallation."
            return 17 # EEXIST
        fi
    done

    if bfutils_running_daemons && [ -n "${__bfutils_return}" ]; then
        bfutils_log ERROR "${export_error_codes['FUSEDAEMON_STILL_ALIVE']} Some fsx fusedaemon is still alive, abort uninstallation"
        return 17 # EEXIST
    fi

    dependency_uninstall "${_uninstall_target_path}"; bfutils_return_if_err

    mount_scripts_uninstall; bfutils_return_if_err

    bfservice_stop_service 'fsx_umount'; bfutils_return_if_err

    bfservice_disable_fsx_umount "${_uninstall_conf_path}"; bfutils_return_if_err

    bfservice_stop_service 'fusemonitor'; bfutils_return_if_err

    bfservice_disable_fusemonitor "${_uninstall_conf_path}"; bfutils_return_if_err

    bfservice_stop_service 'fsx.target'; bfutils_return_if_err # stop fsx.target at last

    bfservice_disable_fsx_target "${_uninstall_conf_path}"; bfutils_return_if_err

    systemctl reset-failed 2>/dev/null

    bfutils_work_remove "${_uninstall_work_path}"; bfutils_return_if_err

    bfutils_conf_remove "${_uninstall_conf_path}"; bfutils_return_if_err

    # uninstall env before remove binary, user may use fsx -v to check version
    env_uninstall "${_uninstall_target_path}"; bfutils_return_if_err

    # deb/rpm flow need ignore bianry remove.
    if [ "${FLAGS_ignore_remove_binary}" -eq "${FLAGS_FALSE}" ]; then
        bfutils_binary_remove "${_uninstall_target_path}"; bfutils_return_if_err
    fi

    if [ "${FLAGS_ignore_remove_logs}" -eq "${FLAGS_FALSE}" ]; then
        bfutils_log_remove "${_uninstall_log_path}"; bfutils_return_if_err
    fi

    bfutils_return
}

recovery_inner_func() {
    local _deploy_dir=$1
    # get deploy_mode and work_dir
    local _work_dir=""
    local _deploy_mode=""

    bfutils_get_service_work_dir "${FLAGS_conf_dir}"; bfutils_return_if_err
    _work_dir="${__bfutils_return}"

    bfutils_get_service_deploy_mode "${FLAGS_conf_dir}"; bfutils_return_if_err
    _deploy_mode="${__bfutils_return}"

    if [ "x${_deploy_mode}" != "xvke" ]; then
        bfutils_log ERROR "Running deploy mode \"${_deploy_mode}\" is not vke, abort start."
        return 22 # EINVAL
    fi
    bfutils_component_policy_init "${_deploy_mode}"; bfutils_return_if_err

    # lock ctl and mount first
    bfutils_lock_ctl "${_work_dir}/conf"; bfutils_return_if_err
    # if exists native_fuse mountpoints, not allowed to recover
    for instance in "${supported_instances[@]}"; do
        bfutils_check_native_fuse_mountpoints ${_deploy_dir} ${instance} ${_work_dir}; bfutils_return_if_err
    done
    # start recovery process
    # first backup all confs for restoring after failure
    bfutils_conf_backup "${FLAGS_conf_dir}"; bfutils_return_if_err

    bfutils_conf_copy "${FLAGS_conf_dir}" "${_deploy_dir}" "${_deploy_mode}"; bfutils_return_if_err

    dependency_update "${FLAGS_prefix}" "${_work_dir}"; bfutils_return_if_err

    bfservice_enable_fsx_target "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${_deploy_mode}"; bfutils_return_if_err

    bfservice_enable_fusemonitor "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${_work_dir}" "${FLAGS_log_dir}" "${_deploy_mode}"; bfutils_return_if_err

    bfservice_start_service 'fusemonitor'; bfutils_return_if_err

    systemctl start fsx.target 2>/dev/null; bfutils_return_if_err  # start fsx.target at last

    bfservice_mountpoint_recovery "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${_work_dir}" "${_deploy_mode}"; bfutils_return_if_err

    mount_scripts_install "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${_work_dir}" "${FLAGS_log_dir}" "${_deploy_mode}"; bfutils_return_if_err

    env_install "${FLAGS_prefix}"; bfutils_return_if_err

    bfutils_defer_clean

    hpvs_rm_src_tar "${FLAGS_prefix}"
    bfutils_log INFO "${FUNCNAME}" succeed
}

fsx_start() {
    local _install_errno=0

    bfutils_log INFO Calling "${FUNCNAME}" version: "${RELEASE_VERSION}" prefix: "\"${FLAGS_prefix}\"" conf_dir: "\"${FLAGS_conf_dir}\"" work_dir: "\"${FLAGS_work_dir}\"" log_dir: "\"${FLAGS_log_dir}\""

    bfutils_defer_clean
    bfutils_defer bfutils_log ERROR Failed to start fsx into "${FLAGS_prefix}"

    # NOTE: Do not check the log directory as a path not exist may be necessary
    if [ -z "${FLAGS_prefix}" ] || [ -z "${FLAGS_conf_dir}" ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi
    local _install_invalid_input=${FLAGS_FALSE}
    if ! [ "x${FLAGS_prefix}" == "x${__flags_prefix_default}" ] && ! [[ "x${FLAGS_prefix}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['PREFIX_INVALID']} prefix: "\"${FLAGS_prefix}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_conf_dir}" == "x${__flags_conf_dir_default}" ] && ! [[ "x${FLAGS_conf_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['CONF_DIR_INVALID']} conf_dir: "\"${FLAGS_conf_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_work_dir}" == "x${__flags_work_dir_default}" ] && ! [[ "x${FLAGS_work_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['WORK_DIR_INVALID']} work_dir: "\"${FLAGS_work_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if ! [ "x${FLAGS_log_dir}" == "x${__flags_log_dir_default}" ] && ! [[ "x${FLAGS_log_dir}" =~ ^(x\/[[:alnum:]_\/]*)$ ]]; then
        bfutils_log ERROR ${export_error_codes['LOG_DIR_INVALID']} log_dir: "\"${FLAGS_log_dir}\"" does not satisfy '"/[a-zA-Z0-9_/]*"'
        _install_invalid_input=${FLAGS_TRUE}
    fi
    if [ "${_install_invalid_input}" -ne ${FLAGS_FALSE} ]; then
        bfutils_set_exit_status 22; bfutils_return # EINVAL
    fi

    # Fault injection (start only): check rule file and source injector.
    # Keep normal path unaffected when rule file does not exist.
    local _fi_rules_file="/tmp/fsx/fault_inject_rules"
    local _fi_injector="${deploy_dir}/tools/inject_ctl_fault.sh"
    if [ -f "${_fi_rules_file}" ] && [ -f "${_fi_injector}" ]; then
        while IFS='|' read -r _fi_target _fi_error _fi_mountpoint; do
            [ -n "${_fi_target}" ] || continue
            [ -n "${_fi_error}" ] || continue
            if [ "x${_fi_target}" != "xstart" ]; then
                continue
            fi
            bfutils_log ERROR "start fault injection error: ${_fi_error}"
            . "${_fi_injector}" --action inject --target start --error "${_fi_error}"
            local _fi_errno=$?
            if [ ${_fi_errno} -ne 0 ]; then
                bfutils_log WARN "start fault injection failed, errno=${_fi_errno}"
                return ${_fi_errno}
            fi
        done < "${_fi_rules_file}"
    fi

    local _install_need_copy=${FLAGS_FALSE}
    if [ "${deploy_dir}" -ef "${FLAGS_prefix}" ]; then
        bfutils_log DEBUG prefix is where we are, no need to copy
    else
        if ! [ -d "${FLAGS_prefix}" ]; then
            if [ -d "$(dirname "${FLAGS_prefix}")" ] && mkdir "${FLAGS_prefix}"; then
                bfutils_log DEBUG Try to install fsx @"'${FLAGS_prefix}'"
            else
                _install_errno=$?
                bfutils_log ERROR ${export_error_codes['CREATE_PREFIX_DIR_FAILED']} Failed to create fsx prefix dir "\"${FLAGS_prefix}\""
                return ${_install_errno}
            fi
        fi
        if [ -n "$(ls -A "${FLAGS_prefix}")" ]; then
            bfutils_log ERROR ${export_error_codes['PREFIX_DIR_NOT_EMPTY']} fsx prefix dir "\"${FLAGS_prefix}\"" is not empty, stop installation
            return 39 # ENOTEMPTY
        fi
        _install_need_copy=${FLAGS_TRUE}
    fi

    if [ "x${FLAGS_prefix}" == "x." ]; then
        FLAGS_prefix="${deploy_dir}"
    else
        local _install_real_prefix="$(realpath -e "${FLAGS_prefix}")"
        if [ "${FLAGS_prefix}" != "${_install_real_prefix}" ] || [[ "${_install_real_prefix}" =~ ^/$ ]]; then
            # bfutils_log ERROR fsx prefix dir "\"${FLAGS_prefix}\"" is invalid, stop installation
            # return 22 # EINVAL
            bfutils_log INFO fsx prefix dir "\"${FLAGS_prefix}\"" is not equal with realpath "\"${_install_real_prefix}\""
        fi
    fi

    if ! [ -d "${FLAGS_conf_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_conf_dir}")" ] && mkdir "${FLAGS_conf_dir}"; then
            bfutils_log DEBUG Try to store fsx configurations into "${FLAGS_conf_dir}"
        elif ! [ -d "$(dirname "${FLAGS_conf_dir}")" ] && [ "${FLAGS_conf_dir}" == "${__flags_conf_dir_default}" ]; then # TODO: Deprecate this
            mkdir -p "${FLAGS_conf_dir}"; bfutils_return_if_err
        else
            bfutils_log ERROR ${export_error_codes['CREATE_CONF_DIR_FAILED']} Failed to create fsx conf dir "${FLAGS_conf_dir}"
            return ${_install_errno}
        fi
    fi

    # 判断是初次安装还是故障恢复. 0:install 1:recovery 2:error
    local _install_state=2
    if ! [ -n "$(ls -A "${FLAGS_conf_dir}")" ]; then
        # conf_dir 空，认为是 install
        _install_state=0
    else
        # conf_dir 非空，判断是否包含 fusemonitor.service 及 conf，任缺其一，则有问题
        if [ -f "${FLAGS_conf_dir}/svc/fusemonitor.service" ] && [ $(find "${FLAGS_conf_dir}" -not -type d | wc -l) -ne 0 ]; then
            while read -r _conf_file_relative_path; do
                if [ -e ${FLAGS_conf_dir}/${_conf_file_relative_path} ]; then
                    _install_state=1
                fi
            done <<< "$(find "${deploy_dir}/conf" -not -type d | perl -pe s{"^${deploy_dir}"}{'.'})";
        fi
    fi

    if [ ${_install_state} -eq 2 ]; then
        local _has_residual_mountpoints=${FLAGS_FALSE}
        # 通过 json 判断是否有挂载点残留，若有挂载点残留则需要 enable fusemonitor.service 后走 recovery 流程
        for instance in "${supported_instances[@]}"; do
            if ! bfutils_check_mountpoints ${deploy_dir} ${instance} ${FLAGS_work_dir}; then
                bfutils_log ERROR fsx has residual ${instance} mountpoints
                _has_residual_mountpoints=${FLAGS_TRUE}
            fi
        done
        if [ ${_has_residual_mountpoints} -eq ${FLAGS_TRUE} ]; then
            bfutils_log ERROR fsx has residual mountpoints, try to enable fusemonitor and recover
            bfservice_enable_fusemonitor "${deploy_dir}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err
            _install_state=1
        else
            bfutils_log ERROR fsx try clean conf_dir and work_dir, then do install
            dependency_uninstall "${FLAGS_prefix}"; bfutils_return_if_err
            bfutils_work_remove "${FLAGS_work_dir}"; bfutils_return_if_err
            bfutils_conf_remove "${FLAGS_conf_dir}"; bfutils_return_if_err
            mkdir -p "${FLAGS_conf_dir}" "${FLAGS_work_dir}"; bfutils_return_if_err
            _install_state=0
        fi
    fi

    if ! [ -d "${FLAGS_log_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_log_dir}")" ] && mkdir "${FLAGS_log_dir}"; then
            bfutils_log DEBUG Try to logging into "\"${FLAGS_log_dir}\""
        else
            bfutils_log ERROR ${export_error_codes['CREATE_LOG_DIR_FAILED']} Failed to create fsx log dir "\"${FLAGS_log_dir}\"" # NOTE: No more than error log
            return 22 # EINVAL
        fi
    fi

    if bfutils_running_processes && [ -n "${__bfutils_return}" ]; then
        bfutils_log ERROR ${export_error_codes['MODULE_STILL_RUNNING']} Some fsx module is running, abort installation
        return 17 # EEXIST
    fi

    if [ ${_install_need_copy} -eq ${FLAGS_TRUE} ]; then
        bfutils_defer bfutils_binary_remove "${FLAGS_prefix}"
        bfutils_binary_copy "${FLAGS_prefix}" "${deploy_dir}"; bfutils_return_if_err
    fi

    if [ ${_install_state} -eq 0 ]; then
        bfutils_log INFO fsx start do install.
        bfutils_component_policy_init "${FLAGS_deploy_mode}"; bfutils_return_if_err

        # lock fsx start process
        bfutils_defer bfutils_work_remove "${FLAGS_work_dir}"
        mkdir -p "${FLAGS_work_dir}/conf"; bfutils_return_if_err
        bfutils_lock_ctl "${FLAGS_work_dir}/conf"; bfutils_return_if_err

        bfutils_defer bfutils_conf_remove "${FLAGS_conf_dir}"
        bfutils_conf_copy "${FLAGS_conf_dir}" "${deploy_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

        bfutils_defer dependency_uninstall "${FLAGS_prefix}"
        dependency_install "${FLAGS_prefix}"; bfutils_return_if_err

        bfutils_defer bfservice_disable_fsx_target "${FLAGS_conf_dir}"
        bfservice_enable_fsx_target "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

        bfutils_defer bfservice_disable_fusemonitor "${FLAGS_conf_dir}"
        bfservice_enable_fusemonitor "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

        bfutils_defer bfservice_stop_service 'fusemonitor'
        bfservice_start_service 'fusemonitor'; bfutils_return_if_err

        bfutils_defer bfservice_stop_service 'fsx.target'
        systemctl start fsx.target 2>/dev/null; bfutils_return_if_err  # fsx.target no running process

        bfutils_defer mount_scripts_uninstall
        mount_scripts_install "${FLAGS_prefix}" "${FLAGS_conf_dir}" "${FLAGS_work_dir}" "${FLAGS_log_dir}" "${FLAGS_deploy_mode}"; bfutils_return_if_err

        bfutils_defer env_uninstall "${FLAGS_prefix}"
        env_install "${FLAGS_prefix}"; bfutils_return_if_err

        bfutils_defer_clean
        bfutils_log INFO "${FUNCNAME}" succeed
        hpvs_rm_src_tar "${FLAGS_prefix}"
        return ${_install_errno}
    else
        bfutils_log INFO fsx start do recovery.

        recovery_inner_func "${deploy_dir}"
        _install_errno=$?

        # if recovery failed, first recover backup conf
        if [ ${_install_errno} -ne 0 ]; then
            local lock_error_code=$(bfutils_get_error_code "lock_error")
            local backup_conf_error_code=$(bfutils_get_error_code "backup_conf_error")

            if [[ "${_install_errno}" != "$lock_error_code" ]] && [[ "${_install_errno}" != "$backup_conf_error_code" ]]; then
                bfutils_backup_conf_recover "${FLAGS_conf_dir}"; bfutils_return_if_err
            fi
        fi

        # no matter success or failure, unlock mounts
        # bfutils_unlock_mounts; bfutils_return_if_err

        if [ ${_install_errno} -ne 0 ]; then
            bfutils_backup_conf_clean "${FLAGS_conf_dir}"; bfutils_return_if_err
            bfutils_log WARN "${_install_errno}"
        else
            bfutils_backup_conf_clean "${FLAGS_conf_dir}"; bfutils_return_if_err
            bfutils_defer_clean
            bfutils_log INFO "${FUNCNAME}" succeed
        fi
        return ${_install_errno}
    fi
}

bfutils_check_uds_service() {
    local instance="${1}"
    local _instance_name=${instance}
    local _error_code=${export_error_codes['FUSEMONITOR_UDS_ERROR']}
    if [ "x${_instance_name}" != "xfusemonitor" ]; then
        _instance_name="fusedaemon_${instance}"
        _error_code=${export_error_codes['FUSEDAEMON_UDS_ERROR']}
    fi

    bfutils_defer bfutils_log ERROR "${_error_code} Failed to check ${_instance_name} uds service"

    run_fusetool inner get_version --instance "${instance}"
    if [ $? -ne 0 ]; then
        return 1
    fi

    bfutils_log DEBUG "Check ${_instance_name} uds service ok, version: ${__bfutils_return}"
    bfutils_defer_clean

    return 0
}

fsx_status() {
    # status will be called frequently, so set logging level to warn if default
    if [ "${FLAGS_minloglevel}" -eq "${__bfutils_log_level_default}" ]; then
        bfutils_set_logging_level ${FLAGS_LEVEL_WARN}
    fi

    bfutils_log INFO Calling "${FUNCNAME}" prefix: "\"${FLAGS_prefix}\"" conf_dir: "\"${FLAGS_conf_dir}\"" work_dir: "\"${FLAGS_work_dir}\"" log_dir: "\"${FLAGS_log_dir}\""

    # check running fusemonitor
    bfutils_running_monitor
    if [ -z "${__bfutils_return}" ]; then
        bfutils_log ERROR "${export_error_codes['FUSEMONITOR_NOT_RUNNING']} fusemonitor is not running"
        return 1
    fi

    # Get running paths
    if bfutils_running_log_path && [ -n "${__bfutils_return}" ]; then
        FLAGS_log_dir="${__bfutils_return}"
    fi
    if bfutils_running_bin_path && [ -n "${__bfutils_return}" ]; then
        FLAGS_prefix="${__bfutils_return}"
    fi
    if bfutils_running_conf_path && [ -n "${__bfutils_return}" ]; then
        FLAGS_conf_dir=${__bfutils_return}
    fi
    if bfutils_running_work_path && [ -n "${__bfutils_return}" ]; then
        FLAGS_work_dir=${__bfutils_return}
    fi

    bfutils_check_uds_service "fusemonitor"; bfutils_return_if_err

    # check mountpoint json and fusedaemon process
    for instance in "${supported_instances[@]}"; do
        bfutils_get_mountpoints_json "${deploy_dir}" "${instance}" "${FLAGS_work_dir}"; bfutils_return_if_err
        if ! [ -n "${__bfutils_return}" ] || [ "x${__bfutils_return}" = "x[]" ]; then
            continue
        fi
        bfutils_running_daemons "${instance}"
        if [ -z "${__bfutils_return}" ]; then
            bfutils_log ERROR "${export_error_codes['FUSEDAEMON_NOT_RUNNING']} fusedaemon_${instance} is not running with alive mountpoints"
            return 1
        fi
        bfutils_check_uds_service "${instance}"; bfutils_return_if_err
    done

    return 0
}

bfutils_init() {
    if [ "${FSX_SHUTILS_INITED+x}" = "x" ]; then
        bfutils_log WARN bfutils_init already called
        return
    fi
    FSX_SHUTILS_INITED=''
    if ! [ -d "${FLAGS_log_dir}" ]; then
        if [ -d "$(dirname "${FLAGS_log_dir}")" ] && mkdir "${FLAGS_log_dir}"; then
            bfutils_log DEBUG Succeed to create fsx log dir "${FLAGS_log_dir}"
        else
            bfutils_log ERROR ${export_error_codes['CREATE_LOG_DIR_FAILED_WHEN_INIT']} Failed to create fsx log dir "${FLAGS_log_dir}"
        fi
    fi
    if [ "${FLAGS_minloglevel}" -ne "${__bfutils_log_level_default}" ]; then
        __bfutils_log_level="${FLAGS_minloglevel}"
    fi
}
