理解 OpenStack 高可用(HA) （4）: Pacemaker 和 OpenStack Resource Agent （RA）

usage() { #RA 的功能，包括 start，stop，validate-all，meta-data，status 和 monitor glance-api 等，每个对应下面的一个函数
    cat <<UEND
        usage: $0 (start|stop|validate-all|meta-data|status|monitor)
        $0 manages an OpenStack ImageService (glance-api) process as an HA resource 
        The 'start' operation starts the imaging service.
        The 'stop' operation stops the imaging service.
        The 'validate-all' operation reports whether the parameters are valid
        The 'meta-data' operation reports this RA's meta-data information
        The 'status' operation reports whether the imaging service is running
        The 'monitor' operation reports whether the imaging service seems to be working
UEND
}

meta_data() { #meta-data 功能，输出一段XML
    cat <<END
...
END
}

#######################################################################
# Functions invoked by resource manager actions

glance_api_validate() { #检查 glance-api，比如libaray是否存在，配置文件是否存在，RA 使用的用户是否存在
    local rc

    check_binary $OCF_RESKEY_binary
    check_binary $OCF_RESKEY_client_binary

    # A config file on shared storage that is not available
    # during probes is OK.
    if [ ! -f $OCF_RESKEY_config ]; then
        if ! ocf_is_probe; then
            ocf_log err "Config $OCF_RESKEY_config doesn't exist"
            return $OCF_ERR_INSTALLED
        fi
        ocf_log_warn "Config $OCF_RESKEY_config not available during a probe"
    fi

    getent passwd $OCF_RESKEY_user >/dev/null 2>&1
    rc=$?
    if [ $rc -ne 0 ]; then
        ocf_log err "User $OCF_RESKEY_user doesn't exist"
        return $OCF_ERR_INSTALLED
    fi

    true
}

glance_api_status() { #获取运行状态，通过检查 pid 文件来确认 glance-api 是否在运行
    local pid
    local rc

    if [ ! -f $OCF_RESKEY_pid ]; then
        ocf_log info "OpenStack ImageService (glance-api) is not running"
        return $OCF_NOT_RUNNING
    else
        pid=`cat $OCF_RESKEY_pid`
    fi

    ocf_run -warn kill -s 0 $pid
    rc=$?
    if [ $rc -eq 0 ]; then
        return $OCF_SUCCESS
    else
        ocf_log info "Old PID file found, but OpenStack ImageService (glance-api) is not running"
        return $OCF_NOT_RUNNING
    fi
}

glance_api_monitor() { #监控 glance-api 服务的运行状态，通过运行 glance image-list 命令
    local rc

    glance_api_status
    rc=$?

    # If status returned anything but success, return that immediately
    if [ $rc -ne $OCF_SUCCESS ]; then
        return $rc
    fi

    # Monitor the RA by retrieving the image list
    if [ -n "$OCF_RESKEY_os_username" ] && [ -n "$OCF_RESKEY_os_password" ] 
    && [ -n "$OCF_RESKEY_os_tenant_name" ] && [ -n "$OCF_RESKEY_os_auth_url" ]; then
        ocf_run -q $OCF_RESKEY_client_binary 
        --os_username "$OCF_RESKEY_os_username" 
        --os_password "$OCF_RESKEY_os_password" 
        --os_tenant_name "$OCF_RESKEY_os_tenant_name" 
        --os_auth_url "$OCF_RESKEY_os_auth_url" 
        index > /dev/null 2>&1
        rc=$?
        if [ $rc -ne 0 ]; then
            ocf_log err "Failed to connect to the OpenStack ImageService (glance-api): $rc"
            return $OCF_NOT_RUNNING
        fi
    fi

    ocf_log debug "OpenStack ImageService (glance-api) monitor succeeded"
    return $OCF_SUCCESS
}

glance_api_start() { #启动 glance-api 服务
    local rc

    glance_api_status
    rc=$?
    if [ $rc -eq $OCF_SUCCESS ]; then
        ocf_log info "OpenStack ImageService (glance-api) already running"
        return $OCF_SUCCESS
    fi

    # run the actual glance-api daemon. Don't use ocf_run as we're sending the tool's output
    # straight to /dev/null anyway and using ocf_run would break stdout-redirection here.
    su ${OCF_RESKEY_user} -s /bin/sh -c "${OCF_RESKEY_binary} --config-file $OCF_RESKEY_config 
      $OCF_RESKEY_additional_parameters"' >> /dev/null 2>&1 & echo $!' > $OCF_RESKEY_pid

    # Spin waiting for the server to come up.
    # Let the CRM/LRM time us out if required
    while true; do
    glance_api_monitor
    rc=$?
    [ $rc -eq $OCF_SUCCESS ] && break
    if [ $rc -ne $OCF_NOT_RUNNING ]; then
        ocf_log err "OpenStack ImageService (glance-api) start failed"
        exit $OCF_ERR_GENERIC
    fi
    sleep 1
    done

    ocf_log info "OpenStack ImageService (glance-api) started"
    return $OCF_SUCCESS
}

glance_api_stop() { #停止 glance-api 服务
    local rc
    local pid

    glance_api_status
    rc=$?
    if [ $rc -eq $OCF_NOT_RUNNING ]; then
        ocf_log info "OpenStack ImageService (glance-api) already stopped"
        return $OCF_SUCCESS
    fi

    # Try SIGTERM
    pid=`cat $OCF_RESKEY_pid`
    ocf_run kill -s TERM $pid
    rc=$?
    if [ $rc -ne 0 ]; then
        ocf_log err "OpenStack ImageService (glance-api) couldn't be stopped"
        exit $OCF_ERR_GENERIC
    fi

    # stop waiting
    shutdown_timeout=15
    if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then
        shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5))
    fi
    count=0
    while [ $count -lt $shutdown_timeout ]; do
        glance_api_status
        rc=$?
        if [ $rc -eq $OCF_NOT_RUNNING ]; then
            break
        fi
        count=`expr $count + 1`
        sleep 1
        ocf_log debug "OpenStack ImageService (glance-api) still hasn't stopped yet. Waiting ..."
    done

    glance_api_status
    rc=$?
    if [ $rc -ne $OCF_NOT_RUNNING ]; then
        # SIGTERM didn't help either, try SIGKILL
        ocf_log info "OpenStack ImageService (glance-api) failed to stop after ${shutdown_timeout}s 
          using SIGTERM. Trying SIGKILL ..."
        ocf_run kill -s KILL $pid
    fi

    ocf_log info "OpenStack ImageService (glance-api) stopped"

    rm -f $OCF_RESKEY_pid

    return $OCF_SUCCESS
}

#######################################################################

case "$1" in
  meta-data)    meta_data
                exit $OCF_SUCCESS;;
  usage|help)   usage
                exit $OCF_SUCCESS;;
esac

# Anything except meta-data and help must pass validation
glance_api_validate || exit $?

# What kind of method was invoked?
case "$1" in
  start)        glance_api_start;;
  stop)         glance_api_stop;;
  status)       glance_api_status;;
  monitor)      glance_api_monitor;;
  validate-all) ;;
  *)            usage
                exit $OCF_ERR_UNIMPLEMENTED;;
esac

primitive p_glance-api ocf:openstack:glance-api 
params config="/etc/glance/glance-api.conf" os_password="secretsecret" 
os_username="admin" os_tenant_name="admin" os_auth_url="http://192.168.42.
103:5000/v2.0/" 
op monitor interval="30s" timeout="30s"

group g_services_api p_api-ip p_keystone p_glance-api p_cinder-api p_neutron-server p_glance-registry p_ceilometer-agent-central