在监控设备的时候，在server端的日志中有时候会见到类似another network error, wait for 15s seconds的异常，今天我们看下这个问题的出现原因和解决方案:

问题定位到poller.c，看下下面两份代码:

这个get_values的部分代码:

  1.     for (i = 0; i < num; i++)
  2.     {
  3.         switch (errcodes[i])
  4.         {
  5.             case SUCCEED:
  6.             case NOTSUPPORTED:
  7.             case AGENT_ERROR:
  8.                 if (HOST_AVAILABLE_TRUE != last_available)
  9.                 {
  10.                     zbx_activate_item_host(&items[i], &timespec);
  11.                     last_available = HOST_AVAILABLE_TRUE;
  12.                 }
  13.                 break;
  14.             case NETWORK_ERROR:
  15.             case GATEWAY_ERROR:
  16.             case TIMEOUT_ERROR:
  17.                 if (HOST_AVAILABLE_FALSE != last_available)
  18.                 {
  19.                     zbx_deactivate_item_host(&items[i], &timespec, results[i].msg);
  20.                     last_available = HOST_AVAILABLE_FALSE;
  21.                 }
  22.                 break;
  23.             case CONFIG_ERROR:
  24.                 /* nothing to do */
  25.                 break;
  26.             default:
  27.                 zbx_error("unknown response code returned: %d", errcodes[i]);
  28.                 THIS_SHOULD_NEVER_HAPPEN;
  29.         }

这里是zbx_deactivate_item_host的代码:

  1. void    zbx_deactivate_item_host(DC_ITEM *item, zbx_timespec_t *ts, const char *error)                       //   #0
  2. {
  3.     const char        *__function_name = "zbx_deactivate_item_host";
  4.     zbx_host_availability_t    in, out;                                                                         //   #1
  5.     unsigned char        agent_type;                                                                          //   #2
  7.     zabbix_log(LOG_LEVEL_DEBUG, "In %s() hostid:" ZBX_FS_UI64 " itemid:" ZBX_FS_UI64 " type:%d",             //   #3
  8.             __function_name, item->host.hostid, item->itemid, (int)item->type);
  10.     zbx_host_availability_init(&in, item->host.hostid);                                                      //   #4
  11.     zbx_host_availability_init(&out,item->host.hostid);                                                      //   #5
  12.  
  13.     if (ZBX_AGENT_UNKNOWN == (agent_type = host_availability_agent_by_item_type(item->type)))                //   #6
  14.         goto out;
  16.     if (FAIL == host_get_availability(&item->host, agent_type, &in))                                         //   #7
  17.         goto out;
  19.     if (FAIL == DChost_deactivate(item->host.hostid, agent_type, ts, &in.agents[agent_type],                 //   #8
  20.             &out.agents[agent_type], error))
  21.     {
  22.         goto out;
  23.     }
  25.     if (FAIL == db_host_update_availability(&out))                                                           //   #9
  26.         goto out;
  28.     host_set_availability(&item->host, agent_type, &out);                                                    //   #10
  29.  
  30.     if (0 == in.agents[agent_type].errors_from)                                                              //   #11
  31.     {
  32.         zabbix_log(LOG_LEVEL_WARNING, "%s item \"%s\" on host \"%s\" failed:"                                //   #12
  33.                 " first network error, wait for %d seconds",
  34.                 zbx_agent_type_string(item->type), item->key_orig, item->host.host,
  35.                 out.agents[agent_type].disable_until - ts->sec);
  36.     }
  37.     else
  38.     {
  39.         if (HOST_AVAILABLE_FALSE != in.agents[agent_type].available)                                         //   #13
  40.         {
  41.             if (HOST_AVAILABLE_FALSE != out.agents[agent_type].available)                                    //   #14
  42.             {
  43.                 zabbix_log(LOG_LEVEL_WARNING, "%s item \"%s\" on host \"%s\" failed:"                        //   #15
  44.                         " another network error, wait for %d seconds",
  45.                         zbx_agent_type_string(item->type), item->key_orig, item->host.host,
  46.                         out.agents[agent_type].disable_until - ts->sec);
  47.             }
  48.             else
  49.             {
  50.                 zabbix_log(LOG_LEVEL_WARNING, "temporarily disabling %s checks on host \"%s\":"              //   #16
  51.                         " host unavailable",
  52.                         zbx_agent_type_string(item->type), item->host.host);
  53.             }
  54.         }
  55.     }
  57.     zabbix_log(LOG_LEVEL_DEBUG, "%s() errors_from:%d available:%d", __function_name,
  58.             out.agents[agent_type].errors_from, out.agents[agent_type].available);
  59. out:
  60.     zbx_host_availability_clean(&out);
  61.     zbx_host_availability_clean(&in);
  63.     zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __function_name);
  64. }

下面看下这里是zbx_deactivate_item_host的代码的逻辑：

#0  zbx_deactivate_item_host函数接收三个参数

  1.         1 结构体指针，主机的一些综合参数
  2.             //dbcache.h 
  3.             typedef struct
  4.             {
  5.                 DC_HOST            host;
  6.                 DC_INTERFACE        interface;
  7.                 zbx_uint64_t        itemid;
  8.                 zbx_uint64_t        lastlogsize;
  9.                 zbx_uint64_t        valuemapid;
  10.                 unsigned char        type;
  11.                 unsigned char        value_type;
  12.                 unsigned char        state;
  13.                 unsigned char        snmpv3_securitylevel;
  14.                 unsigned char        authtype;
  15.                 unsigned char        flags;
  16.                 unsigned char        snmpv3_authprotocol;
  17.                 unsigned char        snmpv3_privprotocol;
  18.                 unsigned char        inventory_link;
  19.                 unsigned char        status;
  20.                 unsigned char        history;
  21.                 unsigned char        trends;
  22.                 unsigned char        follow_redirects;
  23.                 unsigned char        post_type;
  24.                 unsigned char        retrieve_mode;
  25.                 unsigned char        request_method;
  26.                 unsigned char        output_format;
  27.                 unsigned char        verify_peer;
  28.                 unsigned char        verify_host;
  29.                 unsigned char        allow_traps;
  30.                 char            key_orig[ITEM_KEY_LEN * ZBX_MAX_BYTES_IN_UTF8_CHAR + 1], *key;
  31.                 char            *units;
  32.                 char            *delay;
  33.                 int            history_sec;
  34.                 int            nextcheck;
  35.                 int            lastclock;
  36.                 int            mtime;
  37.                 char            trapper_hosts[ITEM_TRAPPER_HOSTS_LEN_MAX];
  38.                 char            logtimefmt[ITEM_LOGTIMEFMT_LEN_MAX];
  39.                 char            snmp_community_orig[ITEM_SNMP_COMMUNITY_LEN_MAX], *snmp_community;
  40.                 char            snmp_oid_orig[ITEM_SNMP_OID_LEN_MAX], *snmp_oid;
  41.                 char            snmpv3_securityname_orig[ITEM_SNMPV3_SECURITYNAME_LEN_MAX], *snmpv3_securityname;
  42.                 char            snmpv3_authpassphrase_orig[ITEM_SNMPV3_AUTHPASSPHRASE_LEN_MAX], *snmpv3_authpassphrase;
  43.                 char            snmpv3_privpassphrase_orig[ITEM_SNMPV3_PRIVPASSPHRASE_LEN_MAX], *snmpv3_privpassphrase;
  44.                 char            ipmi_sensor[ITEM_IPMI_SENSOR_LEN_MAX];
  45.                 char            *params;
  46.                 char            username_orig[ITEM_USERNAME_LEN_MAX], *username;
  47.                 char            publickey_orig[ITEM_PUBLICKEY_LEN_MAX], *publickey;
  48.                 char            privatekey_orig[ITEM_PRIVATEKEY_LEN_MAX], *privatekey;
  49.                 char            password_orig[ITEM_PASSWORD_LEN_MAX], *password;
  50.                 char            snmpv3_contextname_orig[ITEM_SNMPV3_CONTEXTNAME_LEN_MAX], *snmpv3_contextname;
  51.                 char            jmx_endpoint_orig[ITEM_JMX_ENDPOINT_LEN_MAX], *jmx_endpoint;
  52.                 char            timeout_orig[ITEM_TIMEOUT_LEN_MAX], *timeout;
  53.                 char            url_orig[ITEM_URL_LEN_MAX], *url;
  54.                 char            query_fields_orig[ITEM_QUERY_FIELDS_LEN_MAX], *query_fields;
  55.                 char            *posts;
  56.                 char            status_codes_orig[ITEM_STATUS_CODES_LEN_MAX], *status_codes;
  57.                 char            http_proxy_orig[ITEM_HTTP_PROXY_LEN_MAX], *http_proxy;
  58.                 char            *headers;
  59.                 char            ssl_cert_file_orig[ITEM_SSL_CERT_FILE_LEN_MAX], *ssl_cert_file;
  60.                 char            ssl_key_file_orig[ITEM_SSL_KEY_FILE_LEN_MAX], *ssl_key_file;
  61.                 char            ssl_key_password_orig[ITEM_SSL_KEY_PASSWORD_LEN_MAX], *ssl_key_password;
  62.                 char            *error;
  63.             }
  64.             DC_ITEM;
  65.         2 结构体指针
  66.             //common.h
  67.             typedef struct
  68.             {
  69.                 int    sec;    /* seconds */
  70.                 int    ns;    /* nanoseconds */
  71.             }
  72.             zbx_timespec_t;
  73.             
  74.         3 错误信息

#1 定义了两个结构体数组 in 和 out

  1.         //db.h
  2.         typedef struct
  3.         {
  4.             /* flags specifying which fields are set, see ZBX_FLAGS_AGENT_STATUS_* defines */
  5.             unsigned char    flags;
  7.             /* agent availability fields */
  8.             unsigned char    available;
  9.             char        *error;
  10.             int        errors_from;
  11.             int        disable_until;
  12.         }
  13.         zbx_agent_availability_t;
  15.         typedef struct
  16.         {
  17.             zbx_uint64_t            hostid;
  19.             zbx_agent_availability_t    agents[ZBX_AGENT_MAX];         //这里的ZBX_AGENT_MAX 为4 ，分别代表ZABBIX, SNMP, IPMI, JMX4种类型
  20.         }
  21.         zbx_host_availability_t;

#2 声明unsigned char agent_type，unsigned char和char的区别是char表示-128-127，unsigned char 表示0-255，这里的255会在后面遇到，所以需要255的这个表示范围

#3 记录DEBUG 的log，如果需要显示这份日志，需要将server端的配置文件debug等级更改为5，不过我不建议你这么做

#4 初始化主机IN可用性数据

  1.     //dbconfig.c
  2.     void    zbx_host_availability_init(zbx_host_availability_t *availability, zbx_uint64_t hostid)
  3.     {
  4.         memset(availability, 0, sizeof(zbx_host_availability_t));
  5.         availability->hostid = hostid;
  6.     }

 

#5 同#4一样，只不过是OUT

 

#6 为agent_type赋值，如果agent_type不属于#1中的四种，跳至out处

  1.     1、host_availability_agent_by_item_type 位于poller.c，接收item的type字段，用来判断监控类型
  2.     //poller.c
  3.         static unsigned char    host_availability_agent_by_item_type(unsigned char type)
  4.         {
  5.             switch (type)
  6.             {
  7.                 case ITEM_TYPE_ZABBIX:
  8.                     return ZBX_AGENT_ZABBIX;
  9.                     break;
  10.                 case ITEM_TYPE_SNMPv1:
  11.                 case ITEM_TYPE_SNMPv2c:
  12.                 case ITEM_TYPE_SNMPv3:
  13.                     return ZBX_AGENT_SNMP;
  14.                     break;
  15.                 case ITEM_TYPE_IPMI:
  16.                     return ZBX_AGENT_IPMI;
  17.                     break;
  18.                 case ITEM_TYPE_JMX:
  19.                     return ZBX_AGENT_JMX;
  20.                     break;
  21.                 default:
  22.                     return ZBX_AGENT_UNKNOWN;
  23.             }
  24.         }
  25.     2、ZBX_AGENT_UNKNOWN 常量 为 255 对应之前的 #2

 

#7 根据agent_type来判断主机的可用性，网络设备会匹配到ZBX_AGENT_SNMP，四个值分别代表的意思是

  1.     //poller.c
  2.     static int    host_get_availability(const DC_HOST *dc_host, unsigned char agent, zbx_host_availability_t *ha)
  3.     {
  4.         zbx_agent_availability_t    *availability = &ha->agents[agent];
  6.         availability->flags = ZBX_FLAGS_AGENT_STATUS;
  8.         switch (agent)
  9.         {
  10.             case ZBX_AGENT_ZABBIX:
  11.                 availability->available = dc_host->available;
  12.                 availability->error = zbx_strdup(NULL, dc_host->error);
  13.                 availability->errors_from = dc_host->errors_from;
  14.                 availability->disable_until = dc_host->disable_until;
  15.                 break;
  16.             case ZBX_AGENT_SNMP:
  17.                 availability->available = dc_host->snmp_available;  //主机的snmp可用状态
  18.                 availability->error = zbx_strdup(NULL, dc_host->snmp_error);  //错误信息
  19.                 availability->errors_from = dc_host->snmp_errors_from;      //错误发生时间
  20.                 availability->disable_until = dc_host->snmp_disable_until;  //下次延迟检测时间
  21.                 break;
  22.             case ZBX_AGENT_IPMI:
  23.                 availability->available = dc_host->ipmi_available;
  24.                 availability->error = zbx_strdup(NULL, dc_host->ipmi_error);
  25.                 availability->errors_from = dc_host->ipmi_errors_from;
  26.                 availability->disable_until = dc_host->ipmi_disable_until;
  27.                 break;
  28.             case ZBX_AGENT_JMX:
  29.                 availability->available = dc_host->jmx_available;
  30.                 availability->error = zbx_strdup(NULL, dc_host->jmx_error);
  31.                 availability->disable_until = dc_host->jmx_disable_until;
  32.                 availability->errors_from = dc_host->jmx_errors_from;
  33.                 break;
  34.             default:
  35.                 return FAIL;
  36.         }
  38.         ha->hostid = dc_host->hostid;
  40.         return SUCCEED;
  41.     }
  43.     //dbcache.h
  44.     typedef struct
  45.     {
  46.         zbx_uint64_t    hostid;
  47.         zbx_uint64_t    proxy_hostid;
  48.         char        host[HOST_HOST_LEN_MAX];
  49.         char        name[HOST_NAME_LEN * ZBX_MAX_BYTES_IN_UTF8_CHAR + 1];
  50.         unsigned char    maintenance_status;
  51.         unsigned char    maintenance_type;
  52.         int        maintenance_from;
  53.         int        errors_from;
  54.         unsigned char    available;
  55.         int        disable_until;
  56.         int        snmp_errors_from;
  57.         unsigned char    snmp_available;
  58.         int        snmp_disable_until;
  59.         int        ipmi_errors_from;
  60.         unsigned char    ipmi_available;
  61.         int        ipmi_disable_until;
  62.         signed char    ipmi_authtype;
  63.         unsigned char    ipmi_privilege;
  64.         char        ipmi_username[HOST_IPMI_USERNAME_LEN_MAX];
  65.         char        ipmi_password[HOST_IPMI_PASSWORD_LEN_MAX];
  66.         int        jmx_errors_from;
  67.         unsigned char    jmx_available;
  68.         int        jmx_disable_until;
  69.         char        inventory_mode;
  70.         unsigned char    status;
  71.         unsigned char    tls_connect;
  72.         unsigned char    tls_accept;
  73.     #if defined(HAVE_POLARSSL) || defined(HAVE_GNUTLS) || defined(HAVE_OPENSSL)
  74.         char        tls_issuer[HOST_TLS_ISSUER_LEN_MAX];
  75.         char        tls_subject[HOST_TLS_SUBJECT_LEN_MAX];
  76.         char        tls_psk_identity[HOST_TLS_PSK_IDENTITY_LEN_MAX];
  77.         char        tls_psk[HOST_TLS_PSK_LEN_MAX];
  78.     #endif
  79.         char        error[HOST_ERROR_LEN_MAX];
  80.         char        snmp_error[HOST_ERROR_LEN_MAX];
  81.         char        ipmi_error[HOST_ERROR_LEN_MAX];
  82.         char        jmx_error[HOST_ERROR_LEN_MAX];
  83.     }
  84.     DC_HOST;
  85.     
  86.     //db.h
  87.     #define ZBX_FLAGS_AGENT_STATUS_AVAILABLE    0x00000001
  88.     #define ZBX_FLAGS_AGENT_STATUS_ERROR        0x00000002
  89.     #define ZBX_FLAGS_AGENT_STATUS_ERRORS_FROM    0x00000004
  90.     #define ZBX_FLAGS_AGENT_STATUS_DISABLE_UNTIL    0x00000008
  91.     #define ZBX_FLAGS_AGENT_STATUS        (ZBX_FLAGS_AGENT_STATUS_AVAILABLE |    \
  92.                         ZBX_FLAGS_AGENT_STATUS_ERROR |        \
  93.                         ZBX_FLAGS_AGENT_STATUS_ERRORS_FROM |    \
  94.                         ZBX_FLAGS_AGENT_STATUS_DISABLE_UNTIL) 
  96.      
  97.     //common.h
  98.     #define    FAIL        -1

 

#8  根据agent_type 设置主机状态

  1.     //dbconfig.c
  2.     int    DChost_deactivate(zbx_uint64_t hostid, unsigned char agent_type, const zbx_timespec_t *ts,
  3.             zbx_agent_availability_t *in, zbx_agent_availability_t *out, const char *error_msg)
  4.     {
  5.         int        ret = FAIL, errors_from,disable_until;
  6.         const char    *error;
  7.         unsigned char    available;
  8.         ZBX_DC_HOST    *dc_host;
  11.         /* don't try deactivating host if the unreachable delay has not passed since the first error */
  12.         if (CONFIG_UNREACHABLE_DELAY > ts->sec - in->errors_from) 
  13.             goto out;
  15.         WRLOCK_CACHE;
  17.         if (NULL == (dc_host = (ZBX_DC_HOST *)zbx_hashset_search(&config->hosts, &hostid)))
  18.             goto unlock;
  20.         /* Don't try deactivating host if:                */
  21.         /* - (server, proxy) it's not monitored any more; */
  22.         /* - (server) it's monitored by proxy.            */
  23.         if ((0 != (program_type & ZBX_PROGRAM_TYPE_SERVER) && 0 != dc_host->proxy_hostid) ||
  24.                 HOST_STATUS_MONITORED != dc_host->status)
  25.         {
  26.             goto unlock;
  27.         }
  29.         DChost_get_agent_availability(dc_host, agent_type, in);
  31.         available = in->available;
  32.         error = in->error;
  34.         if (0 == in->errors_from)
  35.         {
  36.             /* first error, schedule next unreachable check */
  37.             errors_from = ts->sec;
  38.             disable_until = ts->sec + CONFIG_UNREACHABLE_DELAY;
  39.         }
  40.         else
  41.         {
  42.             errors_from = in->errors_from;
  43.             disable_until = in->disable_until;
  45.             /* Check if other pollers haven't already attempted deactivating host. */
  46.             /* In that case should wait the initial unreachable delay before       */
  47.             /* trying to make it unavailable.                                      */
  48.             if (CONFIG_UNREACHABLE_DELAY <= ts->sec - errors_from)
  49.             {
  50.                 /* repeating error */
  51.                 if (CONFIG_UNREACHABLE_PERIOD > ts->sec - errors_from)
  52.                 {
  53.                     /* leave host available, schedule next unreachable check */
  54.                     disable_until = ts->sec + CONFIG_UNREACHABLE_DELAY;
  55.                 }
  56.                 else
  57.                 {
  58.                     /* make host unavailable, schedule next unavailable check */
  59.                     disable_until = ts->sec + CONFIG_UNAVAILABLE_DELAY;
  60.                     available = HOST_AVAILABLE_FALSE;
  61.                     error = error_msg;
  62.                 }
  63.             }
  64.         }
  66.         zbx_agent_availability_init(out, available, error, errors_from, disable_until);
  67.         DChost_set_agent_availability(dc_host, ts->sec, agent_type, out);
  69.         if (ZBX_FLAGS_AGENT_STATUS_NONE != out->flags)
  70.             ret = SUCCEED;
  71.     unlock:
  72.         UNLOCK_CACHE;
  73.     out:
  74.         return ret;
  75.     }

主要看下这段:

  1. if (0 == in->errors_from)
  2.         {
  3.             /* first error, schedule next unreachable check */
  4.             errors_from = ts->sec;
  5.             disable_until = ts->sec + CONFIG_UNREACHABLE_DELAY;
  6.         }
  7.         else
  8.         {
  9.             errors_from = in->errors_from;
  10.             disable_until = in->disable_until;
  11.  
  12.             /* Check if other pollers haven't already attempted deactivating host. */
  13.             /* In that case should wait the initial unreachable delay before       */
  14.             /* trying to make it unavailable.                                      */
  15.             if (CONFIG_UNREACHABLE_DELAY <= ts->sec - errors_from)
  16.             {
  17.                 /* repeating error */
  18.                 if (CONFIG_UNREACHABLE_PERIOD > ts->sec - errors_from)
  19.                 {
  20.                     /* leave host available, schedule next unreachable check */
  21.                     disable_until = ts->sec + CONFIG_UNREACHABLE_DELAY;
  22.                 }
  23.                 else
  24.                 {
  25.                     /* make host unavailable, schedule next unavailable check */
  26.                     disable_until = ts->sec + CONFIG_UNAVAILABLE_DELAY;
  27.                     available = HOST_AVAILABLE_FALSE;
  28.                     error = error_msg;
  29.                 }
  30.             }
  31.         }
  1.         如果错误第一次出现:
  2.             错误发生时间=检查的时间戳
  3.             下次的检查时间 = 时间戳+15s
  4.         否则:
  5.             错误发生时间 = in->errors_from
  6.             下次检查时间 = in->disable_until
  8.             检查的时间戳-错误发生时间>=15s:
  9.                 检查的时间戳-错误发生时间< 45s:
  10.                     下次的检查时间 = 检查的时间戳+15s
  11.                 否则:
  12.                     下一次检查时间 =检查的时间戳+15s
  13.                     主机可用性为不可用
  14.

  用配置文件来解释就是: 如果由于网络等原因没有实现项目的及时监控，第一次的监控间隔为UnreachableDelay时间(15s),如果这次也失败了，那么从第一次失败到本次检查在UnreachablePeriod时间内，会再次在UnreachableDelay时间后监控

#9 更新数据库中的主机可用性信息

  1.     // poller.c
  2.     static int    db_host_update_availability(const zbx_host_availability_t *ha)
  3.     {
  4.         char    *sql = NULL;
  5.         size_t    sql_alloc = 0, sql_offset = 0;
  7.         if (SUCCEED == zbx_sql_add_host_availability(&sql, &sql_alloc, &sql_offset, ha))
  8.         {
  9.             DBbegin();
  10.             DBexecute("%s", sql);
  11.             DBcommit();
  13.             zbx_free(sql);
  15.             return SUCCEED;
  16.         }
  18.         return FAIL;
  19.     }

 

#10 根据agent_type设置主机可用性信息

  1.     //poller.c
  2.     static int    host_set_availability(DC_HOST *dc_host, unsigned char agent, const zbx_host_availability_t *ha)
  3.     {
  4.         const zbx_agent_availability_t    *availability = &ha->agents[agent];
  5.         unsigned char            *pavailable;
  6.         int                *perrors_from, *pdisable_until;
  7.         char                *perror;
  9.         switch (agent)
  10.         {
  11.             case ZBX_AGENT_ZABBIX:
  12.                 pavailable = &dc_host->available;
  13.                 perror = dc_host->error;
  14.                 perrors_from = &dc_host->errors_from;
  15.                 pdisable_until = &dc_host->disable_until;
  16.                 break;
  17.             case ZBX_AGENT_SNMP:
  18.                 pavailable = &dc_host->snmp_available;
  19.                 perror = dc_host->snmp_error;
  20.                 perrors_from = &dc_host->snmp_errors_from;
  21.                 pdisable_until = &dc_host->snmp_disable_until;
  22.                 break;
  23.             case ZBX_AGENT_IPMI:
  24.                 pavailable = &dc_host->ipmi_available;
  25.                 perror = dc_host->ipmi_error;
  26.                 perrors_from = &dc_host->ipmi_errors_from;
  27.                 pdisable_until = &dc_host->ipmi_disable_until;
  28.                 break;
  29.             case ZBX_AGENT_JMX:
  30.                 pavailable = &dc_host->jmx_available;
  31.                 perror = dc_host->jmx_error;
  32.                 pdisable_until = &dc_host->jmx_disable_until;
  33.                 perrors_from = &dc_host->jmx_errors_from;
  34.                 break;
  35.             default:
  36.                 return FAIL;
  37.         }
  39.         if (0 != (availability->flags & ZBX_FLAGS_AGENT_STATUS_AVAILABLE))
  40.             *pavailable = availability->available;
  42.         if (0 != (availability->flags & ZBX_FLAGS_AGENT_STATUS_ERROR))
  43.             zbx_strlcpy(perror, availability->error, HOST_ERROR_LEN_MAX);
  45.         if (0 != (availability->flags & ZBX_FLAGS_AGENT_STATUS_ERRORS_FROM))
  46.             *perrors_from = availability->errors_from;
  48.         if (0 != (availability->flags & ZBX_FLAGS_AGENT_STATUS_DISABLE_UNTIL))
  49.             *pdisable_until = availability->disable_until;
  51.         return SUCCEED;
  52.     }

#11-16
    如果是第一次检查：
       　　 记录日志first network error, wait for 15 seconds
    否则:
       　　 如果数据库中的主机如果显示可用:
            　　　　记录日志another network error, wait for 15 seconds
        　　否则
            　　　　记录日志temporarily disabling(这是前段页面的绿色图标会变为红色)

 

 

从上面的代码可以看出，在三中情况下会产生network error, wait for 15s seconds的日志，分别是在poller过程中产生的网络错误，网关问题，或者是检查超时。总结下来就是:zabbix server 与zabbix agentd的连接和数据的收发不能成功或者在取得数据的一系列处理中花费的时间超过了zabbix server 的Timeout参数情况下发生。

 

从正常取值到出现异常的处理过程是这样的:

正常取值   UnreachableDelay  UnreachableDelay   UnreachableDelay      UnnavailableDelay        恢复
                  |                                       |                              |
                  |                                       |                              |
                  ———————–UnreachablePeriod————
    1             2                                                              3                                4                           5

 

       过程                                        日志                                                               

  • 1 获取正常监控数据
    2 发生错误                 ————>first network
    3 再次发生错误          ————>another network
    4 置为不可用            ————>temporarily disabling
    5 恢复                        ————>resuming

日志中的15s在配置文件中对应的配置UnreachableDelay,默认为15s，在源码中的位置是server.c中的CONFIG_UNREACHABLE_DELAY，
但注意这个配置不会解决任何network error的问题，只是为计算下一个检查时间提供时间依据。还有大家应该注意到了UnreachableDelay参数和UnreachablePeriod是倍数关系。我们在调优的时候需要注意下。

从zabbix 1.8版使用至今，根据我这几年的经验分析产生此类日志基本出现在网络设备，服务器很少出现，这与SNMP使用UDP协议有关系，但主要问题还是几方面问题:

  • 1、网络不稳定
  • 2、设备端问题
  • 3、poller排队了
  • 4、Timeout超时了

这四种中的Timeout和poller又是有相互联系的，关于服务器如何设置poller，我后面的文章再介绍，先暂时分别来看下这四种情况:

网络不稳定多出现于几种情况:

  •     1、使用公网实现和IDC互连，也就是被检查设备和server不在一个IDC，这种情况建议在另一端增加proxy，使对端设备的检测都在内网进行
  •     2、使用云端网络，使用云端的网络互连方式打通云端设备和IDC的互连，这种情况的网络对于用户来说就是一种黑盒，基本无法排障，如果你使用大厂的服务，会偶尔出现日志报错，但不会影响到使用体验

网络设备端问题的情况:

  • 1、设备性能:如何判断网络设备端问题呢?可以在网络设备上debug snmp信息，看每个包是否是都回了还是报错了，这种情况可以将snmp的采取间隔加大，
  • 2、对端和server连接的端口带宽打满了

poller排队处理;
    poller数量是由zabbix_server配置文件中的startpollers指定，poller.c主要做几件事:1、从队列中获取item的数据 2、获取item获取监控数据 3、把数据放入缓存
    poller只会处理被动状态的监控项:
        如果你是服务器出现此类日志:解决方法一种是增大poller的数量，一种是把被动模式改为主动模式，
        如果你是网络设备:改用脚本实现，或者增大poller数量

关于Timeout ,这里有同学可能会说将服务器的检查时间调长为30s，这种设置如果检查设备少没关系，数量比较多我不建议这样调整，超过2s的检测项都改在agentd改用脚本实现吧

 

以上，是我使用zabbix中关于日志报警wait for 15s seconds 的一些理解和心得，如果文章内容对你有所帮助，请点个赞吧。如果你发现文中有错误的方面，也请留言给我，谢谢！

版权声明：本文为flashBoxer原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://www.cnblogs.com/flashBoxer/p/10074661.html