后端线上服务监控与报警方案

[22-Oct-2015 18:39:04 Asia/Shanghai] PHP Fatal error:  Call to a member function prepare() on null in /home/work/phplib/db/Database.class.php on line 238

/**
 * 统一截获并处理错误
 * @return bool
 */public static function registErrorHandler() {
    $e_types = array (
        E_ERROR   => 'PHP Fatal',
        E_WARNING => 'PHP Warning',
        E_PARSE   => 'PHP Parse Error',
        E_NOTICE  => 'PHP Notice'
    );
    register_shutdown_function(function () use ($e_types) {
        $error = error_get_last();        if ($error['type'] != E_NOTICE && !empty($error['message'])) {            $error['trace'] = self::getStackTrace();            self::error_handler($error);
        }
    });
    set_error_handler(function ($type, $message, $file, $line) use ($e_types) {
        if ($type != E_NOTICE && !empty($message)) {            $error = array (                'type'    => $type,                'message' => $message,                'file'    => $file,                'line'    => $line,                'trace'   => self::getStackTrace()
            );            self::error_handler($error);            // 被截获的错误，重新输出到错误文件
            error_log(($e_types[$type] ?: 'Unknown Problem') . ' :  ' . $message . ' in ' . $file . ' on line ' . $line . "\n");
        }
    }, E_ALL);
}

// 全局异常捕获MonitorManager::registErrorHandler();

/**
 * 添加监控日志，日志会被异步收集到日志平台进行展示
 *
 * @param $type
 * @param $content
 */public static function collect($type, $content) {
    // 线上集群，并且开关处于打开状态，才进行收集
    if (Utilities::isOnlineCluster() && SwitchManager::getSwitch('collect', SwitchManager::SWITCH_MONITOR)) {        // 检查当前这种监控类型是否支持
        if (self::support($type) && !self::checkWhiteList($type, $content)) {            self::queueInstance()->enQueue(json_encode(array (                'type'    => $type,                'data'    => $content,                'cluster' => Utilities::getClusterName(),                'reqtime' => isset($_SERVER['REQUEST_TIME']) ? $_SERVER['REQUEST_TIME'] : time(),                'extinfo' => array (                    'domain'    => $_SERVER['SERVER_NAME'],                    'path'      => isset($_SERVER['QUERY_STRING']) ? str_replace('?' . $_SERVER['QUERY_STRING'], '', $_SERVER['REQUEST_URI']) : 'script',                    'userAgent' => isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '',                    'referer'   => isset($_SERVER['HTTP_REFERER']) ? $_SERVER['HTTP_REFERER'] : '',                    'serverIp'  => Utilities::getServerPhpIP(),                    'reqData'   => json_encode($_REQUEST, true)
                )
            ), true));
        }
    }
}

<?phpnamespace Mlservice\Script\Monitor;use Framework\Libs\Monitor\MonitorManager;use Framework\Libs\Util\Utilities;/**
 * 从MC队列中，将各种错误日志上报到日志平台进行汇总监控、报警等
 * Class Collect
 * @package Mlservice\Script\Monitor
 * @author  xianliezhao
 */class Collect extends \Framework\FrameworkScript {
    private $limit = 5;    private $interval = 600;    public function run() {
        // 检查脚本可执行
        $this->checkCluster();        $start = time();        while (true) {            $index = 0;            $params = array ();            while (true) {                $index++;                $item = MonitorManager::queueInstance()->deQueue();                if (!empty($item)) {                    $params[] = $item;
                } else {                    // 如果数据为空，则10分钟清理一次队列，做一次初始化，且自杀进程
                    if (time() - $start > $this->interval) {
                        MonitorManager::queueInstance()->makeEmpty();                        exit(0);
                    }                    break;
                }                if ($index >= $this->limit) {                    break;
                }
            }            // 发送到服务器，统一收集
            if (!empty($params)) {
                Utilities::apiRequest('bizfe', 'feapi/monitor/mon/collect', $params);
            } else {
                sleep(1);
            }
        }
    }
}

/**
 * 错误日志采集的表结构
 * @type {*|Model}
 */var monModel = connection.model('monitor', new Schema({
    type: String,
    cluster: String,
    product: String,
    data: Object,
    extinfo: Object,
    reqtime: Number
}, {
    autoIndex: true}));

/**
 * 对数据进行加工，然后保存日志到db，批量保存成功以后再进行报警检测
 * @param messageModel
 */function save(messageModel) {
    return function (reqParams, callback) {
        var params = [];        for (var i in reqParams) {            var item = JSON.parse(reqParams[i]);            // 忽略来自标准环境的任何错误
            if (item.extinfo.domain.indexOf('rdlab') > 0) {                continue;
            }            // 从域名中记录下出问题的模块名称
            item.product = item.extinfo.domain.split('\.')[0];
            item = cleanParams(messageModel, item);
            params.push(item);
        }        var count = params.length;        if (count == 0) {
            callback();
        }        var saveData = [];        var done = 0;        // 批量保存
        params.forEach(function (item) {
            new messageModel(item).save(function (err, product, effectRows) {
                !err && (item._id = product._id);
                saveData.push(item);                if (++done == count) {                    // 当所有的错误日志都进入db成功以后，开始进行报警检测（内存中会维护一个错误池）
                    alarm.addAndCheckPool(saveData);                    if (err) {
                        callback(err, null);
                    } else {
                        callback(null, {error_code: 0, data: {}});
                    }
                }
            });
        });
    };
}

/**
 * 每秒检测一次，各个错误类型只要达到邮件或者短信的最大阈值，则进行报警
 */var doAlarm = function () {
    var INTERVAL_TIME = 1000;    // 启用监控
    if (!alarmListenIng) {
        alarmListenIng = true;
    } else {        return false;
    }

    setInterval(function () {
        // 遍历所有类型，判断是否进行报警
        Object.keys(cachePool).forEach(function (type) {

            // 当前时间
            var nowTime = (new Date()).getTime();            // 控制每分钟最多只能报警N次
            var alarmCount = cachePool[type]['alarmCount'];            if (alarmCount === undefined) {
                initCacheByType(type, 1);
            } else {                var lastMtime = cachePool[type]['lastMtime'];                if ((nowTime - lastMtime) / 1000 > 60) {                    // 超过1分钟，直接进行数据重新初始化
                    initCacheByType(type, 1);
                } else if (alarmCount >= cachePool[type]['alarmCpm']) {                    // 如果每分钟的报警次数超过阈值，就不报警了
                    return false;
                }
            }            // 控制每N秒内报警一次
            var lastAlarmTime = cachePool[type]['lastAlarmTime'];            if (lastAlarmTime === undefined) {
                cachePool[type]['lastAlarmTime'] = nowTime;
            } else if (Math.ceil((nowTime - lastAlarmTime) / 1000) >= cachePool[type]['timeInterval']) {
                cachePool[type]['lastAlarmTime'] = nowTime;                // 这种情况下，才表明需要报警
                if (cachePool[type]['alarms'] && cachePool[type]['alarms']['total'] >= cachePool[type]['maxNumForMail']) {
                    cachePool[type]['type'] = type;
                    cachePool[type]['alarmCount'] += 1;                    var theAlarmData = cachePool[type];
                    theAlarmData.theTime = Math.ceil((nowTime - lastAlarmTime) / 1000) || 1;                    // 邮件报警
                    sendEmail(buildEmailAlarmContent(theAlarmData));                    // 如果是出错量比设定的短信阈值还大，则短信报警
                    if (cachePool[type]['maxNumForSms'] <= cachePool[type]['alarms']['total']) {
                        sendSmsMessage(buildSmsAlarmContent(theAlarmData));
                    }                    // 清空
                    initCacheByType(type, 0);
                }
            }
        });
    }, INTERVAL_TIME);
};

    /**
     * 报警阈值设定
     */
    alarmLimits: {
        db_log: {
            timeInterval: 3, // 每隔3s监控一次
            maxNumForMail: 10, // 邮件报警阈值
            maxNumForSms: 50, // 短信报警阈值
            alarmCpm: 5 // 表示每分钟最多报警5次
        },
        redis_log: {
            timeInterval: 3,
            maxNumForMail: 10,
            maxNumForSms: 50,
            alarmCpm: 5
        },
        mc_log: {
            timeInterval: 3,
            maxNumForMail: 5,
            maxNumForSms: 20,
            alarmCpm: 5
        },
        mq_error: {
            timeInterval: 3,
            maxNumForMail: 10,
            maxNumForSms: 30,
            alarmCpm: 5
        },
        php_error: {
            timeInterval: 3,
            maxNumForMail: 5,
            maxNumForSms: 20,
            alarmCpm: 5
        },
        php_warning: {
            timeInterval: 3,
            maxNumForMail: 10,
            maxNumForSms: 50,
            alarmCpm: 5
        }
    }

[x.x.x.x] [-] [23/Oct/2015:14:59:59 +0800] [GET /goods/goods_info?goods_id=276096349&fields=platform_type%2Cgoods_id%2Cgoods_param%2Cgoods_detail%2Cshop_id%2Cfirst_sort_id%2Csize%2Cgoods_desc HTTP/1.1] [200] [1957] [xxx.com/share/goods_details] [Snake Connect] [-] [0.010] [y.y.y.y:9999] [0.010] [-] [uid:0;ip:0.0.0.0;v:0;master:0;is_mob:0]

[$remote_addr] [$remote_user] [$time_local] [$request] [$status] [$body_bytes_sent] [$http_referer] [$http_user_agent] [$http_user_agent] [$http_x_forwarded_for] [$request_time] [$upstream_addr] [$upstream_response_time] [$request_body]

# 日期date_time=$(date -d "-1 hours" "+%Y%m%d%H")# 文件位置file_name="/home/service/nginx/log/xxx.mlservice.access.${date_time}.log"# 获取监控列表api_list=$(awk '{print $6}' ${file_name} | grep -v 'your_filter_api_here' | sed -e "s/\?.*//" -e "s/^\///" | tr '[A-Z]' '[a-z]' | sort | uniq -c | sort -nr | head -n 20 | awk '{print $2}') ;

goods/goods_info
goods/campaign_info
inventory/get_skuinfo
inventory/set_inventory
campaign/update_campaign
campaign/goods_info
inventory/spu_set
inventory/inventory_decr
...