需求
GrayLog每天每周产生的安全告警数量较多,这些产生的告警都通过PrometheusAlert推送到钉钉webhook告警机器人
想对这些告警做一些汇总统计,报表分析
(图片点击放大查看)
解决思路:
由于GrayLog Web界面上的告警界面无汇总统计的功能,只能寻找其他解决思路,查询prometheusalert是否有统计功能 但搜索过PrometheusAlert的相关功能,与需求不太吻合。
突发奇想,看能否从PrometheusAlert.log的日志进行下手
通过查看PrometheusAlert.log的一些规律,最终确定的解决方法
包含value.go的这一行就是所需要的告警日志
2022/10/14 14:15:15.735 [D] [value.go:476] [1665728115735353131] {"event_deFinition_id":"62d40b5bfbbe0a2fd4faf65d","event_deFinition_type":"aggregation-v1","event_deFinition_title":"堡垒机绕过提醒","event_deFinition_description":"安全助手检测到绕过堡垒机直接登陆服务器行为","job_deFinition_id":"62d3ff2bfbbe0a2fd4fadd35","job_trigger_id":"6348fe7213d8e16f7fe61732","event":{"id":"01GFAJ3W0SZSGJNTK01QKMRQXB","event_deFinition_type":"aggregation-v1","event_deFinition_id":"62d40b5bfbbe0a2fd4faf65d","origin_context":"urn:graylog:message:es:linuxserver_23:e6490781-4b86-11ed-aea0-005056b6acae","timestamp":"2022-10-14T06:10:46.428Z","timestamp_processing":"2022-10-14T06:15:14.713Z","timerange_start":null,"timerange_end":null,"streams":[],"source_streams":["62d3eed0fbbe0a2fd4facacd"],"message":"堡垒机绕过提醒","source":"localhost","key_tuple":[],"key":"","priority":2,"alert":true,"fields":{},"group_by_fields":{}},"backlog":[{"index":"linuxserver_23","message":"Accepted password for root from 192.168.29.41 port 60382 ssh2","timestamp":"2022-10-14T06:10:46.428Z","fields":{"process_id":"3194","gl2_accounted_message_size":292,"application_name":"sshd","level":6,"gl2_remote_ip":"172.16.252.134","gl2_remote_port":64781,"facility_num":10,"Linux_server_ssh_login_ip":"192.168.29.41","gl2_message_id":"01GFAHV3ZRYPS0G895J23348A6","gl2_source_node":"d20e3549-da0a-4ae9-af4a-d352e1c3deb5","gl2_source_input":"62d3ec98fbbe0a2fd4fac816","facility":"security/authorization"},"id":"e6490781-4b86-11ed-aea0-005056b6acae","source":"ec-server-test-172-16-252-134","stream_ids":["62d3eed0fbbe0a2fd4facacd"]}]}
1、使用rsyslog服务来读取prometheusalert.log日志文件
[root@centos ~]# cd /etc/rsyslog.d/
[root@centos rsyslog.d]# vi prometheusalert_read.conf
[root@centos rsyslog.d]# cat prometheusalert_read.conf
module(load="imfile" PollingInterval="1")
# Input for FILE1
#wildcard is allowed at file level only
input(
type="imfile"
tag="Alertlog"
ruleset="filelog"
Facility="local0"
Severity="info"
PersistStateInterval="1"
reopenOnTruncate="on"
freshStartTail="on"
file="/opt/PrometheusAlert/logs/prometheusalertcenter.log"
)
# Define a template for file events
template(name="GraylogFormatFilelog" type="string" string="%msg%\n")
#Replace the Target and Port values with your GrayLogServer IP address and port.
ruleset(name="filelog") {
action(
type="omfwd"
protocol="udp"
target="192.168.31.170"
port="1524"
template="GraylogFormatFilelog"
queue.type="LinkedList"
queue.filename="fileq1"
queue.saveonshutdown="on"
action.resumeRetryCount="-1"
) stop
}
[root@centos rsyslog.d]# systemctl restart rsyslog
(图片点击放大查看)
2、创建Index,Input和Stream
[root@centos ~]# firewall-cmd --permanent --zone=public --add-port=1524/udp
success
[root@centos ~]# firewall-cmd --reload
success
[root@centos ~]# systemctl restart rsyslog.service
[root@centos ~]#
(图片点击放大查看)
(图片点击放大查看)
(图片点击放大查看)
(图片点击放大查看)
(图片点击放大查看)
3、字段提取
(图片点击放大查看)
例如这里使用正则表达式进行字段的提取
(图片点击放大查看)
(图片点击放大查看)
{
"extractors": [
{
"title": "alert_json_message",
"extractor_type": "regex",
"converters": [],
"order": 1,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "alert_json_message",
"extractor_config": {
"regex_value": "^.*\\[[0-9]{19}\\](.+)$"
},
"condition_type": "string",
"condition_value": "value.go"
},
{
"title": "json_extractor",
"extractor_type": "json",
"converters": [],
"order": 2,
"cursor_strategy": "copy",
"source_field": "alert_json_message",
"target_field": "",
"extractor_config": {
"list_separator": ", ",
"kv_separator": "=",
"key_prefix": "",
"key_separator": "_",
"replace_key_whitespace": false,
"key_whitespace_replacement": "_"
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "field_cut",
"extractor_type": "copy_input",
"converters": [
{
"type": "tokenizer",
"config": {}
}
],
"order": 3,
"cursor_strategy": "copy",
"source_field": "backlog",
"target_field": "backlog_detail",
"extractor_config": {},
"condition_type": "none",
"condition_value": ""
},
{
"title": "level_replace",
"extractor_type": "regex_replace",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"replacement": "\"alertlevel\"",
"regex": "\"level\""
},
"condition_type": "none",
"condition_value": ""
},
{
"title": "facility_num_replace",
"extractor_type": "regex_replace",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"replacement": "\"alert_facility_num\"",
"regex": "\"facility_num\""
},
"condition_type": "none",
"condition_value": ""
}
],
"version": "4.2.10"
}
4、字段展示和报表大屏配置
(图片点击放大查看)
最后的效果
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。