elastic / elastic/detection-rules

Review use of non-ecs-schema

Open
#1,776 6 comments 0 reactions 0 assignees View on GitHub
backlog schema v8.2.0 v8.7.0
Dominant language
Python
Stars
2.7k
Forks
696
Avg merge
4d 17h
Merged PRs (30d)
87

Description

The exceptions defined in the [non-ecs-schema](https://github.com/elastic/detection-rules/blob/main/detection_rules/etc/non-ecs-schema.json) has grown significantly as of late, mostly to accommodate winlogbeat-specific fields.

Old Version

```json
{
"endgame-*": {
"endgame": {
"metadata": {
"type": "keyword"
},
"event_subtype_full": "keyword"
}
},
"winlogbeat-*": {
"winlog": {
"event_data": {
"AccessList": "keyword",
"AllowedToDelegateTo": "keyword",
"AttributeLDAPDisplayName": "keyword",
"AttributeValue": "keyword",
"CallerProcessName": "keyword",
"CallTrace": "keyword",
"ClientProcessId": "keyword",
"GrantedAccess": "keyword",
"NewTargetUserName": "keyword",
"ObjectDN": "keyword",
"OldTargetUserName": "keyword",
"OriginalFileName": "keyword",
"ParentProcessId": "keyword",
"RelativeTargetName": "keyword",
"ShareName": "keyword",
"SubjectLogonId": "keyword",
"TargetImage": "keyword",
"TargetLogonId": "keyword",
"TargetProcessGUID": "keyword",
"TargetSid": "keyword"
}
},
"winlog.logon.type": "keyword",
"powershell.file.script_block_text": "text"
},
"filebeat-*": {
"o365.audit.NewValue": "keyword",
"o365audit.Parameters.ForwardTo": "keyword",
"o365audit.Parameters.ForwardAsAttachmentTo": "keyword",
"o365audit.Parameters.RedirectTo": "keyword"
},
"logs-endpoint.events.*": {
"process.Ext.token.integrity_level_name": "keyword",
"process.parent.Ext.real.pid": "long"
},
"logs-windows.*": {
"powershell.file.script_block_text": "text"
}
}
```

Jan 30th non-ecs-schema

```json

{
"endgame-*": {
"endgame": {
"metadata": {
"type": "keyword"
},
"event_subtype_full": "keyword"
}
},
"winlogbeat-*": {
"winlog": {
"event_data": {
"AccessList": "keyword",
"AccessMask": "keyword",
"AccessMaskDescription": "keyword",
"AllowedToDelegateTo": "keyword",
"AttributeLDAPDisplayName": "keyword",
"AttributeValue": "keyword",
"CallerProcessName": "keyword",
"CallTrace": "keyword",
"ClientProcessId": "keyword",
"GrantedAccess": "keyword",
"NewTargetUserName": "keyword",
"ObjectClass": "keyword",
"ObjectDN": "keyword",
"ObjectName": "keyword",
"OldTargetUserName": "keyword",
"OriginalFileName": "keyword",
"ParentProcessId": "keyword",
"ProcessName": "keyword",
"Properties": "keyword",
"RelativeTargetName": "keyword",
"ShareName": "keyword",
"SubjectLogonId": "keyword",
"SubjectUserName": "keyword",
"SubjectUserSid": "keyword",
"TargetUserName": "keyword",
"TargetImage": "keyword",
"TargetLogonId": "keyword",
"TargetProcessGUID": "keyword",
"TargetSid": "keyword",
"SchemaFriendlyName": "keyword",
"Resource": "keyword",
"PrivilegeList": "keyword",
"AuthenticationPackageName" : "keyword",
"TargetUserSid" : "keyword",
"LogonProcessName": "keyword",
"DnsHostName" : "keyword",
"ServiceFileName": "keyword",
"ImagePath": "keyword",
"TaskName": "keyword",
"Status": "keyword",
"EnabledPrivilegeList": "keyword",
"OperationType": "keyword"
}
},
"winlog.logon.type": "keyword",
"winlog.logon.id": "keyword",
"powershell.file.script_block_text": "text"
},
"filebeat-*": {
"o365.audit.NewValue": "keyword"
},
"logs-endpoint.events.*": {
"process.Ext.token.integrity_level_name": "keyword",
"process.parent.Ext.real.pid": "long",
"process.Ext.effective_parent.executable": "keyword",
"process.Ext.effective_parent.name": "keyword",
"file.Ext.header_bytes": "keyword",
"file.Ext.entropy": "long",
"file.size": "long",
"file.Ext.original.name": "keyword",
"dll.Ext.relative_file_creation_time": "double",
"dll.Ext.relative_file_name_modify_time": "double",
"process.Ext.relative_file_name_modify_time": "double",
"process.Ext.relative_file_creation_time": "double"
},
"logs-windows.*": {
"powershell.file.script_block_text": "text"
},
"logs-kubernetes.*": {
"kubernetes.audit.objectRef.resource": "keyword",
"kubernetes.audit.objectRef.subresource": "keyword",
"kubernetes.audit.verb": "keyword",
"kubernetes.audit.user.username": "keyword",
"kubernetes.audit.impersonatedUser.username": "keyword",
"kubernetes.audit.annotations.authorization_k8s_io/decision": "keyword",
"kubernetes.audit.annotations.authorization_k8s_io/reason": "keyword",
"kubernetes.audit.user.groups": "text",
"kubernetes.audit.requestObject.spec.containers.securityContext.privileged": "boolean",
"kubernetes.audit.requestObject.spec.containers.securityContext.allowPrivilegeEscalation": "boolean",
"kubernetes.audit.requestObject.spec.securityContext.runAsUser": "long",
"kubernetes.audit.requestObject.spec.containers.securityContext.runAsUser": "long",
"kubernetes.audit.requestObject.spec.hostPID": "boolean",
"kubernetes.audit.requestObject.spec.hostNetwork": "boolean",
"kubernetes.audit.requestObject.spec.hostIPC": "boolean",
"kubernetes.audit.requestObject.spec.volumes.hostPath.path": "keyword",
"kubernetes.audit.requestObject.spec.type": "keyword",
"kubernetes.audit.requestObject.rules.resources": "keyword",
"kubernetes.audit.requestObject.rules.verb": "keyword",
"kubernetes.audit.objectRef.namespace": "keyword",
"kubernetes.audit.objectRef.serviceAccountName": "keyword",
"kubernetes.audit.requestObject.spec.serviceAccountName": "keyword",
"kubernetes.audit.responseStatus.reason": "keyword",
"kubernetes.audit.requestObject.spec.containers.securityContext.capabilities.add": "keyword",
"kubernetes.audit.requestObject.spec.containers.image": "text"
},
".alerts-security.*": {
"signal.rule.name": "keyword",
"kibana.alert.rule.threat.tactic.id": "keyword"
}
}

```

We need to review this as well as the rules using it for:
- [ ] filebeat fields: rules can define the module/dataset and not need to define them here
- [ ] integration fields: we do not parse integration specific schemas at the moment, so these may be able to defer to filebeat fields with dataset. `"powershell.file.script_block_text": "text"` may be definable in winlogbeat.
- [ ] since there are so many specific to `winlog.event_data`, we should look into auto parsing it based on the existence of the `winlogbeat-*` index pattern, similar to how modules and datasets are parsed for filebeat rules

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.