mirror of
https://github.com/Velocidex/velociraptor.git
synced 2026-10-12 02:09:59 +00:00
Feature/carve evtx, (#5063)
Problem: When Windows event logs are cleared or deleted, Windows.EventLogs.Cleared can show that a log was cleared, but the events themselves are lost because parse_evtx() only reads intact log files. The old data usually stays on disk, in unallocated space, the pagefile or memory, until it is overwritten. Solution: This PR adds a new carve_evtx() VQL plugin. It scans any file or device for EVTX chunk signatures at any offset and parses each 64KB chunk on its own. - Rows have the same structure as parse_evtx(), plus ChunkOffset, ChunkHeaderValid, ChunkDataValid, ChunkTruncated and ChunkCompressed. - NTFS-compressed chunks (the Windows default for event logs) are decompressed. - Damaged chunks are flagged, not dropped. - With output=, the recovered events are also written to a new .evtx file that Event Viewer can open. Example: SELECT * FROM carve_evtx(filename='''\\.\C:''', accessor="raw_file") WHERE System.Channel = "Application" Known limitation Old records in chunk slack are not recovered yet. This could be a follow-up. --------- Co-authored-by: Mike Cohen <[email protected]>
This commit is contained in:
co-authored by
Mike Cohen
parent
4b35b1120f
commit
fa9a5b4cd1
@@ -2,6 +2,7 @@ package file_store_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/suite"
|
||||
@@ -144,7 +145,8 @@ func (self *FSAccessorTest) TestSparseFiles() {
|
||||
|
||||
buf := make([]byte, 100)
|
||||
n, err := fd.Read(buf)
|
||||
assert.NoError(self.T(), err)
|
||||
assert.Error(self.T(), err)
|
||||
assert.True(self.T(), errors.Is(err, io.EOF))
|
||||
|
||||
assert.Equal(self.T(), string(buf[:n]), "Hello\x00\x00\x00\x00\x00World")
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
//go:build !velomain
|
||||
// +build !velomain
|
||||
|
||||
package assets
|
||||
|
||||
import "www.velocidex.com/golang/velociraptor/utils"
|
||||
|
||||
var (
|
||||
FileDocsReferencesVqlYaml = []byte("")
|
||||
Inventory = map[string][]byte{}
|
||||
)
|
||||
|
||||
// ReadFile is adapTed from ioutil
|
||||
func ReadFile(path string) ([]byte, error) {
|
||||
return nil, utils.NotImplementedError
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
name: Windows.Carving.Evtx
|
||||
description: |
|
||||
Carve EVTX chunks from a file or device and parse their event records.
|
||||
|
||||
When event logs are cleared or deleted, the old data often remains
|
||||
on disk until it is overwritten. This plugin scans any file or
|
||||
device (for example a raw volume, unallocated space, the pagefile
|
||||
or a memory image) for the EVTX chunk signature at any offset and
|
||||
parses each chunk it finds. Each 64kb chunk carries its own string
|
||||
and template tables, so its records can be parsed without the rest
|
||||
of the log file.
|
||||
|
||||
required_permissions:
|
||||
- FILESYSTEM_WRITE
|
||||
|
||||
parameters:
|
||||
- name: Device
|
||||
default: "C:"
|
||||
description: The NTFS drive to carve
|
||||
- name: Accessor
|
||||
description: The accessor to use.
|
||||
default: "ntfs"
|
||||
- name: IocRegex
|
||||
type: regex
|
||||
description: "IOC regex - will match anywhere in the event"
|
||||
default:
|
||||
- name: ChannelRegex
|
||||
description: "Channel regex to enable filtering on path"
|
||||
default: .
|
||||
- name: ProviderRegex
|
||||
description: "Provider regex to enable filtering on provider"
|
||||
default: .
|
||||
type: regex
|
||||
- name: IdRegex
|
||||
default: .
|
||||
type: regex
|
||||
- name: DateAfter
|
||||
type: timestamp
|
||||
description: "search for events after this date. YYYY-MM-DDTmm:hh:ssZ"
|
||||
- name: DateBefore
|
||||
type: timestamp
|
||||
description: "search for events before this date. YYYY-MM-DDTmm:hh:ssZ"
|
||||
- name: ReconstructFile
|
||||
type: bool
|
||||
description: Try to reconstruct an evtx file from the carved chunks.
|
||||
|
||||
imports:
|
||||
- Windows.Sys.AllUsers
|
||||
|
||||
precondition:
|
||||
SELECT OS From info() where OS = 'windows'
|
||||
|
||||
sources:
|
||||
- query: |
|
||||
LET DateAfterTime <= DateAfter || "1600-01-01"
|
||||
LET DateBeforeTime <= DateBefore || "2200-01-01"
|
||||
LET S = scope()
|
||||
|
||||
LET Temp <= tempfile(remove_last=TRUE)
|
||||
|
||||
SELECT timestamp(epoch=int(int=System.TimeCreated.SystemTime)) AS EventTime,
|
||||
System.Computer as Computer,
|
||||
System.Channel as Channel,
|
||||
System.Provider.Name as Provider,
|
||||
System.EventID.Value as EventID,
|
||||
System.EventRecordID as EventRecordID,
|
||||
System.Security.UserID as UserSID,
|
||||
LookupSIDCache(SID=System.Security.UserID || "") AS Username,
|
||||
S.EventData AS EventData,
|
||||
S.UserData AS UserData,
|
||||
S.Message AS Message,
|
||||
ChunkInfo
|
||||
FROM carve_evtx(filename=Device,
|
||||
accessor=Accessor,
|
||||
output=if(condition=ReconstructFile, then=Temp)
|
||||
)
|
||||
WHERE ( EventData OR UserData OR Message )
|
||||
AND EventTime < DateBeforeTime
|
||||
AND EventTime > DateAfterTime
|
||||
AND Channel =~ ChannelRegex
|
||||
AND Provider =~ ProviderRegex
|
||||
AND EventID =~ IdRegex
|
||||
AND format(format='%v %v %v', args=[EventData, UserData, Message]) =~ IocRegex
|
||||
|
||||
- name: UploadRecostruction
|
||||
query: |
|
||||
SELECT upload(file=Temp) AS Upload
|
||||
FROM scope()
|
||||
WHERE ReconstructFile
|
||||
|
||||
column_types:
|
||||
- name: Upload
|
||||
type: upload_preview
|
||||
@@ -0,0 +1,40 @@
|
||||
Queries:
|
||||
# Build a fake disk image in memory: unrelated data, then an EVTX
|
||||
# chunk at an unaligned offset, then more unrelated data. The chunk
|
||||
# records the System log being cleared (Event ID 104).
|
||||
- LET Chunk <= read_file(
|
||||
filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx',
|
||||
offset=4096, length=65536)
|
||||
- LET Noise <= read_file(
|
||||
filename=srcDir + '/artifacts/testdata/files/SAM', length=777)
|
||||
- LET Image <= Noise + Chunk + Noise
|
||||
|
||||
# The chunk is found at offset 777 with valid checksums.
|
||||
- SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.kDataValid, ChunkInfo.Truncated,
|
||||
System.EventID.Value AS EventID, System.EventRecordID AS RecordID,
|
||||
System.Channel AS Channel
|
||||
FROM carve_evtx(filename=Image, accessor="data")
|
||||
|
||||
# Carving the original log file finds the chunk after the file header.
|
||||
- SELECT ChunkInfo.Offset, System.EventID.Value AS EventID
|
||||
FROM carve_evtx(
|
||||
filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx')
|
||||
|
||||
# Starting after the chunk finds nothing.
|
||||
- SELECT * FROM carve_evtx(filename=Image, accessor="data", start_offset=778)
|
||||
|
||||
# A damaged chunk header is still parsed but flagged as invalid.
|
||||
- LET Damaged <= Noise + Chunk[:256] + "XXXX" + Chunk[260:]
|
||||
- SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.DataValid,
|
||||
System.EventID.Value AS EventID
|
||||
FROM carve_evtx(filename=Damaged, accessor="data")
|
||||
|
||||
# Recovered chunks can be written to a new EVTX file. The duplicate
|
||||
# chunk is written once and the damaged chunk is repaired so the
|
||||
# file can be read by parse_evtx() and Event Viewer.
|
||||
- LET Out <= tempfile(extension='.evtx')
|
||||
- SELECT count() AS Rows
|
||||
FROM carve_evtx(filename=Image + Image + Damaged, accessor="data", output=Out)
|
||||
GROUP BY 1
|
||||
- SELECT System.EventID.Value AS EventID, System.EventRecordID AS RecordID
|
||||
FROM parse_evtx(filename=Out)
|
||||
@@ -0,0 +1,78 @@
|
||||
# Build a fake disk image in memory: unrelated data, then an EVTX
|
||||
# chunk at an unaligned offset, then more unrelated data. The chunk
|
||||
# records the System log being cleared (Event ID 104).
|
||||
Query: LET Chunk <= read_file( filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx', offset=4096, length=65536)
|
||||
Output: []
|
||||
|
||||
Query: LET Noise <= read_file( filename=srcDir + '/artifacts/testdata/files/SAM', length=777)
|
||||
Output: []
|
||||
|
||||
Query: LET Image <= Noise + Chunk + Noise
|
||||
Output: []
|
||||
|
||||
# The chunk is found at offset 777 with valid checksums.
|
||||
Query: SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.kDataValid, ChunkInfo.Truncated, System.EventID.Value AS EventID, System.EventRecordID AS RecordID, System.Channel AS Channel FROM carve_evtx(filename=Image, accessor="data")
|
||||
Output: [
|
||||
{
|
||||
"ChunkInfo.Offset": 777,
|
||||
"ChunkInfo.HeaderValid": true,
|
||||
"ChunkInfo.kDataValid": null,
|
||||
"ChunkInfo.Truncated": false,
|
||||
"EventID": 104,
|
||||
"RecordID": 27736,
|
||||
"Channel": "System"
|
||||
}
|
||||
]
|
||||
|
||||
# Carving the original log file finds the chunk after the file header.
|
||||
Query: SELECT ChunkInfo.Offset, System.EventID.Value AS EventID FROM carve_evtx( filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx')
|
||||
Output: [
|
||||
{
|
||||
"ChunkInfo.Offset": 4096,
|
||||
"EventID": 104
|
||||
}
|
||||
]
|
||||
|
||||
# Starting after the chunk finds nothing.
|
||||
Query: SELECT * FROM carve_evtx(filename=Image, accessor="data", start_offset=778)
|
||||
Output: []
|
||||
|
||||
# A damaged chunk header is still parsed but flagged as invalid.
|
||||
Query: LET Damaged <= Noise + Chunk[:256] + "XXXX" + Chunk[260:]
|
||||
Output: []
|
||||
|
||||
Query: SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.DataValid, System.EventID.Value AS EventID FROM carve_evtx(filename=Damaged, accessor="data")
|
||||
Output: [
|
||||
{
|
||||
"ChunkInfo.Offset": 777,
|
||||
"ChunkInfo.HeaderValid": false,
|
||||
"ChunkInfo.DataValid": true,
|
||||
"EventID": 104
|
||||
}
|
||||
]
|
||||
|
||||
# Recovered chunks can be written to a new EVTX file. The duplicate
|
||||
# chunk is written once and the damaged chunk is repaired so the
|
||||
# file can be read by parse_evtx() and Event Viewer.
|
||||
Query: LET Out <= tempfile(extension='.evtx')
|
||||
Output: []
|
||||
|
||||
Query: SELECT count() AS Rows FROM carve_evtx(filename=Image + Image + Damaged, accessor="data", output=Out) GROUP BY 1
|
||||
Output: [
|
||||
{
|
||||
"Rows": 3
|
||||
}
|
||||
]
|
||||
|
||||
Query: SELECT System.EventID.Value AS EventID, System.EventRecordID AS RecordID FROM parse_evtx(filename=Out)
|
||||
Output: [
|
||||
{
|
||||
"EventID": 104,
|
||||
"RecordID": 27736
|
||||
},
|
||||
{
|
||||
"EventID": 104,
|
||||
"RecordID": 27736
|
||||
}
|
||||
]
|
||||
|
||||
+414
-9
@@ -6,7 +6,34 @@ Output: [
|
||||
"Artifact": "name: BrokenYaml\nsomeweirdfield: 1\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) yaml_error: YAML Error: yaml: unmarshal errors:\n line 2: field someweirdfield not found in type proto.Artifact"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "yaml_error",
|
||||
"Message": "YAML Error: %[1]v",
|
||||
"Args": [
|
||||
{
|
||||
"Err": {
|
||||
"Errors": [
|
||||
"line 2: field someweirdfield not found in type proto.Artifact"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {},
|
||||
@@ -16,7 +43,51 @@ Output: [
|
||||
"Artifact": "name: BrokenQuery\nsources:\n- query: SELECT * FROM nosuchplugin()\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) artifact_vql_error: BrokenQuery: query: (1,15) unknown_plugin: Unknown plugin nosuchplugin()"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"BrokenQuery",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "unknown_plugin",
|
||||
"Message": "Unknown %[2]v %[1]v()",
|
||||
"Args": [
|
||||
"nosuchplugin",
|
||||
"plugin"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 14,
|
||||
"Line": 1,
|
||||
"Column": 15
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 28,
|
||||
"Line": 1,
|
||||
"Column": 29
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {},
|
||||
@@ -26,7 +97,31 @@ Output: [
|
||||
"Artifact": "name: BrokenSubArtifact\nsources:\n- query: SELECT * FROM Artifact.No.Such.Artifact()\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) artifact_vql_error: BrokenSubArtifact: query: Unknown artifact reference No.Such.Artifact"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"BrokenSubArtifact",
|
||||
{
|
||||
"Err": {}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {},
|
||||
@@ -71,11 +166,300 @@ Output: [
|
||||
"FILESYSTEM_READ"
|
||||
],
|
||||
"Errors": [
|
||||
"(0,0) artifact_vql_error: KWArgs: query: (5,32) required_arg_missing: While calling function alert(), required arg name is not provided",
|
||||
"(0,0) artifact_vql_error: KWArgs: query: (10,15) kwargs_mixed_call: Calling plugin `switch` with kwargs (**) as well as with arg `{a {10:22 14:2}}`",
|
||||
"(0,0) artifact_vql_error: KWArgs: query: (11,10) kwargs_mixed_call: Calling function `Foo` with kwargs (**) as well as with arg `{X {11:30 11:33}}`",
|
||||
"(0,0) artifact_vql_error: KWArgs: query: (12,10) kwargs_mixed_call: Calling function `dict` with kwargs (**) as well as with arg `{A {12:15 12:18}}`",
|
||||
"(0,0) artifact_vql_error: KWArgs: query: (13,8) kwargs_mixed_call: Calling plugin `Artifact.Generic.Client.Info` with kwargs (**) as well as with arg `{X {13:53 13:56}}`"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"KWArgs",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "required_arg_missing",
|
||||
"Message": "While calling %[2]v %[3]v(), required arg %[1]v is not provided",
|
||||
"Args": [
|
||||
"name",
|
||||
"function",
|
||||
"alert"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 147,
|
||||
"Line": 5,
|
||||
"Column": 32
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 171,
|
||||
"Line": 6,
|
||||
"Column": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"KWArgs",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "kwargs_mixed_call",
|
||||
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
|
||||
"Args": [
|
||||
"switch",
|
||||
{
|
||||
"Name": "a",
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 258,
|
||||
"Line": 10,
|
||||
"Column": 22
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 387,
|
||||
"Line": 14,
|
||||
"Column": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"plugin"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 251,
|
||||
"Line": 10,
|
||||
"Column": 15
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 402,
|
||||
"Line": 15,
|
||||
"Column": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"KWArgs",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "kwargs_mixed_call",
|
||||
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
|
||||
"Args": [
|
||||
"Foo",
|
||||
{
|
||||
"Name": "X",
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 291,
|
||||
"Line": 11,
|
||||
"Column": 30
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 294,
|
||||
"Line": 11,
|
||||
"Column": 33
|
||||
}
|
||||
}
|
||||
},
|
||||
"function"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 271,
|
||||
"Line": 11,
|
||||
"Column": 10
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 295,
|
||||
"Line": 11,
|
||||
"Column": 34
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"KWArgs",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "kwargs_mixed_call",
|
||||
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
|
||||
"Args": [
|
||||
"dict",
|
||||
{
|
||||
"Name": "A",
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 311,
|
||||
"Line": 12,
|
||||
"Column": 15
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 314,
|
||||
"Line": 12,
|
||||
"Column": 18
|
||||
}
|
||||
}
|
||||
},
|
||||
"function"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 306,
|
||||
"Line": 12,
|
||||
"Column": 10
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 331,
|
||||
"Line": 13,
|
||||
"Column": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"KWArgs",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "kwargs_mixed_call",
|
||||
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
|
||||
"Args": [
|
||||
"Artifact.Generic.Client.Info",
|
||||
{
|
||||
"Name": "X",
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 381,
|
||||
"Line": 13,
|
||||
"Column": 53
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 384,
|
||||
"Line": 13,
|
||||
"Column": 56
|
||||
}
|
||||
}
|
||||
},
|
||||
"plugin"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 336,
|
||||
"Line": 13,
|
||||
"Column": 8
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 386,
|
||||
"Line": 14,
|
||||
"Column": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": [
|
||||
{
|
||||
@@ -299,7 +683,28 @@ Output: [
|
||||
"Artifact": "name: Generic.Client.Info\nsources:\n- query: SELECT * FROM scope()\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) yaml_error: YAML Error: Unable to override built in artifact Generic.Client.Info"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "yaml_error",
|
||||
"Message": "YAML Error: %[1]v",
|
||||
"Args": [
|
||||
{}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {},
|
||||
|
||||
+135
-3
@@ -56,7 +56,51 @@ Output: [
|
||||
"Artifact": "name: NotCallingRequiredArg\nsources:\n- query: |\n LET Foo(X, Y=1) = X+Y\n SELECT Foo() FROM scope()\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) artifact_vql_error: NotCallingRequiredArg: query: (1,9) required_arg_missing: While calling VQL definition Foo(), required arg X is not provided"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"NotCallingRequiredArg",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "required_arg_missing",
|
||||
"Message": "While calling VQL definition %[2]v(), required arg %[1]v is not provided",
|
||||
"Args": [
|
||||
"X",
|
||||
"Foo"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 8,
|
||||
"Line": 1,
|
||||
"Column": 9
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 9,
|
||||
"Line": 1,
|
||||
"Column": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {
|
||||
@@ -78,8 +122,96 @@ Output: [
|
||||
"Artifact": "name: CallingBadArg\nsources:\n- query: |\n LET Foo(X, Y=1) = X+Y\n SELECT Foo(Z=1) FROM scope()\n",
|
||||
"Permissions": null,
|
||||
"Errors": [
|
||||
"(0,0) artifact_vql_error: CallingBadArg: query: (2,12) invalid_arg: Invalid arg Z for VQL definition Foo",
|
||||
"(0,0) artifact_vql_error: CallingBadArg: query: (1,9) required_arg_missing: While calling VQL definition Foo(), required arg X is not provided"
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"CallingBadArg",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "invalid_arg",
|
||||
"Message": "Invalid arg %[1]v for VQL definition %[2]v",
|
||||
"Args": [
|
||||
"Z",
|
||||
"Foo"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 33,
|
||||
"Line": 2,
|
||||
"Column": 12
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 36,
|
||||
"Line": 2,
|
||||
"Column": 15
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "artifact_vql_error",
|
||||
"Message": "%[1]v: query: %[2]v",
|
||||
"Args": [
|
||||
"CallingBadArg",
|
||||
{
|
||||
"Target": "",
|
||||
"Name": "required_arg_missing",
|
||||
"Message": "While calling VQL definition %[2]v(), required arg %[1]v is not provided",
|
||||
"Args": [
|
||||
"X",
|
||||
"Foo"
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 8,
|
||||
"Line": 1,
|
||||
"Column": 9
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 9,
|
||||
"Line": 1,
|
||||
"Column": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Pos": {
|
||||
"Pos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
},
|
||||
"EndPos": {
|
||||
"Filename": "",
|
||||
"Offset": 0,
|
||||
"Line": 0,
|
||||
"Column": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Warnings": null,
|
||||
"Definitions": {
|
||||
|
||||
@@ -892,6 +892,107 @@
|
||||
- linux_amd64_cgo
|
||||
- windows_386_cgo
|
||||
- windows_amd64_cgo
|
||||
- name: carve_evtx
|
||||
description: |
|
||||
Carve EVTX chunks from a file or device and parse their event records.
|
||||
|
||||
When event logs are cleared or deleted, the old data often remains
|
||||
on disk until it is overwritten. This plugin scans any file or
|
||||
device (for example a raw volume, unallocated space, the pagefile
|
||||
or a memory image) for the EVTX chunk signature at any offset and
|
||||
parses each chunk it finds. Each 64kb chunk carries its own string
|
||||
and template tables, so its records can be parsed without the rest
|
||||
of the log file.
|
||||
|
||||
Each row has the same structure as `parse_evtx()` with these
|
||||
additional columns:
|
||||
|
||||
- `ChunkOffset`: The offset of the chunk in the carved file.
|
||||
- `ChunkHeaderValid`: The chunk header checksum is correct.
|
||||
- `ChunkDataValid`: The event records checksum is correct.
|
||||
- `ChunkTruncated`: The file ended before the end of the chunk.
|
||||
- `ChunkCompressed`: The chunk was found inside an NTFS compressed
|
||||
file and was decompressed. Windows compresses the event logs by
|
||||
default, so most chunks carved from a raw volume are compressed.
|
||||
|
||||
Carved records may be incomplete or partially overwritten. Records
|
||||
from chunks with invalid checksums should be treated with caution.
|
||||
The same chunk may be found several times (for example in a live
|
||||
log file and in unallocated space).
|
||||
|
||||
### Example
|
||||
|
||||
```vql
|
||||
SELECT ChunkOffset, ChunkHeaderValid,
|
||||
System.TimeCreated.SystemTime AS Timestamp,
|
||||
System.EventID.Value AS EventID,
|
||||
System.Channel AS Channel, EventData
|
||||
FROM carve_evtx(filename="C:", accessor="ntfs")
|
||||
WHERE EventID IN (1102, 104, 4624, 7045)
|
||||
```
|
||||
|
||||
To carve the pagefile on a running system use the `ntfs` accessor:
|
||||
|
||||
```vql
|
||||
SELECT * FROM carve_evtx(filename="C:/pagefile.sys", accessor="ntfs")
|
||||
```
|
||||
|
||||
### Saving the recovered events
|
||||
|
||||
Set `output` to also write the recovered chunks to a new EVTX
|
||||
file that can be opened with Event Viewer, `parse_evtx()` or any
|
||||
other EVTX tool. This requires the `FILESYSTEM_WRITE` permission.
|
||||
Write it to a different disk than the one being carved: writing
|
||||
to the same disk can overwrite the deleted data being recovered.
|
||||
|
||||
- Each chunk is written once, even if it was found several times.
|
||||
- Windows refuses to open a file containing any chunk with a bad
|
||||
checksum. Damaged chunks are therefore cut down to their leading
|
||||
intact records, and their checksums are recomputed.
|
||||
- Windows stops reading when the record numbers in the chunk
|
||||
headers do not follow on from each other, so the chunk headers
|
||||
are renumbered. The event record ids of the events are not
|
||||
changed.
|
||||
|
||||
```vql
|
||||
SELECT count() AS Events
|
||||
FROM carve_evtx(filename="C:", accessor="ntfs",
|
||||
output="C:/cases/recovered.evtx")
|
||||
GROUP BY 1
|
||||
```
|
||||
type: Plugin
|
||||
version: 1
|
||||
args:
|
||||
- name: filename
|
||||
type: accessors.OSPath
|
||||
description: The file or device to carve (e.g. 'C:' with the 'ntfs' accessor,
|
||||
or C:/pagefile.sys with the ntfs accessor).
|
||||
required: true
|
||||
- name: accessor
|
||||
type: string
|
||||
description: The accessor to use.
|
||||
- name: messagedb
|
||||
type: string
|
||||
description: A Message database from https://github.com/Velocidex/evtx-data.
|
||||
- name: start_offset
|
||||
type: int64
|
||||
description: Start carving at this offset (default 0).
|
||||
- name: end_offset
|
||||
type: int64
|
||||
description: Stop carving at this offset (default end of file).
|
||||
- name: output
|
||||
type: string
|
||||
description: If set, also write the recovered chunks to a new EVTX file at this
|
||||
path so they can be opened with Event Viewer. Each chunk is written once.
|
||||
category: parsers
|
||||
metadata:
|
||||
permissions: FILESYSTEM_READ,FILESYSTEM_WRITE
|
||||
platforms:
|
||||
- darwin_amd64_cgo
|
||||
- darwin_arm64_cgo
|
||||
- linux_amd64_cgo
|
||||
- windows_386_cgo
|
||||
- windows_amd64_cgo
|
||||
- name: carve_usn
|
||||
description: |
|
||||
Carve for the USN journal entries from a device.
|
||||
@@ -13644,3 +13745,4 @@
|
||||
- linux_amd64_cgo
|
||||
- windows_386_cgo
|
||||
- windows_amd64_cgo
|
||||
|
||||
|
||||
@@ -81,9 +81,9 @@ require (
|
||||
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
|
||||
gopkg.in/sourcemap.v1 v1.0.5 // indirect
|
||||
howett.net/plist v1.0.0
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe
|
||||
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57
|
||||
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5
|
||||
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d
|
||||
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9
|
||||
www.velocidex.com/golang/go-prefetch v0.0.0-20260922101442-4bae97962980
|
||||
www.velocidex.com/golang/oleparse v0.0.0-20250312121321-f7c2b4ec0959
|
||||
|
||||
@@ -1119,12 +1119,12 @@ software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJ
|
||||
software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ=
|
||||
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf1AsYiIzUMJMIdFsVdrIw/BjrGzZbrsnz9xmeZmlYU=
|
||||
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4=
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8 h1:Me1/9DB8ZMH6+ZNFWPlfn3JEFgZNH59Cfk83wcQ2PEk=
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8/go.mod h1:AEH50xeMcAUGiCXLAueUOq800jFgFyqngyPt88fDzow=
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe h1:CaGlM+7W48tSmI1k7/aodJXwmOQwgKSB/UkQ+zY6V5I=
|
||||
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe/go.mod h1:AEH50xeMcAUGiCXLAueUOq800jFgFyqngyPt88fDzow=
|
||||
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57 h1:CRdOpwPmVFzjfSrJXLj+xHOMyKfK4+WlJTf9IDMdXtk=
|
||||
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57/go.mod h1:H1Gw62XkspXEtZPT2ep+L57GgdKIDHwueazxKnn5EXc=
|
||||
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5 h1:GDqKvZ2fdZVHQRSMczl41u2DPJEgbYdK6aaVkCBB8Jc=
|
||||
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5/go.mod h1:4MSO8W9iNMXyBpjSpxApWfMjJUb9IWFD2Yis5JPZaSY=
|
||||
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d h1:RzEtNw2k3jX8w5HLneg73Oz3vqi3YoYSHenoKjAC8S4=
|
||||
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d/go.mod h1:4MSO8W9iNMXyBpjSpxApWfMjJUb9IWFD2Yis5JPZaSY=
|
||||
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9 h1:y5nOjOAyjH36mEGZsX2Wx3XpS78e9Jk2wK1M99rn4TY=
|
||||
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9/go.mod h1:rEueOnxvtbV/uFfCeD44KvC6TL9ts/1S3VNmRudSaAc=
|
||||
www.velocidex.com/golang/go-prefetch v0.0.0-20260922101442-4bae97962980 h1:YZhTjoaHG0/wuIOYHDg4q5nlKLx4BOkfORsiv8hWmxw=
|
||||
|
||||
@@ -190,6 +190,11 @@ Workspace symbols for query:
|
||||
"kind": 12,
|
||||
"location": null
|
||||
},
|
||||
{
|
||||
"name": "carve_evtx",
|
||||
"kind": 5,
|
||||
"location": null
|
||||
},
|
||||
{
|
||||
"name": "carve_usn",
|
||||
"kind": 5,
|
||||
|
||||
@@ -132,7 +132,11 @@ func (self *PagedReader) ReadAt(buf []byte, offset int64) (
|
||||
to_read = len(page_buf) - page_offset
|
||||
copy(buf[buf_idx:buf_idx+to_read],
|
||||
page_buf[page_offset:page_offset+to_read])
|
||||
return buf_idx + to_read, err
|
||||
|
||||
// EOF because the end of the buffer is past the file's
|
||||
// end. We assume if the page buffer is short is because
|
||||
// it is at EOF.
|
||||
return buf_idx + to_read, io.EOF
|
||||
}
|
||||
|
||||
copy(buf[buf_idx:buf_idx+to_read],
|
||||
|
||||
@@ -34,11 +34,9 @@ func TestPagedReader(t *testing.T) {
|
||||
reads := 0
|
||||
for {
|
||||
n, err := fd.Read(buf)
|
||||
if n == 0 || err == io.EOF {
|
||||
if n == 0 || err != nil && err != io.EOF {
|
||||
break
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
|
||||
reads++
|
||||
result += string(buf[:n])
|
||||
}
|
||||
|
||||
@@ -84,6 +84,34 @@ func (self *ReadSeekReaderAdapter) Read(buf []byte) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadAt reads from the delegate without changing the current offset
|
||||
// or the EOF state used by Read(), so reading past the end does not
|
||||
// cause later reads to fail.
|
||||
func (self *ReadSeekReaderAdapter) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
self.mu.Lock()
|
||||
size := self.size
|
||||
self.mu.Unlock()
|
||||
|
||||
if offset < 0 {
|
||||
return 0, IOError
|
||||
}
|
||||
|
||||
if size > 0 {
|
||||
if offset >= size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if offset+int64(len(buf)) > size {
|
||||
n, err := self.reader.ReadAt(buf[:size-offset], offset)
|
||||
if err == nil {
|
||||
err = io.EOF
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
|
||||
return self.reader.ReadAt(buf, offset)
|
||||
}
|
||||
|
||||
func (self *ReadSeekReaderAdapter) SetSize(size int64) {
|
||||
self.mu.Lock()
|
||||
defer self.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Reading past the end with Read() sets a sticky EOF flag. ReadAt()
|
||||
// must not be affected by it.
|
||||
func TestReadSeekReaderAdapterReadAt(t *testing.T) {
|
||||
data := []byte("0123456789")
|
||||
adapter := NewReadSeekReaderAdapter(bytes.NewReader(data), nil)
|
||||
adapter.SetSize(int64(len(data)))
|
||||
|
||||
// Exhaust the reader with Read().
|
||||
_, err := io.ReadAll(adapter)
|
||||
assert.NoError(t, err)
|
||||
|
||||
n, err := adapter.Read(make([]byte, 1))
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
|
||||
// ReadAt still works anywhere in the file.
|
||||
buf := make([]byte, 4)
|
||||
n, err = adapter.ReadAt(buf, 2)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "2345", string(buf[:n]))
|
||||
|
||||
// A read crossing the size is cut short with EOF.
|
||||
n, err = adapter.ReadAt(buf, 8)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
assert.Equal(t, "89", string(buf[:n]))
|
||||
|
||||
// Reading past the end does not break later reads.
|
||||
n, err = adapter.ReadAt(buf, 20)
|
||||
assert.Equal(t, 0, n)
|
||||
assert.Equal(t, io.EOF, err)
|
||||
|
||||
n, err = adapter.ReadAt(buf, 0)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "0123", string(buf[:n]))
|
||||
|
||||
// MakeReaderAtter uses the stateless ReadAt.
|
||||
_, ok := MakeReaderAtter(adapter).(*ReadSeekReaderAdapter)
|
||||
assert.True(t, ok)
|
||||
}
|
||||
@@ -98,11 +98,6 @@ func (self VerifyFunction) Call(ctx context.Context,
|
||||
}
|
||||
|
||||
func stateToDict(state *launcher.AnalysisState) *ordereddict.Dict {
|
||||
var errors []string
|
||||
for _, e := range state.Errors {
|
||||
errors = append(errors, e.Error())
|
||||
}
|
||||
|
||||
definitions := make(map[string]*ordereddict.Dict)
|
||||
for key, d := range state.Definitions {
|
||||
var args []string
|
||||
@@ -120,7 +115,7 @@ func stateToDict(state *launcher.AnalysisState) *ordereddict.Dict {
|
||||
return ordereddict.NewDict().
|
||||
Set("Artifact", state.Artifact).
|
||||
Set("Permissions", state.Permissions).
|
||||
Set("Errors", errors).
|
||||
Set("Errors", state.Errors).
|
||||
Set("Warnings", state.Warnings).
|
||||
Set("Definitions", definitions).
|
||||
Set("Suppressions", state.Suppressions)
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
Velociraptor - Dig Deeper
|
||||
Copyright (C) 2019-2025 Rapid7 Inc.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package event_logs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/Velocidex/ordereddict"
|
||||
"github.com/dustin/go-humanize"
|
||||
"www.velocidex.com/golang/evtx"
|
||||
"www.velocidex.com/golang/velociraptor/accessors"
|
||||
"www.velocidex.com/golang/velociraptor/accessors/file"
|
||||
"www.velocidex.com/golang/velociraptor/acls"
|
||||
"www.velocidex.com/golang/velociraptor/utils"
|
||||
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
|
||||
"www.velocidex.com/golang/velociraptor/vql/readers"
|
||||
vfilter "www.velocidex.com/golang/vfilter"
|
||||
"www.velocidex.com/golang/vfilter/arg_parser"
|
||||
)
|
||||
|
||||
type _CarveEvtxPluginArgs struct {
|
||||
Filename *accessors.OSPath `vfilter:"required,field=filename,doc=The file or device to carve (e.g. 'C:' with the 'ntfs' accessor, or C:/pagefile.sys with the ntfs accessor)."`
|
||||
Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use."`
|
||||
Database string `vfilter:"optional,field=messagedb,doc=A Message database from https://github.com/Velocidex/evtx-data."`
|
||||
StartOffset int64 `vfilter:"optional,field=start_offset,doc=Start carving at this offset (default 0)."`
|
||||
EndOffset int64 `vfilter:"optional,field=end_offset,doc=Stop carving at this offset (default end of file)."`
|
||||
Output string `vfilter:"optional,field=output,doc=If set, also write the recovered chunks to a new EVTX file at this path so they can be opened with Event Viewer. Each chunk is written once."`
|
||||
}
|
||||
|
||||
type _CarveEvtxPlugin struct{}
|
||||
|
||||
func (self _CarveEvtxPlugin) Call(
|
||||
ctx context.Context,
|
||||
scope vfilter.Scope,
|
||||
args *ordereddict.Dict) <-chan vfilter.Row {
|
||||
output_chan := make(chan vfilter.Row)
|
||||
|
||||
go func() {
|
||||
defer close(output_chan)
|
||||
defer vql_subsystem.RegisterMonitor(ctx, "carve_evtx", args)()
|
||||
defer utils.RecoverVQL(scope)
|
||||
|
||||
arg := &_CarveEvtxPluginArgs{}
|
||||
err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
resolver, err := getMessageResolver(scope, arg.Database)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var writer *evtxWriter
|
||||
if arg.Output != "" {
|
||||
out_fd, err := openOutputFile(scope, arg.Output)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: %v", err)
|
||||
return
|
||||
}
|
||||
defer out_fd.Close()
|
||||
|
||||
writer, err = newEvtxWriter(out_fd)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := writer.Close()
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
|
||||
return
|
||||
}
|
||||
scope.Log("carve_evtx: Wrote %v unique chunks to %v",
|
||||
writer.Chunks(), arg.Output)
|
||||
}()
|
||||
}
|
||||
|
||||
scope.Log("carve_evtx: Carving %v for EVTX chunks",
|
||||
utils.Elide(arg.Filename.String(), 50))
|
||||
|
||||
stats := &carveStats{}
|
||||
var records_emitted int64
|
||||
|
||||
reader, err := readers.NewAccessorReader(
|
||||
scope, arg.Accessor, arg.Filename, 100)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: %v", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
for chunk := range carveChunks(
|
||||
ctx, scope,
|
||||
reader, arg.StartOffset, arg.EndOffset, stats) {
|
||||
scope.ChargeOp()
|
||||
|
||||
if writer != nil && includeInOutput(chunk) {
|
||||
_, err := writer.WriteChunk(chunk)
|
||||
if err != nil {
|
||||
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
records, err := parseCarvedChunk(chunk)
|
||||
if err != nil && len(records) == 0 {
|
||||
scope.Log("carve_evtx: chunk at offset %#x: %v", chunk.Offset, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, record := range records {
|
||||
row := makeCarvedRow(chunk, record, resolver)
|
||||
if row == nil {
|
||||
continue
|
||||
}
|
||||
records_emitted++
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case output_chan <- row:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if stats.StopError != nil {
|
||||
scope.Log("carve_evtx: Stopped at offset %#x: %v",
|
||||
arg.StartOffset+stats.BytesScanned, stats.StopError)
|
||||
}
|
||||
|
||||
scope.Log("carve_evtx: Scanned %v bytes, found %v chunk signatures, %v plausible chunks, %v records",
|
||||
humanize.Bytes(uint64(stats.BytesScanned)),
|
||||
stats.Candidates, stats.Chunks, records_emitted)
|
||||
}()
|
||||
|
||||
return output_chan
|
||||
}
|
||||
|
||||
// Chunks written to the output file.
|
||||
func includeInOutput(chunk *carvedChunk) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Create the output file after checking that the query may write
|
||||
// there.
|
||||
func openOutputFile(scope vfilter.Scope, path string) (*os.File, error) {
|
||||
err := vql_subsystem.CheckAccess(scope, acls.FILESYSTEM_WRITE)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = file.CheckPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
}
|
||||
|
||||
// Emit the same event structure as parse_evtx() with details about
|
||||
// where the record was found and how trustworthy its chunk is.
|
||||
func makeCarvedRow(chunk *carvedChunk, record *evtx.EventRecord,
|
||||
resolver evtx.MessageResolver) *ordereddict.Dict {
|
||||
|
||||
event_map, ok := record.Event.(*ordereddict.Dict)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
event, pres := ordereddict.GetMap(event_map, "Event")
|
||||
if !pres {
|
||||
return nil
|
||||
}
|
||||
|
||||
if resolver != nil {
|
||||
event.Set("Message", evtx.ExpandMessage(event, resolver))
|
||||
}
|
||||
|
||||
return event.Set("ChunkInfo", ordereddict.NewDict().
|
||||
Set("Offset", chunk.Offset).
|
||||
Set("HeaderValid", chunk.HeaderChecksumValid).
|
||||
Set("DataValid", chunk.DataChecksumValid).
|
||||
Set("Truncated", chunk.Truncated).
|
||||
Set("Compressed", chunk.Compressed))
|
||||
}
|
||||
|
||||
func (self _CarveEvtxPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
|
||||
return &vfilter.PluginInfo{
|
||||
Name: "carve_evtx",
|
||||
Doc: "Carve EVTX chunks from a file or device (e.g. unallocated space, " +
|
||||
"the pagefile or a memory image) and parse their event records.",
|
||||
ArgType: type_map.AddType(scope, &_CarveEvtxPluginArgs{}),
|
||||
// Writing the output file checks FILESYSTEM_WRITE when it is used.
|
||||
Metadata: vql_subsystem.VQLMetadata().
|
||||
Permissions(acls.FILESYSTEM_READ, acls.FILESYSTEM_WRITE).Build(),
|
||||
Version: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
vql_subsystem.RegisterPlugin(&_CarveEvtxPlugin{})
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
/*
|
||||
Velociraptor - Dig Deeper
|
||||
Copyright (C) 2019-2025 Rapid7 Inc.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package event_logs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"www.velocidex.com/golang/evtx"
|
||||
ntfs "www.velocidex.com/golang/go-ntfs/parser"
|
||||
"www.velocidex.com/golang/velociraptor/utils"
|
||||
vfilter "www.velocidex.com/golang/vfilter"
|
||||
)
|
||||
|
||||
const (
|
||||
// How much data we scan for the chunk signature in each read.
|
||||
carveBlockSize = 1024 * 1024
|
||||
|
||||
// Offsets into the chunk header. See
|
||||
// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc
|
||||
chunkHeaderSizeValue = 0x80
|
||||
chunkFreeSpaceOffset = 0x30
|
||||
chunkHeaderChecksumStart = 0x78
|
||||
chunkHeaderChecksumEnd = 0x80
|
||||
|
||||
// The library refuses to parse more records than this from a
|
||||
// single chunk so a header claiming more is not plausible.
|
||||
maxRecordsPerChunk = 1024 * 10
|
||||
|
||||
// LZNT1 block header: bit 15 is set for compressed blocks,
|
||||
// bits 12-14 are always 3 and the low 12 bits are the size of
|
||||
// the block minus 3.
|
||||
lznt1BlockSize = 0x1000
|
||||
lznt1UnitSize = 0x10000
|
||||
chunkOffsetInUnit = 0x1000
|
||||
lznt1PrefixSize = 3
|
||||
lznt1SignatureMask = 0x7000
|
||||
lznt1Signature = 0x3000
|
||||
lznt1CompressedHeaderMask = 0xb000
|
||||
lznt1SizeMask = 0x0fff
|
||||
)
|
||||
|
||||
var chunkMagic = []byte(evtx.EVTX_CHUNK_HEADER_MAGIC)
|
||||
|
||||
// A chunk signature found in the carved data.
|
||||
type carvedChunk struct {
|
||||
// Offset of the chunk in the carved stream.
|
||||
Offset int64
|
||||
|
||||
Header evtx.ChunkHeader
|
||||
|
||||
// The CRC32 of the chunk header matches the stored checksum.
|
||||
HeaderChecksumValid bool
|
||||
|
||||
// The CRC32 of the event records matches the stored checksum.
|
||||
DataChecksumValid bool
|
||||
|
||||
// The stream ended before the full chunk could be read. The
|
||||
// missing data is zero filled.
|
||||
Truncated bool
|
||||
|
||||
// The chunk was found inside an NTFS compressed file and was
|
||||
// decompressed.
|
||||
Compressed bool
|
||||
|
||||
// The full chunk, always EVTX_CHUNK_SIZE long.
|
||||
Data []byte
|
||||
}
|
||||
|
||||
type carveStats struct {
|
||||
BytesScanned int64
|
||||
Candidates int64
|
||||
Chunks int64
|
||||
|
||||
// Why the scan stopped before reaching the end offset, if it did.
|
||||
StopError error
|
||||
}
|
||||
|
||||
// Scan the reader between start and end (end <= 0 means until the
|
||||
// end of the stream) for EVTX chunks. Signatures are searched at any
|
||||
// byte offset because chunks in memory, the pagefile or unallocated
|
||||
// space are not necessarily aligned. Only chunks with a plausible
|
||||
// header are emitted - their checksums are reported but not enforced
|
||||
// because partially overwritten chunks still contain useful records.
|
||||
func carveChunks(
|
||||
ctx context.Context,
|
||||
scope vfilter.Scope,
|
||||
reader io.ReaderAt,
|
||||
start, end int64, stats *carveStats) <-chan *carvedChunk {
|
||||
|
||||
output_chan := make(chan *carvedChunk)
|
||||
|
||||
go func() {
|
||||
defer close(output_chan)
|
||||
defer utils.RecoverVQL(scope)
|
||||
|
||||
logger, closer := utils.NewDeduplicatedLogger(30 * time.Second)
|
||||
defer closer()
|
||||
|
||||
start_time := utils.GetTime().Now()
|
||||
|
||||
buf := make([]byte, carveBlockSize)
|
||||
|
||||
// The end of the previous block is kept so a signature
|
||||
// straddling two blocks is still found. Carrying it in
|
||||
// memory keeps every read at the block offsets.
|
||||
overlap := len(chunkMagic) - 1
|
||||
window := make([]byte, 0, overlap+carveBlockSize)
|
||||
var tail []byte
|
||||
|
||||
// Scan from an aligned offset and ignore signatures before
|
||||
// the start.
|
||||
offset := start
|
||||
for end <= 0 || offset < end {
|
||||
to_read := int64(len(buf))
|
||||
if end > 0 && offset+to_read > end {
|
||||
to_read = end - offset
|
||||
}
|
||||
|
||||
// Readers may return fewer bytes than requested, with or
|
||||
// without an error. Only a read returning nothing ends
|
||||
// the scan.
|
||||
n, err := reader.ReadAt(buf[:to_read], int64(offset))
|
||||
if n <= 0 {
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
stats.StopError = err
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
window = append(append(window[:0], tail...), buf[:n]...)
|
||||
window_offset := offset - int64(len(tail))
|
||||
|
||||
for idx := 0; idx < len(window); {
|
||||
hit := bytes.Index(window[idx:], chunkMagic)
|
||||
if hit < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
chunk_offset := window_offset + int64(idx+hit)
|
||||
idx += hit + 1
|
||||
if chunk_offset < start {
|
||||
continue
|
||||
}
|
||||
|
||||
stats.Candidates++
|
||||
chunk, ok := readChunk(reader, chunk_offset)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
stats.Chunks++
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case output_chan <- chunk:
|
||||
}
|
||||
}
|
||||
|
||||
// A full signature can never be inside the tail, so no
|
||||
// signature is counted twice.
|
||||
tail_start := len(window) - overlap
|
||||
if tail_start < 0 {
|
||||
tail_start = 0
|
||||
}
|
||||
tail = append(tail[:0], window[tail_start:]...)
|
||||
|
||||
offset += int64(n)
|
||||
|
||||
stats.BytesScanned = offset - start
|
||||
|
||||
// Report progress - this could take a while so it is nice
|
||||
// to know how it's going.
|
||||
logger.Log(
|
||||
// This will be called a lot so log lazily.
|
||||
func(msg string, args ...interface{}) {
|
||||
scope.Log("carve_evtx: scanned %v bytes in %v",
|
||||
humanize.Bytes(uint64(stats.BytesScanned)),
|
||||
utils.GetTime().Now().Sub(
|
||||
start_time).Round(time.Second))
|
||||
}, "")
|
||||
|
||||
if errors.Is(err, io.EOF) {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return output_chan
|
||||
}
|
||||
|
||||
// Read the chunk at offset and check that its header is plausible.
|
||||
func readChunk(reader io.ReaderAt, offset int64) (*carvedChunk, bool) {
|
||||
data := make([]byte, evtx.EVTX_CHUNK_SIZE)
|
||||
n, _ := reader.ReadAt(data, offset)
|
||||
result, ok := checkChunk(offset, data, n)
|
||||
if ok {
|
||||
return result, true
|
||||
}
|
||||
|
||||
data, n = readCompressedChunk(reader, offset)
|
||||
result, ok = checkChunk(offset, data, n)
|
||||
if ok {
|
||||
result.Compressed = true
|
||||
}
|
||||
return result, ok
|
||||
}
|
||||
|
||||
// NTFS compresses files in units of 16 clusters using LZNT1, and
|
||||
// the event logs are compressed by default. Every 4kb block in a
|
||||
// unit is compressed separately, so a chunk starting on a block
|
||||
// boundary still has its signature as 8 literal bytes - preceded by
|
||||
// the 2 byte block header and a zero flag byte.
|
||||
//
|
||||
// Chunks follow the 4kb file header so a chunk spans two
|
||||
// compression units. Each unit starts on a new cluster which we
|
||||
// assume to be 4kb.
|
||||
func readCompressedChunk(reader io.ReaderAt, offset int64) ([]byte, int) {
|
||||
start := offset - lznt1PrefixSize
|
||||
if start < 0 {
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
// Compressed data is never larger than the uncompressed data
|
||||
// plus block headers and the slack at the end of a unit.
|
||||
raw := make([]byte, evtx.EVTX_CHUNK_SIZE+2*lznt1BlockSize)
|
||||
n, _ := reader.ReadAt(raw, start)
|
||||
raw = raw[:n]
|
||||
|
||||
if n < lznt1PrefixSize || raw[2] != 0 ||
|
||||
binary.LittleEndian.Uint16(raw)&lznt1CompressedHeaderMask !=
|
||||
lznt1CompressedHeaderMask {
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
data := make([]byte, 0, evtx.EVTX_CHUNK_SIZE)
|
||||
skipped_to_next_unit := false
|
||||
i := 0
|
||||
for len(data) < evtx.EVTX_CHUNK_SIZE && i+2 <= len(raw) {
|
||||
header := binary.LittleEndian.Uint16(raw[i:])
|
||||
|
||||
// The compression unit has ended - either we have read all
|
||||
// its blocks or there is no block header. The next unit
|
||||
// starts on the next cluster.
|
||||
unit_full := len(data) == lznt1UnitSize-chunkOffsetInUnit
|
||||
if (unit_full && !skipped_to_next_unit) ||
|
||||
header&lznt1SignatureMask != lznt1Signature {
|
||||
if skipped_to_next_unit {
|
||||
// The next unit was not compressible so it is
|
||||
// stored as is.
|
||||
remaining := evtx.EVTX_CHUNK_SIZE - len(data)
|
||||
if remaining > len(raw)-i {
|
||||
remaining = len(raw) - i
|
||||
}
|
||||
data = append(data, raw[i:i+remaining]...)
|
||||
break
|
||||
}
|
||||
skipped_to_next_unit = true
|
||||
|
||||
next_unit := start + int64(i) + lznt1BlockSize - 1
|
||||
next_unit -= next_unit % lznt1BlockSize
|
||||
i = int(next_unit - start)
|
||||
continue
|
||||
}
|
||||
|
||||
block_end := i + int(header&lznt1SizeMask) + 3
|
||||
if block_end > len(raw) {
|
||||
break
|
||||
}
|
||||
|
||||
block, err := lznt1Decompress(raw[i:block_end])
|
||||
if err != nil || len(block) == 0 || len(block) > lznt1BlockSize {
|
||||
break
|
||||
}
|
||||
data = append(data, block...)
|
||||
i = block_end
|
||||
}
|
||||
|
||||
if len(data) > evtx.EVTX_CHUNK_SIZE {
|
||||
data = data[:evtx.EVTX_CHUNK_SIZE]
|
||||
}
|
||||
n = len(data)
|
||||
|
||||
// Always return a full chunk, zero filled.
|
||||
return append(data, make([]byte, evtx.EVTX_CHUNK_SIZE-n)...), n
|
||||
}
|
||||
|
||||
// Check that the chunk has a plausible header. n is the number of
|
||||
// valid bytes in data.
|
||||
func checkChunk(offset int64, data []byte, n int) (*carvedChunk, bool) {
|
||||
if n < evtx.EVTX_CHUNK_HEADER_SIZE {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
result := &carvedChunk{
|
||||
Offset: offset,
|
||||
Data: data,
|
||||
Truncated: n < evtx.EVTX_CHUNK_SIZE,
|
||||
}
|
||||
|
||||
err := binary.Read(bytes.NewReader(data),
|
||||
binary.LittleEndian, &result.Header)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
header := &result.Header
|
||||
free_space := binary.LittleEndian.Uint32(data[chunkFreeSpaceOffset:])
|
||||
|
||||
if header.HeaderSize != chunkHeaderSizeValue ||
|
||||
header.FirstEventRecNumber > header.LastEventRecNumber ||
|
||||
header.LastEventRecNumber-header.FirstEventRecNumber >= maxRecordsPerChunk ||
|
||||
header.LastEventRecOffset >= evtx.EVTX_CHUNK_SIZE ||
|
||||
free_space < evtx.EVTX_CHUNK_HEADER_SIZE ||
|
||||
free_space > evtx.EVTX_CHUNK_SIZE {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// The header checksum covers the first 120 bytes and the
|
||||
// string/template tables, skipping the flags and the checksum.
|
||||
crc := crc32.NewIEEE()
|
||||
_, _ = crc.Write(data[:chunkHeaderChecksumStart])
|
||||
_, _ = crc.Write(data[chunkHeaderChecksumEnd:evtx.EVTX_CHUNK_HEADER_SIZE])
|
||||
result.HeaderChecksumValid = crc.Sum32() == header.CheckSum
|
||||
|
||||
result.DataChecksumValid = crc32.ChecksumIEEE(
|
||||
data[evtx.EVTX_CHUNK_HEADER_SIZE:free_space]) == header.EventRecordCheckSum
|
||||
|
||||
return result, true
|
||||
}
|
||||
|
||||
// LZNT1Decompress can panic on malformed blocks (e.g. a back
|
||||
// reference truncated by the end of the block). Carved data is often
|
||||
// malformed so contain any panic to this block.
|
||||
func lznt1Decompress(block []byte) (result []byte, err error) {
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r != nil {
|
||||
err = fmt.Errorf("LZNT1 decompression panic: %v", r)
|
||||
utils.PrintStack()
|
||||
}
|
||||
}()
|
||||
|
||||
return ntfs.LZNT1Decompress(block)
|
||||
}
|
||||
|
||||
// Parse the records in a carved chunk. Carved data may be corrupt so
|
||||
// any panic in the parser is contained to this chunk.
|
||||
func parseCarvedChunk(chunk *carvedChunk) (records []*evtx.EventRecord, err error) {
|
||||
defer func() {
|
||||
r := recover()
|
||||
if r != nil {
|
||||
err = errors.New("parser panic while parsing carved chunk")
|
||||
utils.PrintStack()
|
||||
}
|
||||
}()
|
||||
|
||||
parsed, err := evtx.NewChunk(bytes.NewReader(chunk.Data), 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return parsed.Parse(0)
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package event_logs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Velocidex/ordereddict"
|
||||
"www.velocidex.com/golang/evtx"
|
||||
ntfs "www.velocidex.com/golang/go-ntfs/parser"
|
||||
"www.velocidex.com/golang/velociraptor/utils"
|
||||
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
|
||||
"www.velocidex.com/golang/velociraptor/vtesting/assert"
|
||||
)
|
||||
|
||||
// This log contains a single chunk recording the System log being
|
||||
// cleared (Event ID 104).
|
||||
const testEvtx = "../../../artifacts/testdata/files/DE_104_system_log_cleared.evtx"
|
||||
|
||||
func getTestChunk(t *testing.T) []byte {
|
||||
data, err := os.ReadFile(testEvtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// The first chunk follows the 4kb file header.
|
||||
chunk := data[0x1000 : 0x1000+evtx.EVTX_CHUNK_SIZE]
|
||||
assert.Equal(t, evtx.EVTX_CHUNK_HEADER_MAGIC, string(chunk[:8]))
|
||||
return chunk
|
||||
}
|
||||
|
||||
func noise(rng *rand.Rand, size int) []byte {
|
||||
result := make([]byte, size)
|
||||
rng.Read(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func carveAll(t *testing.T, image []byte, start, end int64) (
|
||||
[]*carvedChunk, *carveStats) {
|
||||
scope := vql_subsystem.MakeScope()
|
||||
|
||||
stats := &carveStats{}
|
||||
var result []*carvedChunk
|
||||
for chunk := range carveChunks(context.Background(), scope,
|
||||
bytes.NewReader(image), start, end, stats) {
|
||||
result = append(result, chunk)
|
||||
}
|
||||
return result, stats
|
||||
}
|
||||
|
||||
func TestCarveChunks(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
// A copy of the chunk with a damaged string table - the header
|
||||
// is still plausible but its checksum no longer matches.
|
||||
damaged := append([]byte{}, chunk...)
|
||||
damaged[0x100] ^= 0xff
|
||||
|
||||
image := &bytes.Buffer{}
|
||||
image.Write(noise(rng, 100003))
|
||||
|
||||
// 1. An intact chunk at an unaligned offset.
|
||||
intact_offset := int64(image.Len())
|
||||
image.Write(chunk)
|
||||
image.Write(noise(rng, 5000))
|
||||
|
||||
// A false positive: the signature followed by garbage.
|
||||
image.WriteString(evtx.EVTX_CHUNK_HEADER_MAGIC)
|
||||
image.Write(noise(rng, 1000))
|
||||
|
||||
// 2. A chunk with a damaged header.
|
||||
damaged_offset := int64(image.Len())
|
||||
image.Write(damaged)
|
||||
image.Write(noise(rng, 7777))
|
||||
|
||||
// 3. A chunk cut off by the end of the image.
|
||||
truncated_offset := int64(image.Len())
|
||||
image.Write(chunk[:0x8000])
|
||||
|
||||
chunks, stats := carveAll(t, image.Bytes(), 0, 0)
|
||||
|
||||
assert.Equal(t, int64(image.Len()), stats.BytesScanned)
|
||||
assert.Equal(t, int64(4), stats.Candidates)
|
||||
assert.Equal(t, int64(3), stats.Chunks)
|
||||
assert.Equal(t, 3, len(chunks))
|
||||
|
||||
assert.Equal(t, intact_offset, chunks[0].Offset)
|
||||
assert.True(t, chunks[0].HeaderChecksumValid)
|
||||
assert.True(t, chunks[0].DataChecksumValid)
|
||||
assert.False(t, chunks[0].Truncated)
|
||||
|
||||
assert.Equal(t, damaged_offset, chunks[1].Offset)
|
||||
assert.False(t, chunks[1].HeaderChecksumValid)
|
||||
assert.True(t, chunks[1].DataChecksumValid)
|
||||
|
||||
assert.Equal(t, truncated_offset, chunks[2].Offset)
|
||||
assert.True(t, chunks[2].HeaderChecksumValid)
|
||||
assert.True(t, chunks[2].Truncated)
|
||||
assert.Equal(t, evtx.EVTX_CHUNK_SIZE, len(chunks[2].Data))
|
||||
|
||||
// All the records in the intact chunk are recovered.
|
||||
records, err := parseCarvedChunk(chunks[0])
|
||||
assert.NoError(t, err)
|
||||
header := chunks[0].Header
|
||||
assert.Equal(t, int(header.LastEventRecNumber-header.FirstEventRecNumber+1),
|
||||
len(records))
|
||||
|
||||
// The log cleared event is among them.
|
||||
found := false
|
||||
for _, record := range records {
|
||||
row := makeCarvedRow(chunks[0], record, nil)
|
||||
assert.NotNil(t, row)
|
||||
|
||||
value, _ := ordereddict.GetAny(row, "System.EventID.Value")
|
||||
event_id, _ := utils.ToInt64(value)
|
||||
if event_id == 104 {
|
||||
found = true
|
||||
}
|
||||
|
||||
chunk_info_any, pres := row.Get("ChunkInfo")
|
||||
assert.True(t, pres)
|
||||
|
||||
chunk_offset, _ := chunk_info_any.(*ordereddict.Dict).GetInt64("Offset")
|
||||
assert.Equal(t, intact_offset, chunk_offset)
|
||||
}
|
||||
assert.True(t, found)
|
||||
}
|
||||
|
||||
// A signature split across two read blocks must be found exactly once.
|
||||
func TestCarveChunkAcrossBlockBoundary(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(2))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
for _, delta := range []int{-8, -7, -3, -1, 0, 1} {
|
||||
offset := carveBlockSize + delta
|
||||
image := noise(rng, offset)
|
||||
image = append(image, chunk...)
|
||||
image = append(image, noise(rng, 100)...)
|
||||
|
||||
chunks, _ := carveAll(t, image, 0, 0)
|
||||
assert.Equal(t, 1, len(chunks), "delta %v", delta)
|
||||
assert.Equal(t, int64(offset), chunks[0].Offset, "delta %v", delta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCarveRange(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(3))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
image := noise(rng, 1000)
|
||||
image = append(image, chunk...)
|
||||
second := len(image)
|
||||
image = append(image, chunk...)
|
||||
|
||||
// Skipping past the first chunk only finds the second.
|
||||
chunks, _ := carveAll(t, image, 1001, 0)
|
||||
assert.Equal(t, 1, len(chunks))
|
||||
assert.Equal(t, int64(second), chunks[0].Offset)
|
||||
|
||||
// A range ending before the second chunk only finds the first.
|
||||
chunks, _ = carveAll(t, image, 0, int64(second))
|
||||
assert.Equal(t, 1, len(chunks))
|
||||
assert.Equal(t, int64(1000), chunks[0].Offset)
|
||||
}
|
||||
|
||||
// Accessors return a ReadSeekReaderAdapter rather than a plain
|
||||
// io.ReaderAt. Its EOF flag is sticky, so a chunk read that runs past
|
||||
// the end of the stream must not stop the scan early.
|
||||
func TestCarveThroughAccessorAdapter(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(5))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
image := noise(rng, carveBlockSize/2)
|
||||
first := len(image)
|
||||
image = append(image, chunk...)
|
||||
image = append(image, noise(rng, 2*carveBlockSize)...)
|
||||
second := len(image)
|
||||
image = append(image, chunk...)
|
||||
image = append(image, noise(rng, 1000)...)
|
||||
|
||||
// A chunk cut off by the end of the stream.
|
||||
third := len(image)
|
||||
image = append(image, chunk[:0x4000]...)
|
||||
|
||||
scope := vql_subsystem.MakeScope()
|
||||
|
||||
for _, with_size := range []bool{false, true} {
|
||||
adapter := utils.NewReadSeekReaderAdapter(bytes.NewReader(image), nil)
|
||||
if with_size {
|
||||
adapter.SetSize(int64(len(image)))
|
||||
}
|
||||
|
||||
stats := &carveStats{}
|
||||
var offsets []int64
|
||||
for c := range carveChunks(context.Background(), scope,
|
||||
utils.MakeReaderAtter(adapter), 0, 0, stats) {
|
||||
offsets = append(offsets, c.Offset)
|
||||
}
|
||||
|
||||
assert.Equal(t, []int64{int64(first), int64(second), int64(third)},
|
||||
offsets, "with_size %v", with_size)
|
||||
assert.Equal(t, int64(len(image)), stats.BytesScanned,
|
||||
"with_size %v", with_size)
|
||||
}
|
||||
}
|
||||
|
||||
// Behaves like a raw Windows device which rejects reads that are not
|
||||
// sector aligned.
|
||||
type deviceReader struct {
|
||||
reader *bytes.Reader
|
||||
}
|
||||
|
||||
func (self *deviceReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
if offset%512 != 0 || len(buf)%512 != 0 {
|
||||
return 0, errors.New("The parameter is incorrect.")
|
||||
}
|
||||
return self.reader.ReadAt(buf, offset)
|
||||
}
|
||||
|
||||
func NewDeviceReader(data []byte) (io.ReaderAt, error) {
|
||||
return utils.NewPagedReader(
|
||||
&deviceReader{bytes.NewReader(data)},
|
||||
4096, 100)
|
||||
}
|
||||
|
||||
// Unaligned chunks and offsets must still be carved from a device.
|
||||
func TestCarveFromAlignedDevice(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(6))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
image := noise(rng, 3001)
|
||||
unaligned := len(image)
|
||||
image = append(image, chunk...)
|
||||
image = append(image, noise(rng, 4096-(len(image)%4096))...)
|
||||
aligned := len(image)
|
||||
image = append(image, chunk...)
|
||||
image = append(image, noise(rng, carveBlockSize+123)...)
|
||||
last := len(image)
|
||||
image = append(image, chunk[:0x3000]...)
|
||||
|
||||
scope := vql_subsystem.MakeScope()
|
||||
|
||||
for _, tc := range []struct {
|
||||
start, end int64
|
||||
expected []int64
|
||||
}{
|
||||
{0, 0, []int64{int64(unaligned), int64(aligned), int64(last)}},
|
||||
{int64(unaligned), 0, []int64{int64(unaligned), int64(aligned), int64(last)}},
|
||||
{int64(unaligned) + 1, int64(last) + 777,
|
||||
[]int64{int64(aligned), int64(last)}},
|
||||
{0, int64(aligned) + 5, []int64{int64(unaligned)}},
|
||||
} {
|
||||
stats := &carveStats{}
|
||||
var offsets []int64
|
||||
var chunks []*carvedChunk
|
||||
reader, err := NewDeviceReader(image)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for c := range carveChunks(context.Background(), scope,
|
||||
reader,
|
||||
tc.start, tc.end, stats) {
|
||||
offsets = append(offsets, c.Offset)
|
||||
chunks = append(chunks, c)
|
||||
}
|
||||
|
||||
assert.NoError(t, stats.StopError, "range %v-%v", tc.start, tc.end)
|
||||
assert.Equal(t, tc.expected, offsets, "range %v-%v", tc.start, tc.end)
|
||||
|
||||
for _, c := range chunks {
|
||||
assert.True(t, c.HeaderChecksumValid, "chunk %v", c.Offset)
|
||||
assert.Equal(t, c.Offset == int64(last), c.Truncated, "chunk %v", c.Offset)
|
||||
assert.Equal(t, chunk[:0x3000], c.Data[:0x3000], "chunk %v", c.Offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlignedReadAt(t *testing.T) {
|
||||
data := noise(rand.New(rand.NewSource(7)), 10000)
|
||||
reader, err := NewDeviceReader(data)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, tc := range []struct {
|
||||
offset int64
|
||||
length int
|
||||
n int
|
||||
eof bool
|
||||
}{
|
||||
{0, 4096, 4096, false},
|
||||
{1, 10, 10, false},
|
||||
{4095, 2, 2, false},
|
||||
{9990, 10, 10, false},
|
||||
{9990, 20, 10, true},
|
||||
{10000, 10, 0, true},
|
||||
{20000, 10, 0, true},
|
||||
} {
|
||||
buf := make([]byte, tc.length)
|
||||
n, err := reader.ReadAt(buf, tc.offset)
|
||||
assert.Equal(t, tc.n, n, "offset %v", tc.offset)
|
||||
assert.Equal(t, tc.eof, errors.Is(err, io.EOF), "offset %v err %v", tc.offset, err)
|
||||
if n > 0 {
|
||||
assert.Equal(t, data[tc.offset:tc.offset+int64(n)], buf[:n], "offset %v", tc.offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compress a block of up to 4kb with LZNT1 the way NTFS does.
|
||||
func lznt1CompressBlock(in []byte) []byte {
|
||||
out := []byte{0, 0}
|
||||
pos := 0
|
||||
for pos < len(in) {
|
||||
flag_idx := len(out)
|
||||
out = append(out, 0)
|
||||
|
||||
for bit := 0; bit < 8 && pos < len(in); bit++ {
|
||||
// The split between offset and length bits depends on
|
||||
// the position in the block.
|
||||
length_bits := 12
|
||||
for p := pos - 1; p >= 0x10; p >>= 1 {
|
||||
length_bits--
|
||||
}
|
||||
max_offset := 1 << (16 - length_bits)
|
||||
max_length := (1 << length_bits) - 1 + 3
|
||||
|
||||
best_len, best_off := 0, 0
|
||||
for off := 1; off <= pos && off <= max_offset; off++ {
|
||||
l := 0
|
||||
for l < max_length && pos+l < len(in) && in[pos+l-off] == in[pos+l] {
|
||||
l++
|
||||
}
|
||||
if l > best_len {
|
||||
best_len, best_off = l, off
|
||||
}
|
||||
if l == max_length {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if best_len >= 3 {
|
||||
token := uint16(best_off-1)<<length_bits | uint16(best_len-3)
|
||||
out = binary.LittleEndian.AppendUint16(out, token)
|
||||
out[flag_idx] |= 1 << bit
|
||||
pos += best_len
|
||||
} else {
|
||||
out = append(out, in[pos])
|
||||
pos++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store incompressible blocks as is.
|
||||
if len(out)-2 >= lznt1BlockSize {
|
||||
out = binary.LittleEndian.AppendUint16(nil, 0x3000|uint16(len(in)-1))
|
||||
return append(out, in...)
|
||||
}
|
||||
binary.LittleEndian.PutUint16(out, 0xb000|uint16(len(out)-3))
|
||||
return out
|
||||
}
|
||||
|
||||
// Lay out a file on disk the way NTFS compression does: 64kb units
|
||||
// each starting on a new 4kb cluster. The slack at the end of a unit
|
||||
// holds stale data. Units listed in raw_units are stored
|
||||
// uncompressed.
|
||||
func ntfsCompress(rng *rand.Rand, file []byte, raw_units ...int) []byte {
|
||||
var result []byte
|
||||
for unit_idx := 0; unit_idx*lznt1UnitSize < len(file); unit_idx++ {
|
||||
unit := file[unit_idx*lznt1UnitSize:]
|
||||
if len(unit) > lznt1UnitSize {
|
||||
unit = unit[:lznt1UnitSize]
|
||||
}
|
||||
|
||||
var compressed []byte
|
||||
for i := 0; i < len(unit); i += lznt1BlockSize {
|
||||
end := i + lznt1BlockSize
|
||||
if end > len(unit) {
|
||||
end = len(unit)
|
||||
}
|
||||
compressed = append(compressed, lznt1CompressBlock(unit[i:end])...)
|
||||
}
|
||||
|
||||
for _, raw := range raw_units {
|
||||
if raw == unit_idx {
|
||||
compressed = unit
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, compressed...)
|
||||
if rem := len(result) % lznt1BlockSize; rem != 0 {
|
||||
result = append(result, noise(rng, lznt1BlockSize-rem)...)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestLZNT1Compressor(t *testing.T) {
|
||||
data, err := os.ReadFile(testEvtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for i := 0; i < len(data); i += lznt1BlockSize {
|
||||
block := data[i : i+lznt1BlockSize]
|
||||
decompressed, err := ntfs.LZNT1Decompress(lznt1CompressBlock(block))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, block, decompressed, "block at %#x", i)
|
||||
}
|
||||
}
|
||||
|
||||
// The event logs are NTFS compressed by default so carving a raw
|
||||
// volume needs to decompress chunks.
|
||||
func TestCarveCompressedChunk(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(8))
|
||||
file, err := os.ReadFile(testEvtx)
|
||||
assert.NoError(t, err)
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
for _, raw_units := range [][]int{nil, {1}} {
|
||||
disk := noise(rng, 7*lznt1BlockSize)
|
||||
disk = append(disk, ntfsCompress(rng, file, raw_units...)...)
|
||||
disk = append(disk, noise(rng, 3*lznt1BlockSize)...)
|
||||
|
||||
// The plain chunk header is not visible on disk.
|
||||
assert.False(t, bytes.Contains(disk, chunk[:evtx.EVTX_CHUNK_HEADER_SIZE]))
|
||||
|
||||
chunks, stats := carveAll(t, disk, 0, 0)
|
||||
assert.Equal(t, int64(1), stats.Chunks, "raw units %v", raw_units)
|
||||
if len(chunks) != 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
c := chunks[0]
|
||||
assert.True(t, c.Compressed)
|
||||
assert.True(t, c.HeaderChecksumValid)
|
||||
assert.True(t, c.DataChecksumValid)
|
||||
assert.False(t, c.Truncated)
|
||||
assert.Equal(t, chunk, c.Data, "raw units %v", raw_units)
|
||||
|
||||
records, err := parseCarvedChunk(c)
|
||||
assert.NoError(t, err)
|
||||
found := false
|
||||
for _, record := range records {
|
||||
row := makeCarvedRow(c, record, nil)
|
||||
value, _ := ordereddict.GetAny(row, "System.EventID.Value")
|
||||
event_id, _ := utils.ToInt64(value)
|
||||
found = found || event_id == 104
|
||||
|
||||
chunk_info_any, pres := row.Get("ChunkInfo")
|
||||
assert.True(t, pres)
|
||||
compressed, _ := chunk_info_any.(*ordereddict.Dict).Get("Compressed")
|
||||
assert.Equal(t, true, compressed)
|
||||
}
|
||||
assert.True(t, found)
|
||||
}
|
||||
}
|
||||
|
||||
// A compressed chunk cut off by the end of the image is still
|
||||
// recovered as far as possible.
|
||||
func TestCarveTruncatedCompressedChunk(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(9))
|
||||
file, err := os.ReadFile(testEvtx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
disk := noise(rng, lznt1BlockSize)
|
||||
compressed := ntfsCompress(rng, file)
|
||||
disk = append(disk, compressed[:len(compressed)/3]...)
|
||||
|
||||
chunks, _ := carveAll(t, disk, 0, 0)
|
||||
assert.Equal(t, 1, len(chunks))
|
||||
if len(chunks) == 1 {
|
||||
assert.True(t, chunks[0].Compressed)
|
||||
assert.True(t, chunks[0].Truncated)
|
||||
assert.True(t, chunks[0].HeaderChecksumValid)
|
||||
}
|
||||
}
|
||||
|
||||
// A compressed block whose last token is a back reference cut short
|
||||
// by the end of the block. This crashed LZNT1Decompress on a real
|
||||
// volume.
|
||||
func truncatedReferenceBlock() []byte {
|
||||
block := []byte{0x0a, 0xb0, 0x00}
|
||||
block = append(block, evtx.EVTX_CHUNK_HEADER_MAGIC...)
|
||||
return append(block, 0x01, 0x41)
|
||||
}
|
||||
|
||||
func TestCarveTruncatedLZNT1Reference(t *testing.T) {
|
||||
_, err := lznt1Decompress(truncatedReferenceBlock())
|
||||
assert.Error(t, err)
|
||||
|
||||
rng := rand.New(rand.NewSource(10))
|
||||
disk := noise(rng, 5000)
|
||||
disk = append(disk, truncatedReferenceBlock()...)
|
||||
disk = append(disk, noise(rng, 5000)...)
|
||||
|
||||
chunks, stats := carveAll(t, disk, 0, 0)
|
||||
assert.Equal(t, 0, len(chunks))
|
||||
assert.Equal(t, int64(1), stats.Candidates)
|
||||
}
|
||||
|
||||
// Carved data is attacker controlled - run with
|
||||
// go test -fuzz FuzzCarveChunk ./vql/parsers/event_logs/
|
||||
func FuzzCarveChunk(f *testing.F) {
|
||||
scope := vql_subsystem.MakeScope()
|
||||
|
||||
data, err := os.ReadFile(testEvtx)
|
||||
if err == nil && len(data) >= 0x1000+evtx.EVTX_CHUNK_SIZE {
|
||||
f.Add(data[0x1000 : 0x1000+evtx.EVTX_CHUNK_SIZE])
|
||||
}
|
||||
f.Add([]byte(evtx.EVTX_CHUNK_HEADER_MAGIC))
|
||||
f.Add(truncatedReferenceBlock())
|
||||
if err == nil {
|
||||
f.Add(ntfsCompress(rand.New(rand.NewSource(1)), data))
|
||||
}
|
||||
|
||||
f.Fuzz(func(t *testing.T, image []byte) {
|
||||
stats := &carveStats{}
|
||||
for c := range carveChunks(context.Background(), scope,
|
||||
bytes.NewReader(image), 0, 0, stats) {
|
||||
records, _ := parseCarvedChunk(c)
|
||||
for _, r := range records {
|
||||
makeCarvedRow(c, r, nil)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Random data with forced signatures must never crash the carver.
|
||||
func TestCarveGarbage(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(4))
|
||||
chunk := getTestChunk(t)
|
||||
|
||||
for i := 0; i < 50; i++ {
|
||||
// Keep a valid looking header but scramble the records.
|
||||
data := append([]byte{}, chunk...)
|
||||
rng.Read(data[evtx.EVTX_CHUNK_HEADER_SIZE:])
|
||||
|
||||
chunks, _ := carveAll(t, data, 0, 0)
|
||||
for _, c := range chunks {
|
||||
records, _ := parseCarvedChunk(c)
|
||||
for _, r := range records {
|
||||
makeCarvedRow(c, r, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,37 @@ type _ParseEvtxPluginArgs struct {
|
||||
Workers int64 `vfilter:"optional,field=workers,doc=If specified we use this many workers to parse the file in parallel (default 1)."`
|
||||
}
|
||||
|
||||
// Get a resolver to expand event messages. The resolver is closed
|
||||
// when the root scope is destroyed.
|
||||
func getMessageResolver(
|
||||
scope vfilter.Scope, database string) (evtx.MessageResolver, error) {
|
||||
var resolver evtx.MessageResolver
|
||||
var err error
|
||||
|
||||
if database != "" {
|
||||
resolver, err = evtx.NewDBResolver(database)
|
||||
} else {
|
||||
// If the plugin did not specify a database, use the local
|
||||
// resolver - On windows this will search DLLs for the messages.
|
||||
resolver, err = evtx.GetNativeResolver(evtx.MessageResolverOpts{
|
||||
LangPreferenceRegeExp: vql_subsystem.GetStringFromRow(
|
||||
scope, scope, constants.EVTX_PREFERRED_LANG),
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Close the db when we are done.
|
||||
err = vql_subsystem.GetRootScope(scope).AddDestructor(resolver.Close)
|
||||
if err != nil {
|
||||
resolver.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resolver, nil
|
||||
}
|
||||
|
||||
type _ParseEvtxPlugin struct{}
|
||||
|
||||
func (self _ParseEvtxPlugin) Call(
|
||||
@@ -58,31 +89,12 @@ func (self _ParseEvtxPlugin) Call(
|
||||
return
|
||||
}
|
||||
|
||||
var resolver evtx.MessageResolver
|
||||
if arg.Database != "" {
|
||||
resolver, err = evtx.NewDBResolver(arg.Database)
|
||||
} else {
|
||||
// If the plugin did not specify a database, use the local
|
||||
// resolver - On windows this will search DLLs for the messages.
|
||||
resolver, err = evtx.GetNativeResolver(evtx.MessageResolverOpts{
|
||||
LangPreferenceRegeExp: vql_subsystem.GetStringFromRow(
|
||||
scope, scope, constants.EVTX_PREFERRED_LANG),
|
||||
})
|
||||
}
|
||||
|
||||
resolver, err := getMessageResolver(scope, arg.Database)
|
||||
if err != nil {
|
||||
scope.Log("parse_evtx: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Close the db when we are done.
|
||||
err = vql_subsystem.GetRootScope(scope).AddDestructor(resolver.Close)
|
||||
if err != nil {
|
||||
resolver.Close()
|
||||
scope.Log("parse_evtx: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
for _, filename := range arg.Filenames {
|
||||
func() {
|
||||
defer utils.RecoverVQL(scope)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
/*
|
||||
Velociraptor - Dig Deeper
|
||||
Copyright (C) 2019-2025 Rapid7 Inc.
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package event_logs
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"www.velocidex.com/golang/evtx"
|
||||
)
|
||||
|
||||
const (
|
||||
// Offsets into the EVTX file header. See
|
||||
// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc
|
||||
fileHeaderSize = 0x1000
|
||||
fileHeaderUsedSize = 0x80
|
||||
fileHeaderChecksumStart = 0x78
|
||||
|
||||
// The file header stores the number of chunks in 16 bits.
|
||||
maxChunksPerFile = 0xffff
|
||||
|
||||
chunkDataChecksumOffset = 0x34
|
||||
|
||||
// An event record starts with "**\0\0", its size, its record
|
||||
// id and its timestamp.
|
||||
recordMagic = 0x00002a2a
|
||||
recordHeaderSize = 0x18
|
||||
)
|
||||
|
||||
// Writes carved chunks into a new EVTX file so the recovered events
|
||||
// can be opened with Event Viewer and other EVTX tools. Each chunk
|
||||
// carries its own string and template tables so chunks from
|
||||
// different logs can be combined in one file.
|
||||
type evtxWriter struct {
|
||||
fd io.WriterAt
|
||||
|
||||
chunks uint64
|
||||
next_record uint64
|
||||
|
||||
// Windows only reads on while the record numbers in the chunk
|
||||
// headers continue from one chunk to the next, but carved
|
||||
// chunks come from many logs. The chunk headers are renumbered
|
||||
// - the event record ids shown for each event are not changed.
|
||||
next_number uint64
|
||||
|
||||
// The same chunk is often found several times (e.g. the live
|
||||
// log and old copies in unallocated space). Write it once.
|
||||
seen map[[sha256.Size]byte]bool
|
||||
}
|
||||
|
||||
func newEvtxWriter(fd io.WriterAt) (*evtxWriter, error) {
|
||||
self := &evtxWriter{
|
||||
fd: fd,
|
||||
seen: make(map[[sha256.Size]byte]bool),
|
||||
next_number: 1,
|
||||
}
|
||||
|
||||
// A valid header even if no chunks are written.
|
||||
return self, self.writeHeader()
|
||||
}
|
||||
|
||||
// Add the chunk to the file. Returns false if the chunk was skipped
|
||||
// because it was already written or the file is full.
|
||||
func (self *evtxWriter) WriteChunk(chunk *carvedChunk) (bool, error) {
|
||||
key := sha256.Sum256(chunk.Data)
|
||||
if self.seen[key] || self.chunks >= maxChunksPerFile {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
data := repairChunk(chunk)
|
||||
if data == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
first := binary.LittleEndian.Uint64(data[0x08:])
|
||||
last := binary.LittleEndian.Uint64(data[0x10:])
|
||||
binary.LittleEndian.PutUint64(data[0x08:], self.next_number)
|
||||
binary.LittleEndian.PutUint64(data[0x10:], self.next_number+last-first)
|
||||
setChunkHeaderChecksum(data)
|
||||
|
||||
offset := int64(fileHeaderSize) + int64(self.chunks)*evtx.EVTX_CHUNK_SIZE
|
||||
_, err := self.fd.WriteAt(data, offset)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// When the output file is on the disk being carved, the scan
|
||||
// may find the chunks we wrote. Do not write them again.
|
||||
self.seen[key] = true
|
||||
self.seen[sha256.Sum256(data)] = true
|
||||
self.chunks++
|
||||
self.next_number += last - first + 1
|
||||
if chunk.Header.LastEventRecID >= self.next_record {
|
||||
self.next_record = chunk.Header.LastEventRecID + 1
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Windows refuses to open a file if any chunk has a bad checksum. A
|
||||
// damaged chunk is cut down to its leading intact records and its
|
||||
// header and data checksum are rewritten to match. Returns a copy of
|
||||
// the chunk, or nil if no record survives. The caller sets the header
|
||||
// checksum.
|
||||
func repairChunk(chunk *carvedChunk) []byte {
|
||||
data := append([]byte{}, chunk.Data...)
|
||||
if chunk.HeaderChecksumValid && chunk.DataChecksumValid {
|
||||
return data
|
||||
}
|
||||
|
||||
free_space := int(binary.LittleEndian.Uint32(data[chunkFreeSpaceOffset:]))
|
||||
|
||||
// Records follow each other from the end of the chunk header.
|
||||
// Each starts with a signature and its size, and ends with a
|
||||
// copy of the size.
|
||||
offset := evtx.EVTX_CHUNK_HEADER_SIZE
|
||||
last_offset := 0
|
||||
var last_id uint64
|
||||
count := uint64(0)
|
||||
for offset+recordHeaderSize <= free_space {
|
||||
if binary.LittleEndian.Uint32(data[offset:]) != recordMagic {
|
||||
break
|
||||
}
|
||||
size := int(binary.LittleEndian.Uint32(data[offset+4:]))
|
||||
if size < recordHeaderSize+4 || offset+size > free_space ||
|
||||
int(binary.LittleEndian.Uint32(data[offset+size-4:])) != size {
|
||||
break
|
||||
}
|
||||
last_offset = offset
|
||||
last_id = binary.LittleEndian.Uint64(data[offset+8:])
|
||||
count++
|
||||
offset += size
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop everything after the last intact record.
|
||||
for i := offset; i < len(data); i++ {
|
||||
data[i] = 0
|
||||
}
|
||||
|
||||
first_number := binary.LittleEndian.Uint64(data[0x08:])
|
||||
binary.LittleEndian.PutUint64(data[0x10:], first_number+count-1)
|
||||
binary.LittleEndian.PutUint64(data[0x20:], last_id)
|
||||
binary.LittleEndian.PutUint32(data[0x2c:], uint32(last_offset))
|
||||
binary.LittleEndian.PutUint32(data[chunkFreeSpaceOffset:], uint32(offset))
|
||||
binary.LittleEndian.PutUint32(data[chunkDataChecksumOffset:],
|
||||
crc32.ChecksumIEEE(data[evtx.EVTX_CHUNK_HEADER_SIZE:offset]))
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// The header checksum covers the first 120 bytes and the
|
||||
// string/template tables, skipping the flags and the checksum.
|
||||
func setChunkHeaderChecksum(data []byte) {
|
||||
crc := crc32.NewIEEE()
|
||||
_, _ = crc.Write(data[:chunkHeaderChecksumStart])
|
||||
_, _ = crc.Write(data[chunkHeaderChecksumEnd:evtx.EVTX_CHUNK_HEADER_SIZE])
|
||||
binary.LittleEndian.PutUint32(data[chunkHeaderChecksumStart+4:], crc.Sum32())
|
||||
}
|
||||
|
||||
func (self *evtxWriter) Chunks() uint64 {
|
||||
return self.chunks
|
||||
}
|
||||
|
||||
// Write the final header. Must be called after the last chunk.
|
||||
func (self *evtxWriter) Close() error {
|
||||
return self.writeHeader()
|
||||
}
|
||||
|
||||
func (self *evtxWriter) writeHeader() error {
|
||||
header := make([]byte, fileHeaderSize)
|
||||
copy(header, evtx.EVTX_HEADER_MAGIC)
|
||||
|
||||
last_chunk := uint64(0)
|
||||
if self.chunks > 0 {
|
||||
last_chunk = self.chunks - 1
|
||||
}
|
||||
|
||||
binary.LittleEndian.PutUint64(header[0x08:], 0) // First chunk number
|
||||
binary.LittleEndian.PutUint64(header[0x10:], last_chunk)
|
||||
binary.LittleEndian.PutUint64(header[0x18:], self.next_record)
|
||||
binary.LittleEndian.PutUint32(header[0x20:], fileHeaderUsedSize)
|
||||
binary.LittleEndian.PutUint16(header[0x24:], 2) // Minor version
|
||||
binary.LittleEndian.PutUint16(header[0x26:], 3) // Major version
|
||||
binary.LittleEndian.PutUint16(header[0x28:], fileHeaderSize)
|
||||
binary.LittleEndian.PutUint16(header[0x2a:], uint16(self.chunks))
|
||||
|
||||
// Flags are left clear: the file is not dirty and not full.
|
||||
binary.LittleEndian.PutUint32(header[fileHeaderChecksumStart+4:],
|
||||
crc32.ChecksumIEEE(header[:fileHeaderChecksumStart]))
|
||||
|
||||
_, err := self.fd.WriteAt(header, 0)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package event_logs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"www.velocidex.com/golang/evtx"
|
||||
"www.velocidex.com/golang/velociraptor/vtesting/assert"
|
||||
)
|
||||
|
||||
// An in memory io.WriterAt.
|
||||
type memWriter struct {
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func (self *memWriter) WriteAt(p []byte, off int64) (int, error) {
|
||||
if end := int(off) + len(p); end > len(self.buf) {
|
||||
self.buf = append(self.buf, make([]byte, end-len(self.buf))...)
|
||||
}
|
||||
return copy(self.buf[off:], p), nil
|
||||
}
|
||||
|
||||
func mustCheckChunk(t *testing.T, data []byte) *carvedChunk {
|
||||
chunk, ok := checkChunk(0, data, len(data))
|
||||
assert.True(t, ok)
|
||||
return chunk
|
||||
}
|
||||
|
||||
// Chunks of the second log in the testdata.
|
||||
func otherLogChunks(t *testing.T) []*carvedChunk {
|
||||
data, err := os.ReadFile("../../../artifacts/testdata/files/RDPAuth_Security.evtx")
|
||||
assert.NoError(t, err)
|
||||
|
||||
var result []*carvedChunk
|
||||
for off := 0x1000; off+evtx.EVTX_CHUNK_SIZE <= len(data); off += evtx.EVTX_CHUNK_SIZE {
|
||||
chunk, ok := checkChunk(int64(off), data[off:off+evtx.EVTX_CHUNK_SIZE],
|
||||
evtx.EVTX_CHUNK_SIZE)
|
||||
if ok {
|
||||
result = append(result, chunk)
|
||||
}
|
||||
}
|
||||
assert.True(t, len(result) > 2)
|
||||
return result
|
||||
}
|
||||
|
||||
func recordIds(t *testing.T, chunk *carvedChunk) []uint64 {
|
||||
records, err := parseCarvedChunk(chunk)
|
||||
assert.NoError(t, err)
|
||||
var result []uint64
|
||||
for _, r := range records {
|
||||
result = append(result, r.Header.RecordID)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestEvtxWriter(t *testing.T) {
|
||||
intact := mustCheckChunk(t, getTestChunk(t))
|
||||
others := otherLogChunks(t)
|
||||
|
||||
// The records after the middle of the used space are overwritten.
|
||||
damaged_data := append([]byte{}, others[1].Data...)
|
||||
free_space := int(binary.LittleEndian.Uint32(damaged_data[chunkFreeSpaceOffset:]))
|
||||
mid := evtx.EVTX_CHUNK_HEADER_SIZE + (free_space-evtx.EVTX_CHUNK_HEADER_SIZE)/2
|
||||
copy(damaged_data[mid:free_space], noise(newRng(), free_space-mid))
|
||||
damaged := mustCheckChunk(t, damaged_data)
|
||||
assert.False(t, damaged.DataChecksumValid)
|
||||
|
||||
// Nothing survives when the first record is destroyed.
|
||||
hopeless_data := append([]byte{}, others[2].Data...)
|
||||
copy(hopeless_data[evtx.EVTX_CHUNK_HEADER_SIZE:], "XXXX")
|
||||
hopeless := mustCheckChunk(t, hopeless_data)
|
||||
|
||||
out := &memWriter{}
|
||||
writer, err := newEvtxWriter(out)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Chunks from different logs, a duplicate and damaged chunks.
|
||||
for _, c := range []*carvedChunk{others[0], intact, others[0], damaged, hopeless} {
|
||||
_, err := writer.WriteChunk(c)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
assert.NoError(t, writer.Close())
|
||||
assert.Equal(t, uint64(3), writer.Chunks())
|
||||
|
||||
// The library reads the file back.
|
||||
chunks, err := evtx.GetChunks(bytes.NewReader(out.buf))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(chunks))
|
||||
|
||||
// Every written chunk has valid checksums and the record
|
||||
// numbers in the chunk headers follow on from each other.
|
||||
next := uint64(1)
|
||||
var written []*carvedChunk
|
||||
for i := 0; i < 3; i++ {
|
||||
off := fileHeaderSize + i*evtx.EVTX_CHUNK_SIZE
|
||||
c := mustCheckChunk(t, out.buf[off:off+evtx.EVTX_CHUNK_SIZE])
|
||||
assert.True(t, c.HeaderChecksumValid, "chunk %v", i)
|
||||
assert.True(t, c.DataChecksumValid, "chunk %v", i)
|
||||
assert.Equal(t, next, c.Header.FirstEventRecNumber, "chunk %v", i)
|
||||
next = c.Header.LastEventRecNumber + 1
|
||||
written = append(written, c)
|
||||
}
|
||||
|
||||
// Event record ids are kept as they were.
|
||||
assert.Equal(t, recordIds(t, others[0]), recordIds(t, written[0]))
|
||||
assert.Equal(t, recordIds(t, intact), recordIds(t, written[1]))
|
||||
|
||||
// Only the intact leading records of the damaged chunk are kept.
|
||||
repaired := recordIds(t, written[2])
|
||||
original := recordIds(t, others[1])
|
||||
assert.True(t, len(repaired) > 0)
|
||||
assert.True(t, len(repaired) < len(original))
|
||||
assert.Equal(t, original[:len(repaired)], repaired)
|
||||
|
||||
// The file header describes the chunks.
|
||||
header := out.buf[:fileHeaderSize]
|
||||
assert.Equal(t, evtx.EVTX_HEADER_MAGIC, string(header[:8]))
|
||||
assert.Equal(t, uint16(3), binary.LittleEndian.Uint16(header[0x2a:]))
|
||||
assert.Equal(t, uint64(2), binary.LittleEndian.Uint64(header[0x10:]))
|
||||
}
|
||||
|
||||
// The output file may be on the disk being carved, so chunks we
|
||||
// wrote can be found again by the scan.
|
||||
func TestEvtxWriterSkipsOwnOutput(t *testing.T) {
|
||||
others := otherLogChunks(t)
|
||||
|
||||
out := &memWriter{}
|
||||
writer, err := newEvtxWriter(out)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Written out of order so the second chunk is renumbered.
|
||||
for _, c := range []*carvedChunk{others[1], others[0]} {
|
||||
ok, err := writer.WriteChunk(c)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
}
|
||||
|
||||
// The second chunk was renumbered when written, so it differs
|
||||
// from the original.
|
||||
off := fileHeaderSize + evtx.EVTX_CHUNK_SIZE
|
||||
found_again := mustCheckChunk(t, append([]byte{}, out.buf[off:off+evtx.EVTX_CHUNK_SIZE]...))
|
||||
assert.NotEqual(t, others[0].Data, found_again.Data)
|
||||
|
||||
ok, err := writer.WriteChunk(found_again)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, uint64(2), writer.Chunks())
|
||||
}
|
||||
|
||||
func TestEvtxWriterEmpty(t *testing.T) {
|
||||
out := &memWriter{}
|
||||
writer, err := newEvtxWriter(out)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, writer.Close())
|
||||
assert.Equal(t, fileHeaderSize, len(out.buf))
|
||||
assert.Equal(t, evtx.EVTX_HEADER_MAGIC, string(out.buf[:8]))
|
||||
}
|
||||
|
||||
func newRng() *rand.Rand {
|
||||
return rand.New(rand.NewSource(11))
|
||||
}
|
||||
+24
-1
@@ -207,6 +207,8 @@ func (self *AccessorReader) _Close() error {
|
||||
self.cancel = nil
|
||||
|
||||
reader := self.reader
|
||||
paged_reader := self.paged_reader
|
||||
|
||||
self.reader = nil
|
||||
self.paged_reader = nil
|
||||
self.last_opened = time.Time{}
|
||||
@@ -222,6 +224,14 @@ func (self *AccessorReader) _Close() error {
|
||||
reader.Close()
|
||||
}
|
||||
|
||||
if paged_reader != nil {
|
||||
// Report paged_reader efficiency
|
||||
/*
|
||||
stats := paged_reader.Stats()
|
||||
fmt.Printf("%s: %#v\n", self.File.String(), stats)
|
||||
*/
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -250,8 +260,21 @@ func (self *AccessorReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
lru_size = 100
|
||||
}
|
||||
|
||||
// Pagesize is a sliding scale to maximize efficiency over
|
||||
// memory use. If the lru is large, we use small pages, but if
|
||||
// the lru is small, we use larger pages.
|
||||
page_size := int64(1024 * 8)
|
||||
if lru_size < 100 {
|
||||
// Target < 100Mb
|
||||
page_size = 1024 * 1024
|
||||
|
||||
} else if lru_size < 10 {
|
||||
// Target < 100Mb
|
||||
page_size = 10 * 1024 * 1024
|
||||
}
|
||||
|
||||
paged_reader, err := ntfs.NewPagedReader(
|
||||
utils.MakeReaderAtter(reader), 1024*8, lru_size)
|
||||
utils.MakeReaderAtter(reader), page_size, lru_size)
|
||||
if err != nil {
|
||||
reader.Close()
|
||||
self.mu.Unlock()
|
||||
|
||||
Reference in New Issue
Block a user