Feature/carve evtx, (#5063)

Problem: When Windows event logs are cleared or deleted,
Windows.EventLogs.Cleared can show that a log was cleared, but the
events themselves are lost because parse_evtx() only reads intact log
files. The old data usually stays on disk, in unallocated space, the
pagefile or memory, until it is overwritten.

Solution: 
This PR adds a new carve_evtx() VQL plugin. It scans any file or device
for EVTX chunk signatures at any offset and parses each 64KB chunk on
its own.
- Rows have the same structure as parse_evtx(), plus ChunkOffset,
ChunkHeaderValid, ChunkDataValid, ChunkTruncated and ChunkCompressed.
- NTFS-compressed chunks (the Windows default for event logs) are
decompressed.
- Damaged chunks are flagged, not dropped.
- With output=, the recovered events are also written to a new .evtx
file that Event Viewer can open.

Example:
SELECT * FROM carve_evtx(filename='''\\.\C:''', accessor="raw_file")
WHERE System.Channel = "Application"

Known limitation
Old records in chunk slack are not recovered yet. This could be a
follow-up.

---------

Co-authored-by: Mike Cohen <[email protected]>
This commit is contained in:
Abdelghafour Bouhdyd
2026-10-07 21:34:46 +10:00
committed by GitHub
co-authored by Mike Cohen
parent 4b35b1120f
commit fa9a5b4cd1
23 changed files with 2569 additions and 50 deletions
+3 -1
View File
@@ -2,6 +2,7 @@ package file_store_test
import (
"errors"
"io"
"testing"
"github.com/stretchr/testify/suite"
@@ -144,7 +145,8 @@ func (self *FSAccessorTest) TestSparseFiles() {
buf := make([]byte, 100)
n, err := fd.Read(buf)
assert.NoError(self.T(), err)
assert.Error(self.T(), err)
assert.True(self.T(), errors.Is(err, io.EOF))
assert.Equal(self.T(), string(buf[:n]), "Hello\x00\x00\x00\x00\x00World")
+16
View File
@@ -0,0 +1,16 @@
//go:build !velomain
// +build !velomain
package assets
import "www.velocidex.com/golang/velociraptor/utils"
var (
FileDocsReferencesVqlYaml = []byte("")
Inventory = map[string][]byte{}
)
// ReadFile is adapTed from ioutil
func ReadFile(path string) ([]byte, error) {
return nil, utils.NotImplementedError
}
@@ -0,0 +1,93 @@
name: Windows.Carving.Evtx
description: |
Carve EVTX chunks from a file or device and parse their event records.
When event logs are cleared or deleted, the old data often remains
on disk until it is overwritten. This plugin scans any file or
device (for example a raw volume, unallocated space, the pagefile
or a memory image) for the EVTX chunk signature at any offset and
parses each chunk it finds. Each 64kb chunk carries its own string
and template tables, so its records can be parsed without the rest
of the log file.
required_permissions:
- FILESYSTEM_WRITE
parameters:
- name: Device
default: "C:"
description: The NTFS drive to carve
- name: Accessor
description: The accessor to use.
default: "ntfs"
- name: IocRegex
type: regex
description: "IOC regex - will match anywhere in the event"
default:
- name: ChannelRegex
description: "Channel regex to enable filtering on path"
default: .
- name: ProviderRegex
description: "Provider regex to enable filtering on provider"
default: .
type: regex
- name: IdRegex
default: .
type: regex
- name: DateAfter
type: timestamp
description: "search for events after this date. YYYY-MM-DDTmm:hh:ssZ"
- name: DateBefore
type: timestamp
description: "search for events before this date. YYYY-MM-DDTmm:hh:ssZ"
- name: ReconstructFile
type: bool
description: Try to reconstruct an evtx file from the carved chunks.
imports:
- Windows.Sys.AllUsers
precondition:
SELECT OS From info() where OS = 'windows'
sources:
- query: |
LET DateAfterTime <= DateAfter || "1600-01-01"
LET DateBeforeTime <= DateBefore || "2200-01-01"
LET S = scope()
LET Temp <= tempfile(remove_last=TRUE)
SELECT timestamp(epoch=int(int=System.TimeCreated.SystemTime)) AS EventTime,
System.Computer as Computer,
System.Channel as Channel,
System.Provider.Name as Provider,
System.EventID.Value as EventID,
System.EventRecordID as EventRecordID,
System.Security.UserID as UserSID,
LookupSIDCache(SID=System.Security.UserID || "") AS Username,
S.EventData AS EventData,
S.UserData AS UserData,
S.Message AS Message,
ChunkInfo
FROM carve_evtx(filename=Device,
accessor=Accessor,
output=if(condition=ReconstructFile, then=Temp)
)
WHERE ( EventData OR UserData OR Message )
AND EventTime < DateBeforeTime
AND EventTime > DateAfterTime
AND Channel =~ ChannelRegex
AND Provider =~ ProviderRegex
AND EventID =~ IdRegex
AND format(format='%v %v %v', args=[EventData, UserData, Message]) =~ IocRegex
- name: UploadRecostruction
query: |
SELECT upload(file=Temp) AS Upload
FROM scope()
WHERE ReconstructFile
column_types:
- name: Upload
type: upload_preview
+40
View File
@@ -0,0 +1,40 @@
Queries:
# Build a fake disk image in memory: unrelated data, then an EVTX
# chunk at an unaligned offset, then more unrelated data. The chunk
# records the System log being cleared (Event ID 104).
- LET Chunk <= read_file(
filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx',
offset=4096, length=65536)
- LET Noise <= read_file(
filename=srcDir + '/artifacts/testdata/files/SAM', length=777)
- LET Image <= Noise + Chunk + Noise
# The chunk is found at offset 777 with valid checksums.
- SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.kDataValid, ChunkInfo.Truncated,
System.EventID.Value AS EventID, System.EventRecordID AS RecordID,
System.Channel AS Channel
FROM carve_evtx(filename=Image, accessor="data")
# Carving the original log file finds the chunk after the file header.
- SELECT ChunkInfo.Offset, System.EventID.Value AS EventID
FROM carve_evtx(
filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx')
# Starting after the chunk finds nothing.
- SELECT * FROM carve_evtx(filename=Image, accessor="data", start_offset=778)
# A damaged chunk header is still parsed but flagged as invalid.
- LET Damaged <= Noise + Chunk[:256] + "XXXX" + Chunk[260:]
- SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.DataValid,
System.EventID.Value AS EventID
FROM carve_evtx(filename=Damaged, accessor="data")
# Recovered chunks can be written to a new EVTX file. The duplicate
# chunk is written once and the damaged chunk is repaired so the
# file can be read by parse_evtx() and Event Viewer.
- LET Out <= tempfile(extension='.evtx')
- SELECT count() AS Rows
FROM carve_evtx(filename=Image + Image + Damaged, accessor="data", output=Out)
GROUP BY 1
- SELECT System.EventID.Value AS EventID, System.EventRecordID AS RecordID
FROM parse_evtx(filename=Out)
+78
View File
@@ -0,0 +1,78 @@
# Build a fake disk image in memory: unrelated data, then an EVTX
# chunk at an unaligned offset, then more unrelated data. The chunk
# records the System log being cleared (Event ID 104).
Query: LET Chunk <= read_file( filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx', offset=4096, length=65536)
Output: []
Query: LET Noise <= read_file( filename=srcDir + '/artifacts/testdata/files/SAM', length=777)
Output: []
Query: LET Image <= Noise + Chunk + Noise
Output: []
# The chunk is found at offset 777 with valid checksums.
Query: SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.kDataValid, ChunkInfo.Truncated, System.EventID.Value AS EventID, System.EventRecordID AS RecordID, System.Channel AS Channel FROM carve_evtx(filename=Image, accessor="data")
Output: [
{
"ChunkInfo.Offset": 777,
"ChunkInfo.HeaderValid": true,
"ChunkInfo.kDataValid": null,
"ChunkInfo.Truncated": false,
"EventID": 104,
"RecordID": 27736,
"Channel": "System"
}
]
# Carving the original log file finds the chunk after the file header.
Query: SELECT ChunkInfo.Offset, System.EventID.Value AS EventID FROM carve_evtx( filename=srcDir + '/artifacts/testdata/files/DE_104_system_log_cleared.evtx')
Output: [
{
"ChunkInfo.Offset": 4096,
"EventID": 104
}
]
# Starting after the chunk finds nothing.
Query: SELECT * FROM carve_evtx(filename=Image, accessor="data", start_offset=778)
Output: []
# A damaged chunk header is still parsed but flagged as invalid.
Query: LET Damaged <= Noise + Chunk[:256] + "XXXX" + Chunk[260:]
Output: []
Query: SELECT ChunkInfo.Offset, ChunkInfo.HeaderValid, ChunkInfo.DataValid, System.EventID.Value AS EventID FROM carve_evtx(filename=Damaged, accessor="data")
Output: [
{
"ChunkInfo.Offset": 777,
"ChunkInfo.HeaderValid": false,
"ChunkInfo.DataValid": true,
"EventID": 104
}
]
# Recovered chunks can be written to a new EVTX file. The duplicate
# chunk is written once and the damaged chunk is repaired so the
# file can be read by parse_evtx() and Event Viewer.
Query: LET Out <= tempfile(extension='.evtx')
Output: []
Query: SELECT count() AS Rows FROM carve_evtx(filename=Image + Image + Damaged, accessor="data", output=Out) GROUP BY 1
Output: [
{
"Rows": 3
}
]
Query: SELECT System.EventID.Value AS EventID, System.EventRecordID AS RecordID FROM parse_evtx(filename=Out)
Output: [
{
"EventID": 104,
"RecordID": 27736
},
{
"EventID": 104,
"RecordID": 27736
}
]
+414 -9
View File
@@ -6,7 +6,34 @@ Output: [
"Artifact": "name: BrokenYaml\nsomeweirdfield: 1\n",
"Permissions": null,
"Errors": [
"(0,0) yaml_error: YAML Error: yaml: unmarshal errors:\n line 2: field someweirdfield not found in type proto.Artifact"
{
"Target": "",
"Name": "yaml_error",
"Message": "YAML Error: %[1]v",
"Args": [
{
"Err": {
"Errors": [
"line 2: field someweirdfield not found in type proto.Artifact"
]
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {},
@@ -16,7 +43,51 @@ Output: [
"Artifact": "name: BrokenQuery\nsources:\n- query: SELECT * FROM nosuchplugin()\n",
"Permissions": null,
"Errors": [
"(0,0) artifact_vql_error: BrokenQuery: query: (1,15) unknown_plugin: Unknown plugin nosuchplugin()"
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"BrokenQuery",
{
"Target": "",
"Name": "unknown_plugin",
"Message": "Unknown %[2]v %[1]v()",
"Args": [
"nosuchplugin",
"plugin"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 14,
"Line": 1,
"Column": 15
},
"EndPos": {
"Filename": "",
"Offset": 28,
"Line": 1,
"Column": 29
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {},
@@ -26,7 +97,31 @@ Output: [
"Artifact": "name: BrokenSubArtifact\nsources:\n- query: SELECT * FROM Artifact.No.Such.Artifact()\n",
"Permissions": null,
"Errors": [
"(0,0) artifact_vql_error: BrokenSubArtifact: query: Unknown artifact reference No.Such.Artifact"
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"BrokenSubArtifact",
{
"Err": {}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {},
@@ -71,11 +166,300 @@ Output: [
"FILESYSTEM_READ"
],
"Errors": [
"(0,0) artifact_vql_error: KWArgs: query: (5,32) required_arg_missing: While calling function alert(), required arg name is not provided",
"(0,0) artifact_vql_error: KWArgs: query: (10,15) kwargs_mixed_call: Calling plugin `switch` with kwargs (**) as well as with arg `{a {10:22 14:2}}`",
"(0,0) artifact_vql_error: KWArgs: query: (11,10) kwargs_mixed_call: Calling function `Foo` with kwargs (**) as well as with arg `{X {11:30 11:33}}`",
"(0,0) artifact_vql_error: KWArgs: query: (12,10) kwargs_mixed_call: Calling function `dict` with kwargs (**) as well as with arg `{A {12:15 12:18}}`",
"(0,0) artifact_vql_error: KWArgs: query: (13,8) kwargs_mixed_call: Calling plugin `Artifact.Generic.Client.Info` with kwargs (**) as well as with arg `{X {13:53 13:56}}`"
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"KWArgs",
{
"Target": "",
"Name": "required_arg_missing",
"Message": "While calling %[2]v %[3]v(), required arg %[1]v is not provided",
"Args": [
"name",
"function",
"alert"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 147,
"Line": 5,
"Column": 32
},
"EndPos": {
"Filename": "",
"Offset": 171,
"Line": 6,
"Column": 1
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
},
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"KWArgs",
{
"Target": "",
"Name": "kwargs_mixed_call",
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
"Args": [
"switch",
{
"Name": "a",
"Pos": {
"Pos": {
"Filename": "",
"Offset": 258,
"Line": 10,
"Column": 22
},
"EndPos": {
"Filename": "",
"Offset": 387,
"Line": 14,
"Column": 2
}
}
},
"plugin"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 251,
"Line": 10,
"Column": 15
},
"EndPos": {
"Filename": "",
"Offset": 402,
"Line": 15,
"Column": 1
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
},
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"KWArgs",
{
"Target": "",
"Name": "kwargs_mixed_call",
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
"Args": [
"Foo",
{
"Name": "X",
"Pos": {
"Pos": {
"Filename": "",
"Offset": 291,
"Line": 11,
"Column": 30
},
"EndPos": {
"Filename": "",
"Offset": 294,
"Line": 11,
"Column": 33
}
}
},
"function"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 271,
"Line": 11,
"Column": 10
},
"EndPos": {
"Filename": "",
"Offset": 295,
"Line": 11,
"Column": 34
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
},
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"KWArgs",
{
"Target": "",
"Name": "kwargs_mixed_call",
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
"Args": [
"dict",
{
"Name": "A",
"Pos": {
"Pos": {
"Filename": "",
"Offset": 311,
"Line": 12,
"Column": 15
},
"EndPos": {
"Filename": "",
"Offset": 314,
"Line": 12,
"Column": 18
}
}
},
"function"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 306,
"Line": 12,
"Column": 10
},
"EndPos": {
"Filename": "",
"Offset": 331,
"Line": 13,
"Column": 3
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
},
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"KWArgs",
{
"Target": "",
"Name": "kwargs_mixed_call",
"Message": "Calling %[3]v `%[1]v` with kwargs (**) as well as with arg `%[2]v`",
"Args": [
"Artifact.Generic.Client.Info",
{
"Name": "X",
"Pos": {
"Pos": {
"Filename": "",
"Offset": 381,
"Line": 13,
"Column": 53
},
"EndPos": {
"Filename": "",
"Offset": 384,
"Line": 13,
"Column": 56
}
}
},
"plugin"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 336,
"Line": 13,
"Column": 8
},
"EndPos": {
"Filename": "",
"Offset": 386,
"Line": 14,
"Column": 1
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": [
{
@@ -299,7 +683,28 @@ Output: [
"Artifact": "name: Generic.Client.Info\nsources:\n- query: SELECT * FROM scope()\n",
"Permissions": null,
"Errors": [
"(0,0) yaml_error: YAML Error: Unable to override built in artifact Generic.Client.Info"
{
"Target": "",
"Name": "yaml_error",
"Message": "YAML Error: %[1]v",
"Args": [
{}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {},
+135 -3
View File
@@ -56,7 +56,51 @@ Output: [
"Artifact": "name: NotCallingRequiredArg\nsources:\n- query: |\n LET Foo(X, Y=1) = X+Y\n SELECT Foo() FROM scope()\n",
"Permissions": null,
"Errors": [
"(0,0) artifact_vql_error: NotCallingRequiredArg: query: (1,9) required_arg_missing: While calling VQL definition Foo(), required arg X is not provided"
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"NotCallingRequiredArg",
{
"Target": "",
"Name": "required_arg_missing",
"Message": "While calling VQL definition %[2]v(), required arg %[1]v is not provided",
"Args": [
"X",
"Foo"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 8,
"Line": 1,
"Column": 9
},
"EndPos": {
"Filename": "",
"Offset": 9,
"Line": 1,
"Column": 10
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {
@@ -78,8 +122,96 @@ Output: [
"Artifact": "name: CallingBadArg\nsources:\n- query: |\n LET Foo(X, Y=1) = X+Y\n SELECT Foo(Z=1) FROM scope()\n",
"Permissions": null,
"Errors": [
"(0,0) artifact_vql_error: CallingBadArg: query: (2,12) invalid_arg: Invalid arg Z for VQL definition Foo",
"(0,0) artifact_vql_error: CallingBadArg: query: (1,9) required_arg_missing: While calling VQL definition Foo(), required arg X is not provided"
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"CallingBadArg",
{
"Target": "",
"Name": "invalid_arg",
"Message": "Invalid arg %[1]v for VQL definition %[2]v",
"Args": [
"Z",
"Foo"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 33,
"Line": 2,
"Column": 12
},
"EndPos": {
"Filename": "",
"Offset": 36,
"Line": 2,
"Column": 15
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
},
{
"Target": "",
"Name": "artifact_vql_error",
"Message": "%[1]v: query: %[2]v",
"Args": [
"CallingBadArg",
{
"Target": "",
"Name": "required_arg_missing",
"Message": "While calling VQL definition %[2]v(), required arg %[1]v is not provided",
"Args": [
"X",
"Foo"
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 8,
"Line": 1,
"Column": 9
},
"EndPos": {
"Filename": "",
"Offset": 9,
"Line": 1,
"Column": 10
}
}
}
],
"Pos": {
"Pos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
},
"EndPos": {
"Filename": "",
"Offset": 0,
"Line": 0,
"Column": 0
}
}
}
],
"Warnings": null,
"Definitions": {
+102
View File
@@ -892,6 +892,107 @@
- linux_amd64_cgo
- windows_386_cgo
- windows_amd64_cgo
- name: carve_evtx
description: |
Carve EVTX chunks from a file or device and parse their event records.
When event logs are cleared or deleted, the old data often remains
on disk until it is overwritten. This plugin scans any file or
device (for example a raw volume, unallocated space, the pagefile
or a memory image) for the EVTX chunk signature at any offset and
parses each chunk it finds. Each 64kb chunk carries its own string
and template tables, so its records can be parsed without the rest
of the log file.
Each row has the same structure as `parse_evtx()` with these
additional columns:
- `ChunkOffset`: The offset of the chunk in the carved file.
- `ChunkHeaderValid`: The chunk header checksum is correct.
- `ChunkDataValid`: The event records checksum is correct.
- `ChunkTruncated`: The file ended before the end of the chunk.
- `ChunkCompressed`: The chunk was found inside an NTFS compressed
file and was decompressed. Windows compresses the event logs by
default, so most chunks carved from a raw volume are compressed.
Carved records may be incomplete or partially overwritten. Records
from chunks with invalid checksums should be treated with caution.
The same chunk may be found several times (for example in a live
log file and in unallocated space).
### Example
```vql
SELECT ChunkOffset, ChunkHeaderValid,
System.TimeCreated.SystemTime AS Timestamp,
System.EventID.Value AS EventID,
System.Channel AS Channel, EventData
FROM carve_evtx(filename="C:", accessor="ntfs")
WHERE EventID IN (1102, 104, 4624, 7045)
```
To carve the pagefile on a running system use the `ntfs` accessor:
```vql
SELECT * FROM carve_evtx(filename="C:/pagefile.sys", accessor="ntfs")
```
### Saving the recovered events
Set `output` to also write the recovered chunks to a new EVTX
file that can be opened with Event Viewer, `parse_evtx()` or any
other EVTX tool. This requires the `FILESYSTEM_WRITE` permission.
Write it to a different disk than the one being carved: writing
to the same disk can overwrite the deleted data being recovered.
- Each chunk is written once, even if it was found several times.
- Windows refuses to open a file containing any chunk with a bad
checksum. Damaged chunks are therefore cut down to their leading
intact records, and their checksums are recomputed.
- Windows stops reading when the record numbers in the chunk
headers do not follow on from each other, so the chunk headers
are renumbered. The event record ids of the events are not
changed.
```vql
SELECT count() AS Events
FROM carve_evtx(filename="C:", accessor="ntfs",
output="C:/cases/recovered.evtx")
GROUP BY 1
```
type: Plugin
version: 1
args:
- name: filename
type: accessors.OSPath
description: The file or device to carve (e.g. 'C:' with the 'ntfs' accessor,
or C:/pagefile.sys with the ntfs accessor).
required: true
- name: accessor
type: string
description: The accessor to use.
- name: messagedb
type: string
description: A Message database from https://github.com/Velocidex/evtx-data.
- name: start_offset
type: int64
description: Start carving at this offset (default 0).
- name: end_offset
type: int64
description: Stop carving at this offset (default end of file).
- name: output
type: string
description: If set, also write the recovered chunks to a new EVTX file at this
path so they can be opened with Event Viewer. Each chunk is written once.
category: parsers
metadata:
permissions: FILESYSTEM_READ,FILESYSTEM_WRITE
platforms:
- darwin_amd64_cgo
- darwin_arm64_cgo
- linux_amd64_cgo
- windows_386_cgo
- windows_amd64_cgo
- name: carve_usn
description: |
Carve for the USN journal entries from a device.
@@ -13644,3 +13745,4 @@
- linux_amd64_cgo
- windows_386_cgo
- windows_amd64_cgo
+2 -2
View File
@@ -81,9 +81,9 @@ require (
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/sourcemap.v1 v1.0.5 // indirect
howett.net/plist v1.0.0
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9
www.velocidex.com/golang/go-prefetch v0.0.0-20260922101442-4bae97962980
www.velocidex.com/golang/oleparse v0.0.0-20250312121321-f7c2b4ec0959
+4 -4
View File
@@ -1119,12 +1119,12 @@ software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJ
software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ=
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf1AsYiIzUMJMIdFsVdrIw/BjrGzZbrsnz9xmeZmlYU=
www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4=
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8 h1:Me1/9DB8ZMH6+ZNFWPlfn3JEFgZNH59Cfk83wcQ2PEk=
www.velocidex.com/golang/evtx v0.2.1-0.20260916104054-0ef2ce4e83a8/go.mod h1:AEH50xeMcAUGiCXLAueUOq800jFgFyqngyPt88fDzow=
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe h1:CaGlM+7W48tSmI1k7/aodJXwmOQwgKSB/UkQ+zY6V5I=
www.velocidex.com/golang/evtx v0.2.1-0.20261007074225-6bf21c30f7fe/go.mod h1:AEH50xeMcAUGiCXLAueUOq800jFgFyqngyPt88fDzow=
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57 h1:CRdOpwPmVFzjfSrJXLj+xHOMyKfK4+WlJTf9IDMdXtk=
www.velocidex.com/golang/go-ese v0.2.1-0.20260812030448-458e8abf4a57/go.mod h1:H1Gw62XkspXEtZPT2ep+L57GgdKIDHwueazxKnn5EXc=
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5 h1:GDqKvZ2fdZVHQRSMczl41u2DPJEgbYdK6aaVkCBB8Jc=
www.velocidex.com/golang/go-ntfs v0.2.1-0.20260722055306-c81244c6bcd5/go.mod h1:4MSO8W9iNMXyBpjSpxApWfMjJUb9IWFD2Yis5JPZaSY=
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d h1:RzEtNw2k3jX8w5HLneg73Oz3vqi3YoYSHenoKjAC8S4=
www.velocidex.com/golang/go-ntfs v0.2.2-0.20261007101255-20cc4d33923d/go.mod h1:4MSO8W9iNMXyBpjSpxApWfMjJUb9IWFD2Yis5JPZaSY=
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9 h1:y5nOjOAyjH36mEGZsX2Wx3XpS78e9Jk2wK1M99rn4TY=
www.velocidex.com/golang/go-pe v0.1.1-0.20251107001057-f93001158cd9/go.mod h1:rEueOnxvtbV/uFfCeD44KvC6TL9ts/1S3VNmRudSaAc=
www.velocidex.com/golang/go-prefetch v0.0.0-20260922101442-4bae97962980 h1:YZhTjoaHG0/wuIOYHDg4q5nlKLx4BOkfORsiv8hWmxw=
@@ -190,6 +190,11 @@ Workspace symbols for query:
"kind": 12,
"location": null
},
{
"name": "carve_evtx",
"kind": 5,
"location": null
},
{
"name": "carve_usn",
"kind": 5,
+5 -1
View File
@@ -132,7 +132,11 @@ func (self *PagedReader) ReadAt(buf []byte, offset int64) (
to_read = len(page_buf) - page_offset
copy(buf[buf_idx:buf_idx+to_read],
page_buf[page_offset:page_offset+to_read])
return buf_idx + to_read, err
// EOF because the end of the buffer is past the file's
// end. We assume if the page buffer is short is because
// it is at EOF.
return buf_idx + to_read, io.EOF
}
copy(buf[buf_idx:buf_idx+to_read],
+1 -3
View File
@@ -34,11 +34,9 @@ func TestPagedReader(t *testing.T) {
reads := 0
for {
n, err := fd.Read(buf)
if n == 0 || err == io.EOF {
if n == 0 || err != nil && err != io.EOF {
break
}
assert.NoError(t, err)
reads++
result += string(buf[:n])
}
+28
View File
@@ -84,6 +84,34 @@ func (self *ReadSeekReaderAdapter) Read(buf []byte) (int, error) {
return n, err
}
// ReadAt reads from the delegate without changing the current offset
// or the EOF state used by Read(), so reading past the end does not
// cause later reads to fail.
func (self *ReadSeekReaderAdapter) ReadAt(buf []byte, offset int64) (int, error) {
self.mu.Lock()
size := self.size
self.mu.Unlock()
if offset < 0 {
return 0, IOError
}
if size > 0 {
if offset >= size {
return 0, io.EOF
}
if offset+int64(len(buf)) > size {
n, err := self.reader.ReadAt(buf[:size-offset], offset)
if err == nil {
err = io.EOF
}
return n, err
}
}
return self.reader.ReadAt(buf, offset)
}
func (self *ReadSeekReaderAdapter) SetSize(size int64) {
self.mu.Lock()
defer self.mu.Unlock()
+49
View File
@@ -0,0 +1,49 @@
package utils
import (
"bytes"
"io"
"testing"
"github.com/stretchr/testify/assert"
)
// Reading past the end with Read() sets a sticky EOF flag. ReadAt()
// must not be affected by it.
func TestReadSeekReaderAdapterReadAt(t *testing.T) {
data := []byte("0123456789")
adapter := NewReadSeekReaderAdapter(bytes.NewReader(data), nil)
adapter.SetSize(int64(len(data)))
// Exhaust the reader with Read().
_, err := io.ReadAll(adapter)
assert.NoError(t, err)
n, err := adapter.Read(make([]byte, 1))
assert.Equal(t, 0, n)
assert.Equal(t, io.EOF, err)
// ReadAt still works anywhere in the file.
buf := make([]byte, 4)
n, err = adapter.ReadAt(buf, 2)
assert.NoError(t, err)
assert.Equal(t, "2345", string(buf[:n]))
// A read crossing the size is cut short with EOF.
n, err = adapter.ReadAt(buf, 8)
assert.Equal(t, io.EOF, err)
assert.Equal(t, "89", string(buf[:n]))
// Reading past the end does not break later reads.
n, err = adapter.ReadAt(buf, 20)
assert.Equal(t, 0, n)
assert.Equal(t, io.EOF, err)
n, err = adapter.ReadAt(buf, 0)
assert.NoError(t, err)
assert.Equal(t, "0123", string(buf[:n]))
// MakeReaderAtter uses the stateless ReadAt.
_, ok := MakeReaderAtter(adapter).(*ReadSeekReaderAdapter)
assert.True(t, ok)
}
+1 -6
View File
@@ -98,11 +98,6 @@ func (self VerifyFunction) Call(ctx context.Context,
}
func stateToDict(state *launcher.AnalysisState) *ordereddict.Dict {
var errors []string
for _, e := range state.Errors {
errors = append(errors, e.Error())
}
definitions := make(map[string]*ordereddict.Dict)
for key, d := range state.Definitions {
var args []string
@@ -120,7 +115,7 @@ func stateToDict(state *launcher.AnalysisState) *ordereddict.Dict {
return ordereddict.NewDict().
Set("Artifact", state.Artifact).
Set("Permissions", state.Permissions).
Set("Errors", errors).
Set("Errors", state.Errors).
Set("Warnings", state.Warnings).
Set("Definitions", definitions).
Set("Suppressions", state.Suppressions)
+222
View File
@@ -0,0 +1,222 @@
/*
Velociraptor - Dig Deeper
Copyright (C) 2019-2025 Rapid7 Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package event_logs
import (
"context"
"os"
"github.com/Velocidex/ordereddict"
"github.com/dustin/go-humanize"
"www.velocidex.com/golang/evtx"
"www.velocidex.com/golang/velociraptor/accessors"
"www.velocidex.com/golang/velociraptor/accessors/file"
"www.velocidex.com/golang/velociraptor/acls"
"www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
"www.velocidex.com/golang/velociraptor/vql/readers"
vfilter "www.velocidex.com/golang/vfilter"
"www.velocidex.com/golang/vfilter/arg_parser"
)
type _CarveEvtxPluginArgs struct {
Filename *accessors.OSPath `vfilter:"required,field=filename,doc=The file or device to carve (e.g. 'C:' with the 'ntfs' accessor, or C:/pagefile.sys with the ntfs accessor)."`
Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use."`
Database string `vfilter:"optional,field=messagedb,doc=A Message database from https://github.com/Velocidex/evtx-data."`
StartOffset int64 `vfilter:"optional,field=start_offset,doc=Start carving at this offset (default 0)."`
EndOffset int64 `vfilter:"optional,field=end_offset,doc=Stop carving at this offset (default end of file)."`
Output string `vfilter:"optional,field=output,doc=If set, also write the recovered chunks to a new EVTX file at this path so they can be opened with Event Viewer. Each chunk is written once."`
}
type _CarveEvtxPlugin struct{}
func (self _CarveEvtxPlugin) Call(
ctx context.Context,
scope vfilter.Scope,
args *ordereddict.Dict) <-chan vfilter.Row {
output_chan := make(chan vfilter.Row)
go func() {
defer close(output_chan)
defer vql_subsystem.RegisterMonitor(ctx, "carve_evtx", args)()
defer utils.RecoverVQL(scope)
arg := &_CarveEvtxPluginArgs{}
err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
if err != nil {
scope.Log("carve_evtx: %v", err)
return
}
resolver, err := getMessageResolver(scope, arg.Database)
if err != nil {
scope.Log("carve_evtx: %v", err)
return
}
var writer *evtxWriter
if arg.Output != "" {
out_fd, err := openOutputFile(scope, arg.Output)
if err != nil {
scope.Log("carve_evtx: %v", err)
return
}
defer out_fd.Close()
writer, err = newEvtxWriter(out_fd)
if err != nil {
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
return
}
defer func() {
err := writer.Close()
if err != nil {
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
return
}
scope.Log("carve_evtx: Wrote %v unique chunks to %v",
writer.Chunks(), arg.Output)
}()
}
scope.Log("carve_evtx: Carving %v for EVTX chunks",
utils.Elide(arg.Filename.String(), 50))
stats := &carveStats{}
var records_emitted int64
reader, err := readers.NewAccessorReader(
scope, arg.Accessor, arg.Filename, 100)
if err != nil {
scope.Log("carve_evtx: %v", err)
return
}
defer reader.Close()
for chunk := range carveChunks(
ctx, scope,
reader, arg.StartOffset, arg.EndOffset, stats) {
scope.ChargeOp()
if writer != nil && includeInOutput(chunk) {
_, err := writer.WriteChunk(chunk)
if err != nil {
scope.Log("carve_evtx: Unable to write %v: %v", arg.Output, err)
return
}
}
records, err := parseCarvedChunk(chunk)
if err != nil && len(records) == 0 {
scope.Log("carve_evtx: chunk at offset %#x: %v", chunk.Offset, err)
continue
}
for _, record := range records {
row := makeCarvedRow(chunk, record, resolver)
if row == nil {
continue
}
records_emitted++
select {
case <-ctx.Done():
return
case output_chan <- row:
}
}
}
if stats.StopError != nil {
scope.Log("carve_evtx: Stopped at offset %#x: %v",
arg.StartOffset+stats.BytesScanned, stats.StopError)
}
scope.Log("carve_evtx: Scanned %v bytes, found %v chunk signatures, %v plausible chunks, %v records",
humanize.Bytes(uint64(stats.BytesScanned)),
stats.Candidates, stats.Chunks, records_emitted)
}()
return output_chan
}
// Chunks written to the output file.
func includeInOutput(chunk *carvedChunk) bool {
return true
}
// Create the output file after checking that the query may write
// there.
func openOutputFile(scope vfilter.Scope, path string) (*os.File, error) {
err := vql_subsystem.CheckAccess(scope, acls.FILESYSTEM_WRITE)
if err != nil {
return nil, err
}
err = file.CheckPath(path)
if err != nil {
return nil, err
}
return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
}
// Emit the same event structure as parse_evtx() with details about
// where the record was found and how trustworthy its chunk is.
func makeCarvedRow(chunk *carvedChunk, record *evtx.EventRecord,
resolver evtx.MessageResolver) *ordereddict.Dict {
event_map, ok := record.Event.(*ordereddict.Dict)
if !ok {
return nil
}
event, pres := ordereddict.GetMap(event_map, "Event")
if !pres {
return nil
}
if resolver != nil {
event.Set("Message", evtx.ExpandMessage(event, resolver))
}
return event.Set("ChunkInfo", ordereddict.NewDict().
Set("Offset", chunk.Offset).
Set("HeaderValid", chunk.HeaderChecksumValid).
Set("DataValid", chunk.DataChecksumValid).
Set("Truncated", chunk.Truncated).
Set("Compressed", chunk.Compressed))
}
func (self _CarveEvtxPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.PluginInfo {
return &vfilter.PluginInfo{
Name: "carve_evtx",
Doc: "Carve EVTX chunks from a file or device (e.g. unallocated space, " +
"the pagefile or a memory image) and parse their event records.",
ArgType: type_map.AddType(scope, &_CarveEvtxPluginArgs{}),
// Writing the output file checks FILESYSTEM_WRITE when it is used.
Metadata: vql_subsystem.VQLMetadata().
Permissions(acls.FILESYSTEM_READ, acls.FILESYSTEM_WRITE).Build(),
Version: 1,
}
}
func init() {
vql_subsystem.RegisterPlugin(&_CarveEvtxPlugin{})
}
+392
View File
@@ -0,0 +1,392 @@
/*
Velociraptor - Dig Deeper
Copyright (C) 2019-2025 Rapid7 Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package event_logs
import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"hash/crc32"
"io"
"time"
"github.com/dustin/go-humanize"
"www.velocidex.com/golang/evtx"
ntfs "www.velocidex.com/golang/go-ntfs/parser"
"www.velocidex.com/golang/velociraptor/utils"
vfilter "www.velocidex.com/golang/vfilter"
)
const (
// How much data we scan for the chunk signature in each read.
carveBlockSize = 1024 * 1024
// Offsets into the chunk header. See
// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc
chunkHeaderSizeValue = 0x80
chunkFreeSpaceOffset = 0x30
chunkHeaderChecksumStart = 0x78
chunkHeaderChecksumEnd = 0x80
// The library refuses to parse more records than this from a
// single chunk so a header claiming more is not plausible.
maxRecordsPerChunk = 1024 * 10
// LZNT1 block header: bit 15 is set for compressed blocks,
// bits 12-14 are always 3 and the low 12 bits are the size of
// the block minus 3.
lznt1BlockSize = 0x1000
lznt1UnitSize = 0x10000
chunkOffsetInUnit = 0x1000
lznt1PrefixSize = 3
lznt1SignatureMask = 0x7000
lznt1Signature = 0x3000
lznt1CompressedHeaderMask = 0xb000
lznt1SizeMask = 0x0fff
)
var chunkMagic = []byte(evtx.EVTX_CHUNK_HEADER_MAGIC)
// A chunk signature found in the carved data.
type carvedChunk struct {
// Offset of the chunk in the carved stream.
Offset int64
Header evtx.ChunkHeader
// The CRC32 of the chunk header matches the stored checksum.
HeaderChecksumValid bool
// The CRC32 of the event records matches the stored checksum.
DataChecksumValid bool
// The stream ended before the full chunk could be read. The
// missing data is zero filled.
Truncated bool
// The chunk was found inside an NTFS compressed file and was
// decompressed.
Compressed bool
// The full chunk, always EVTX_CHUNK_SIZE long.
Data []byte
}
type carveStats struct {
BytesScanned int64
Candidates int64
Chunks int64
// Why the scan stopped before reaching the end offset, if it did.
StopError error
}
// Scan the reader between start and end (end <= 0 means until the
// end of the stream) for EVTX chunks. Signatures are searched at any
// byte offset because chunks in memory, the pagefile or unallocated
// space are not necessarily aligned. Only chunks with a plausible
// header are emitted - their checksums are reported but not enforced
// because partially overwritten chunks still contain useful records.
func carveChunks(
ctx context.Context,
scope vfilter.Scope,
reader io.ReaderAt,
start, end int64, stats *carveStats) <-chan *carvedChunk {
output_chan := make(chan *carvedChunk)
go func() {
defer close(output_chan)
defer utils.RecoverVQL(scope)
logger, closer := utils.NewDeduplicatedLogger(30 * time.Second)
defer closer()
start_time := utils.GetTime().Now()
buf := make([]byte, carveBlockSize)
// The end of the previous block is kept so a signature
// straddling two blocks is still found. Carrying it in
// memory keeps every read at the block offsets.
overlap := len(chunkMagic) - 1
window := make([]byte, 0, overlap+carveBlockSize)
var tail []byte
// Scan from an aligned offset and ignore signatures before
// the start.
offset := start
for end <= 0 || offset < end {
to_read := int64(len(buf))
if end > 0 && offset+to_read > end {
to_read = end - offset
}
// Readers may return fewer bytes than requested, with or
// without an error. Only a read returning nothing ends
// the scan.
n, err := reader.ReadAt(buf[:to_read], int64(offset))
if n <= 0 {
if err != nil && !errors.Is(err, io.EOF) {
stats.StopError = err
}
return
}
window = append(append(window[:0], tail...), buf[:n]...)
window_offset := offset - int64(len(tail))
for idx := 0; idx < len(window); {
hit := bytes.Index(window[idx:], chunkMagic)
if hit < 0 {
break
}
chunk_offset := window_offset + int64(idx+hit)
idx += hit + 1
if chunk_offset < start {
continue
}
stats.Candidates++
chunk, ok := readChunk(reader, chunk_offset)
if !ok {
continue
}
stats.Chunks++
select {
case <-ctx.Done():
return
case output_chan <- chunk:
}
}
// A full signature can never be inside the tail, so no
// signature is counted twice.
tail_start := len(window) - overlap
if tail_start < 0 {
tail_start = 0
}
tail = append(tail[:0], window[tail_start:]...)
offset += int64(n)
stats.BytesScanned = offset - start
// Report progress - this could take a while so it is nice
// to know how it's going.
logger.Log(
// This will be called a lot so log lazily.
func(msg string, args ...interface{}) {
scope.Log("carve_evtx: scanned %v bytes in %v",
humanize.Bytes(uint64(stats.BytesScanned)),
utils.GetTime().Now().Sub(
start_time).Round(time.Second))
}, "")
if errors.Is(err, io.EOF) {
return
}
select {
case <-ctx.Done():
return
default:
}
}
}()
return output_chan
}
// Read the chunk at offset and check that its header is plausible.
func readChunk(reader io.ReaderAt, offset int64) (*carvedChunk, bool) {
data := make([]byte, evtx.EVTX_CHUNK_SIZE)
n, _ := reader.ReadAt(data, offset)
result, ok := checkChunk(offset, data, n)
if ok {
return result, true
}
data, n = readCompressedChunk(reader, offset)
result, ok = checkChunk(offset, data, n)
if ok {
result.Compressed = true
}
return result, ok
}
// NTFS compresses files in units of 16 clusters using LZNT1, and
// the event logs are compressed by default. Every 4kb block in a
// unit is compressed separately, so a chunk starting on a block
// boundary still has its signature as 8 literal bytes - preceded by
// the 2 byte block header and a zero flag byte.
//
// Chunks follow the 4kb file header so a chunk spans two
// compression units. Each unit starts on a new cluster which we
// assume to be 4kb.
func readCompressedChunk(reader io.ReaderAt, offset int64) ([]byte, int) {
start := offset - lznt1PrefixSize
if start < 0 {
return nil, 0
}
// Compressed data is never larger than the uncompressed data
// plus block headers and the slack at the end of a unit.
raw := make([]byte, evtx.EVTX_CHUNK_SIZE+2*lznt1BlockSize)
n, _ := reader.ReadAt(raw, start)
raw = raw[:n]
if n < lznt1PrefixSize || raw[2] != 0 ||
binary.LittleEndian.Uint16(raw)&lznt1CompressedHeaderMask !=
lznt1CompressedHeaderMask {
return nil, 0
}
data := make([]byte, 0, evtx.EVTX_CHUNK_SIZE)
skipped_to_next_unit := false
i := 0
for len(data) < evtx.EVTX_CHUNK_SIZE && i+2 <= len(raw) {
header := binary.LittleEndian.Uint16(raw[i:])
// The compression unit has ended - either we have read all
// its blocks or there is no block header. The next unit
// starts on the next cluster.
unit_full := len(data) == lznt1UnitSize-chunkOffsetInUnit
if (unit_full && !skipped_to_next_unit) ||
header&lznt1SignatureMask != lznt1Signature {
if skipped_to_next_unit {
// The next unit was not compressible so it is
// stored as is.
remaining := evtx.EVTX_CHUNK_SIZE - len(data)
if remaining > len(raw)-i {
remaining = len(raw) - i
}
data = append(data, raw[i:i+remaining]...)
break
}
skipped_to_next_unit = true
next_unit := start + int64(i) + lznt1BlockSize - 1
next_unit -= next_unit % lznt1BlockSize
i = int(next_unit - start)
continue
}
block_end := i + int(header&lznt1SizeMask) + 3
if block_end > len(raw) {
break
}
block, err := lznt1Decompress(raw[i:block_end])
if err != nil || len(block) == 0 || len(block) > lznt1BlockSize {
break
}
data = append(data, block...)
i = block_end
}
if len(data) > evtx.EVTX_CHUNK_SIZE {
data = data[:evtx.EVTX_CHUNK_SIZE]
}
n = len(data)
// Always return a full chunk, zero filled.
return append(data, make([]byte, evtx.EVTX_CHUNK_SIZE-n)...), n
}
// Check that the chunk has a plausible header. n is the number of
// valid bytes in data.
func checkChunk(offset int64, data []byte, n int) (*carvedChunk, bool) {
if n < evtx.EVTX_CHUNK_HEADER_SIZE {
return nil, false
}
result := &carvedChunk{
Offset: offset,
Data: data,
Truncated: n < evtx.EVTX_CHUNK_SIZE,
}
err := binary.Read(bytes.NewReader(data),
binary.LittleEndian, &result.Header)
if err != nil {
return nil, false
}
header := &result.Header
free_space := binary.LittleEndian.Uint32(data[chunkFreeSpaceOffset:])
if header.HeaderSize != chunkHeaderSizeValue ||
header.FirstEventRecNumber > header.LastEventRecNumber ||
header.LastEventRecNumber-header.FirstEventRecNumber >= maxRecordsPerChunk ||
header.LastEventRecOffset >= evtx.EVTX_CHUNK_SIZE ||
free_space < evtx.EVTX_CHUNK_HEADER_SIZE ||
free_space > evtx.EVTX_CHUNK_SIZE {
return nil, false
}
// The header checksum covers the first 120 bytes and the
// string/template tables, skipping the flags and the checksum.
crc := crc32.NewIEEE()
_, _ = crc.Write(data[:chunkHeaderChecksumStart])
_, _ = crc.Write(data[chunkHeaderChecksumEnd:evtx.EVTX_CHUNK_HEADER_SIZE])
result.HeaderChecksumValid = crc.Sum32() == header.CheckSum
result.DataChecksumValid = crc32.ChecksumIEEE(
data[evtx.EVTX_CHUNK_HEADER_SIZE:free_space]) == header.EventRecordCheckSum
return result, true
}
// LZNT1Decompress can panic on malformed blocks (e.g. a back
// reference truncated by the end of the block). Carved data is often
// malformed so contain any panic to this block.
func lznt1Decompress(block []byte) (result []byte, err error) {
defer func() {
r := recover()
if r != nil {
err = fmt.Errorf("LZNT1 decompression panic: %v", r)
utils.PrintStack()
}
}()
return ntfs.LZNT1Decompress(block)
}
// Parse the records in a carved chunk. Carved data may be corrupt so
// any panic in the parser is contained to this chunk.
func parseCarvedChunk(chunk *carvedChunk) (records []*evtx.EventRecord, err error) {
defer func() {
r := recover()
if r != nil {
err = errors.New("parser panic while parsing carved chunk")
utils.PrintStack()
}
}()
parsed, err := evtx.NewChunk(bytes.NewReader(chunk.Data), 0)
if err != nil {
return nil, err
}
return parsed.Parse(0)
}
+545
View File
@@ -0,0 +1,545 @@
package event_logs
import (
"bytes"
"context"
"encoding/binary"
"errors"
"io"
"math/rand"
"os"
"testing"
"github.com/Velocidex/ordereddict"
"www.velocidex.com/golang/evtx"
ntfs "www.velocidex.com/golang/go-ntfs/parser"
"www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
"www.velocidex.com/golang/velociraptor/vtesting/assert"
)
// This log contains a single chunk recording the System log being
// cleared (Event ID 104).
const testEvtx = "../../../artifacts/testdata/files/DE_104_system_log_cleared.evtx"
func getTestChunk(t *testing.T) []byte {
data, err := os.ReadFile(testEvtx)
assert.NoError(t, err)
// The first chunk follows the 4kb file header.
chunk := data[0x1000 : 0x1000+evtx.EVTX_CHUNK_SIZE]
assert.Equal(t, evtx.EVTX_CHUNK_HEADER_MAGIC, string(chunk[:8]))
return chunk
}
func noise(rng *rand.Rand, size int) []byte {
result := make([]byte, size)
rng.Read(result)
return result
}
func carveAll(t *testing.T, image []byte, start, end int64) (
[]*carvedChunk, *carveStats) {
scope := vql_subsystem.MakeScope()
stats := &carveStats{}
var result []*carvedChunk
for chunk := range carveChunks(context.Background(), scope,
bytes.NewReader(image), start, end, stats) {
result = append(result, chunk)
}
return result, stats
}
func TestCarveChunks(t *testing.T) {
rng := rand.New(rand.NewSource(1))
chunk := getTestChunk(t)
// A copy of the chunk with a damaged string table - the header
// is still plausible but its checksum no longer matches.
damaged := append([]byte{}, chunk...)
damaged[0x100] ^= 0xff
image := &bytes.Buffer{}
image.Write(noise(rng, 100003))
// 1. An intact chunk at an unaligned offset.
intact_offset := int64(image.Len())
image.Write(chunk)
image.Write(noise(rng, 5000))
// A false positive: the signature followed by garbage.
image.WriteString(evtx.EVTX_CHUNK_HEADER_MAGIC)
image.Write(noise(rng, 1000))
// 2. A chunk with a damaged header.
damaged_offset := int64(image.Len())
image.Write(damaged)
image.Write(noise(rng, 7777))
// 3. A chunk cut off by the end of the image.
truncated_offset := int64(image.Len())
image.Write(chunk[:0x8000])
chunks, stats := carveAll(t, image.Bytes(), 0, 0)
assert.Equal(t, int64(image.Len()), stats.BytesScanned)
assert.Equal(t, int64(4), stats.Candidates)
assert.Equal(t, int64(3), stats.Chunks)
assert.Equal(t, 3, len(chunks))
assert.Equal(t, intact_offset, chunks[0].Offset)
assert.True(t, chunks[0].HeaderChecksumValid)
assert.True(t, chunks[0].DataChecksumValid)
assert.False(t, chunks[0].Truncated)
assert.Equal(t, damaged_offset, chunks[1].Offset)
assert.False(t, chunks[1].HeaderChecksumValid)
assert.True(t, chunks[1].DataChecksumValid)
assert.Equal(t, truncated_offset, chunks[2].Offset)
assert.True(t, chunks[2].HeaderChecksumValid)
assert.True(t, chunks[2].Truncated)
assert.Equal(t, evtx.EVTX_CHUNK_SIZE, len(chunks[2].Data))
// All the records in the intact chunk are recovered.
records, err := parseCarvedChunk(chunks[0])
assert.NoError(t, err)
header := chunks[0].Header
assert.Equal(t, int(header.LastEventRecNumber-header.FirstEventRecNumber+1),
len(records))
// The log cleared event is among them.
found := false
for _, record := range records {
row := makeCarvedRow(chunks[0], record, nil)
assert.NotNil(t, row)
value, _ := ordereddict.GetAny(row, "System.EventID.Value")
event_id, _ := utils.ToInt64(value)
if event_id == 104 {
found = true
}
chunk_info_any, pres := row.Get("ChunkInfo")
assert.True(t, pres)
chunk_offset, _ := chunk_info_any.(*ordereddict.Dict).GetInt64("Offset")
assert.Equal(t, intact_offset, chunk_offset)
}
assert.True(t, found)
}
// A signature split across two read blocks must be found exactly once.
func TestCarveChunkAcrossBlockBoundary(t *testing.T) {
rng := rand.New(rand.NewSource(2))
chunk := getTestChunk(t)
for _, delta := range []int{-8, -7, -3, -1, 0, 1} {
offset := carveBlockSize + delta
image := noise(rng, offset)
image = append(image, chunk...)
image = append(image, noise(rng, 100)...)
chunks, _ := carveAll(t, image, 0, 0)
assert.Equal(t, 1, len(chunks), "delta %v", delta)
assert.Equal(t, int64(offset), chunks[0].Offset, "delta %v", delta)
}
}
func TestCarveRange(t *testing.T) {
rng := rand.New(rand.NewSource(3))
chunk := getTestChunk(t)
image := noise(rng, 1000)
image = append(image, chunk...)
second := len(image)
image = append(image, chunk...)
// Skipping past the first chunk only finds the second.
chunks, _ := carveAll(t, image, 1001, 0)
assert.Equal(t, 1, len(chunks))
assert.Equal(t, int64(second), chunks[0].Offset)
// A range ending before the second chunk only finds the first.
chunks, _ = carveAll(t, image, 0, int64(second))
assert.Equal(t, 1, len(chunks))
assert.Equal(t, int64(1000), chunks[0].Offset)
}
// Accessors return a ReadSeekReaderAdapter rather than a plain
// io.ReaderAt. Its EOF flag is sticky, so a chunk read that runs past
// the end of the stream must not stop the scan early.
func TestCarveThroughAccessorAdapter(t *testing.T) {
rng := rand.New(rand.NewSource(5))
chunk := getTestChunk(t)
image := noise(rng, carveBlockSize/2)
first := len(image)
image = append(image, chunk...)
image = append(image, noise(rng, 2*carveBlockSize)...)
second := len(image)
image = append(image, chunk...)
image = append(image, noise(rng, 1000)...)
// A chunk cut off by the end of the stream.
third := len(image)
image = append(image, chunk[:0x4000]...)
scope := vql_subsystem.MakeScope()
for _, with_size := range []bool{false, true} {
adapter := utils.NewReadSeekReaderAdapter(bytes.NewReader(image), nil)
if with_size {
adapter.SetSize(int64(len(image)))
}
stats := &carveStats{}
var offsets []int64
for c := range carveChunks(context.Background(), scope,
utils.MakeReaderAtter(adapter), 0, 0, stats) {
offsets = append(offsets, c.Offset)
}
assert.Equal(t, []int64{int64(first), int64(second), int64(third)},
offsets, "with_size %v", with_size)
assert.Equal(t, int64(len(image)), stats.BytesScanned,
"with_size %v", with_size)
}
}
// Behaves like a raw Windows device which rejects reads that are not
// sector aligned.
type deviceReader struct {
reader *bytes.Reader
}
func (self *deviceReader) ReadAt(buf []byte, offset int64) (int, error) {
if offset%512 != 0 || len(buf)%512 != 0 {
return 0, errors.New("The parameter is incorrect.")
}
return self.reader.ReadAt(buf, offset)
}
func NewDeviceReader(data []byte) (io.ReaderAt, error) {
return utils.NewPagedReader(
&deviceReader{bytes.NewReader(data)},
4096, 100)
}
// Unaligned chunks and offsets must still be carved from a device.
func TestCarveFromAlignedDevice(t *testing.T) {
rng := rand.New(rand.NewSource(6))
chunk := getTestChunk(t)
image := noise(rng, 3001)
unaligned := len(image)
image = append(image, chunk...)
image = append(image, noise(rng, 4096-(len(image)%4096))...)
aligned := len(image)
image = append(image, chunk...)
image = append(image, noise(rng, carveBlockSize+123)...)
last := len(image)
image = append(image, chunk[:0x3000]...)
scope := vql_subsystem.MakeScope()
for _, tc := range []struct {
start, end int64
expected []int64
}{
{0, 0, []int64{int64(unaligned), int64(aligned), int64(last)}},
{int64(unaligned), 0, []int64{int64(unaligned), int64(aligned), int64(last)}},
{int64(unaligned) + 1, int64(last) + 777,
[]int64{int64(aligned), int64(last)}},
{0, int64(aligned) + 5, []int64{int64(unaligned)}},
} {
stats := &carveStats{}
var offsets []int64
var chunks []*carvedChunk
reader, err := NewDeviceReader(image)
assert.NoError(t, err)
for c := range carveChunks(context.Background(), scope,
reader,
tc.start, tc.end, stats) {
offsets = append(offsets, c.Offset)
chunks = append(chunks, c)
}
assert.NoError(t, stats.StopError, "range %v-%v", tc.start, tc.end)
assert.Equal(t, tc.expected, offsets, "range %v-%v", tc.start, tc.end)
for _, c := range chunks {
assert.True(t, c.HeaderChecksumValid, "chunk %v", c.Offset)
assert.Equal(t, c.Offset == int64(last), c.Truncated, "chunk %v", c.Offset)
assert.Equal(t, chunk[:0x3000], c.Data[:0x3000], "chunk %v", c.Offset)
}
}
}
func TestAlignedReadAt(t *testing.T) {
data := noise(rand.New(rand.NewSource(7)), 10000)
reader, err := NewDeviceReader(data)
assert.NoError(t, err)
for _, tc := range []struct {
offset int64
length int
n int
eof bool
}{
{0, 4096, 4096, false},
{1, 10, 10, false},
{4095, 2, 2, false},
{9990, 10, 10, false},
{9990, 20, 10, true},
{10000, 10, 0, true},
{20000, 10, 0, true},
} {
buf := make([]byte, tc.length)
n, err := reader.ReadAt(buf, tc.offset)
assert.Equal(t, tc.n, n, "offset %v", tc.offset)
assert.Equal(t, tc.eof, errors.Is(err, io.EOF), "offset %v err %v", tc.offset, err)
if n > 0 {
assert.Equal(t, data[tc.offset:tc.offset+int64(n)], buf[:n], "offset %v", tc.offset)
}
}
}
// Compress a block of up to 4kb with LZNT1 the way NTFS does.
func lznt1CompressBlock(in []byte) []byte {
out := []byte{0, 0}
pos := 0
for pos < len(in) {
flag_idx := len(out)
out = append(out, 0)
for bit := 0; bit < 8 && pos < len(in); bit++ {
// The split between offset and length bits depends on
// the position in the block.
length_bits := 12
for p := pos - 1; p >= 0x10; p >>= 1 {
length_bits--
}
max_offset := 1 << (16 - length_bits)
max_length := (1 << length_bits) - 1 + 3
best_len, best_off := 0, 0
for off := 1; off <= pos && off <= max_offset; off++ {
l := 0
for l < max_length && pos+l < len(in) && in[pos+l-off] == in[pos+l] {
l++
}
if l > best_len {
best_len, best_off = l, off
}
if l == max_length {
break
}
}
if best_len >= 3 {
token := uint16(best_off-1)<<length_bits | uint16(best_len-3)
out = binary.LittleEndian.AppendUint16(out, token)
out[flag_idx] |= 1 << bit
pos += best_len
} else {
out = append(out, in[pos])
pos++
}
}
}
// Store incompressible blocks as is.
if len(out)-2 >= lznt1BlockSize {
out = binary.LittleEndian.AppendUint16(nil, 0x3000|uint16(len(in)-1))
return append(out, in...)
}
binary.LittleEndian.PutUint16(out, 0xb000|uint16(len(out)-3))
return out
}
// Lay out a file on disk the way NTFS compression does: 64kb units
// each starting on a new 4kb cluster. The slack at the end of a unit
// holds stale data. Units listed in raw_units are stored
// uncompressed.
func ntfsCompress(rng *rand.Rand, file []byte, raw_units ...int) []byte {
var result []byte
for unit_idx := 0; unit_idx*lznt1UnitSize < len(file); unit_idx++ {
unit := file[unit_idx*lznt1UnitSize:]
if len(unit) > lznt1UnitSize {
unit = unit[:lznt1UnitSize]
}
var compressed []byte
for i := 0; i < len(unit); i += lznt1BlockSize {
end := i + lznt1BlockSize
if end > len(unit) {
end = len(unit)
}
compressed = append(compressed, lznt1CompressBlock(unit[i:end])...)
}
for _, raw := range raw_units {
if raw == unit_idx {
compressed = unit
}
}
result = append(result, compressed...)
if rem := len(result) % lznt1BlockSize; rem != 0 {
result = append(result, noise(rng, lznt1BlockSize-rem)...)
}
}
return result
}
func TestLZNT1Compressor(t *testing.T) {
data, err := os.ReadFile(testEvtx)
assert.NoError(t, err)
for i := 0; i < len(data); i += lznt1BlockSize {
block := data[i : i+lznt1BlockSize]
decompressed, err := ntfs.LZNT1Decompress(lznt1CompressBlock(block))
assert.NoError(t, err)
assert.Equal(t, block, decompressed, "block at %#x", i)
}
}
// The event logs are NTFS compressed by default so carving a raw
// volume needs to decompress chunks.
func TestCarveCompressedChunk(t *testing.T) {
rng := rand.New(rand.NewSource(8))
file, err := os.ReadFile(testEvtx)
assert.NoError(t, err)
chunk := getTestChunk(t)
for _, raw_units := range [][]int{nil, {1}} {
disk := noise(rng, 7*lznt1BlockSize)
disk = append(disk, ntfsCompress(rng, file, raw_units...)...)
disk = append(disk, noise(rng, 3*lznt1BlockSize)...)
// The plain chunk header is not visible on disk.
assert.False(t, bytes.Contains(disk, chunk[:evtx.EVTX_CHUNK_HEADER_SIZE]))
chunks, stats := carveAll(t, disk, 0, 0)
assert.Equal(t, int64(1), stats.Chunks, "raw units %v", raw_units)
if len(chunks) != 1 {
continue
}
c := chunks[0]
assert.True(t, c.Compressed)
assert.True(t, c.HeaderChecksumValid)
assert.True(t, c.DataChecksumValid)
assert.False(t, c.Truncated)
assert.Equal(t, chunk, c.Data, "raw units %v", raw_units)
records, err := parseCarvedChunk(c)
assert.NoError(t, err)
found := false
for _, record := range records {
row := makeCarvedRow(c, record, nil)
value, _ := ordereddict.GetAny(row, "System.EventID.Value")
event_id, _ := utils.ToInt64(value)
found = found || event_id == 104
chunk_info_any, pres := row.Get("ChunkInfo")
assert.True(t, pres)
compressed, _ := chunk_info_any.(*ordereddict.Dict).Get("Compressed")
assert.Equal(t, true, compressed)
}
assert.True(t, found)
}
}
// A compressed chunk cut off by the end of the image is still
// recovered as far as possible.
func TestCarveTruncatedCompressedChunk(t *testing.T) {
rng := rand.New(rand.NewSource(9))
file, err := os.ReadFile(testEvtx)
assert.NoError(t, err)
disk := noise(rng, lznt1BlockSize)
compressed := ntfsCompress(rng, file)
disk = append(disk, compressed[:len(compressed)/3]...)
chunks, _ := carveAll(t, disk, 0, 0)
assert.Equal(t, 1, len(chunks))
if len(chunks) == 1 {
assert.True(t, chunks[0].Compressed)
assert.True(t, chunks[0].Truncated)
assert.True(t, chunks[0].HeaderChecksumValid)
}
}
// A compressed block whose last token is a back reference cut short
// by the end of the block. This crashed LZNT1Decompress on a real
// volume.
func truncatedReferenceBlock() []byte {
block := []byte{0x0a, 0xb0, 0x00}
block = append(block, evtx.EVTX_CHUNK_HEADER_MAGIC...)
return append(block, 0x01, 0x41)
}
func TestCarveTruncatedLZNT1Reference(t *testing.T) {
_, err := lznt1Decompress(truncatedReferenceBlock())
assert.Error(t, err)
rng := rand.New(rand.NewSource(10))
disk := noise(rng, 5000)
disk = append(disk, truncatedReferenceBlock()...)
disk = append(disk, noise(rng, 5000)...)
chunks, stats := carveAll(t, disk, 0, 0)
assert.Equal(t, 0, len(chunks))
assert.Equal(t, int64(1), stats.Candidates)
}
// Carved data is attacker controlled - run with
// go test -fuzz FuzzCarveChunk ./vql/parsers/event_logs/
func FuzzCarveChunk(f *testing.F) {
scope := vql_subsystem.MakeScope()
data, err := os.ReadFile(testEvtx)
if err == nil && len(data) >= 0x1000+evtx.EVTX_CHUNK_SIZE {
f.Add(data[0x1000 : 0x1000+evtx.EVTX_CHUNK_SIZE])
}
f.Add([]byte(evtx.EVTX_CHUNK_HEADER_MAGIC))
f.Add(truncatedReferenceBlock())
if err == nil {
f.Add(ntfsCompress(rand.New(rand.NewSource(1)), data))
}
f.Fuzz(func(t *testing.T, image []byte) {
stats := &carveStats{}
for c := range carveChunks(context.Background(), scope,
bytes.NewReader(image), 0, 0, stats) {
records, _ := parseCarvedChunk(c)
for _, r := range records {
makeCarvedRow(c, r, nil)
}
}
})
}
// Random data with forced signatures must never crash the carver.
func TestCarveGarbage(t *testing.T) {
rng := rand.New(rand.NewSource(4))
chunk := getTestChunk(t)
for i := 0; i < 50; i++ {
// Keep a valid looking header but scramble the records.
data := append([]byte{}, chunk...)
rng.Read(data[evtx.EVTX_CHUNK_HEADER_SIZE:])
chunks, _ := carveAll(t, data, 0, 0)
for _, c := range chunks {
records, _ := parseCarvedChunk(c)
for _, r := range records {
makeCarvedRow(c, r, nil)
}
}
}
}
+32 -20
View File
@@ -38,6 +38,37 @@ type _ParseEvtxPluginArgs struct {
Workers int64 `vfilter:"optional,field=workers,doc=If specified we use this many workers to parse the file in parallel (default 1)."`
}
// Get a resolver to expand event messages. The resolver is closed
// when the root scope is destroyed.
func getMessageResolver(
scope vfilter.Scope, database string) (evtx.MessageResolver, error) {
var resolver evtx.MessageResolver
var err error
if database != "" {
resolver, err = evtx.NewDBResolver(database)
} else {
// If the plugin did not specify a database, use the local
// resolver - On windows this will search DLLs for the messages.
resolver, err = evtx.GetNativeResolver(evtx.MessageResolverOpts{
LangPreferenceRegeExp: vql_subsystem.GetStringFromRow(
scope, scope, constants.EVTX_PREFERRED_LANG),
})
}
if err != nil {
return nil, err
}
// Close the db when we are done.
err = vql_subsystem.GetRootScope(scope).AddDestructor(resolver.Close)
if err != nil {
resolver.Close()
return nil, err
}
return resolver, nil
}
type _ParseEvtxPlugin struct{}
func (self _ParseEvtxPlugin) Call(
@@ -58,31 +89,12 @@ func (self _ParseEvtxPlugin) Call(
return
}
var resolver evtx.MessageResolver
if arg.Database != "" {
resolver, err = evtx.NewDBResolver(arg.Database)
} else {
// If the plugin did not specify a database, use the local
// resolver - On windows this will search DLLs for the messages.
resolver, err = evtx.GetNativeResolver(evtx.MessageResolverOpts{
LangPreferenceRegeExp: vql_subsystem.GetStringFromRow(
scope, scope, constants.EVTX_PREFERRED_LANG),
})
}
resolver, err := getMessageResolver(scope, arg.Database)
if err != nil {
scope.Log("parse_evtx: %s", err.Error())
return
}
// Close the db when we are done.
err = vql_subsystem.GetRootScope(scope).AddDestructor(resolver.Close)
if err != nil {
resolver.Close()
scope.Log("parse_evtx: %s", err.Error())
return
}
for _, filename := range arg.Filenames {
func() {
defer utils.RecoverVQL(scope)
+214
View File
@@ -0,0 +1,214 @@
/*
Velociraptor - Dig Deeper
Copyright (C) 2019-2025 Rapid7 Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package event_logs
import (
"crypto/sha256"
"encoding/binary"
"hash/crc32"
"io"
"www.velocidex.com/golang/evtx"
)
const (
// Offsets into the EVTX file header. See
// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc
fileHeaderSize = 0x1000
fileHeaderUsedSize = 0x80
fileHeaderChecksumStart = 0x78
// The file header stores the number of chunks in 16 bits.
maxChunksPerFile = 0xffff
chunkDataChecksumOffset = 0x34
// An event record starts with "**\0\0", its size, its record
// id and its timestamp.
recordMagic = 0x00002a2a
recordHeaderSize = 0x18
)
// Writes carved chunks into a new EVTX file so the recovered events
// can be opened with Event Viewer and other EVTX tools. Each chunk
// carries its own string and template tables so chunks from
// different logs can be combined in one file.
type evtxWriter struct {
fd io.WriterAt
chunks uint64
next_record uint64
// Windows only reads on while the record numbers in the chunk
// headers continue from one chunk to the next, but carved
// chunks come from many logs. The chunk headers are renumbered
// - the event record ids shown for each event are not changed.
next_number uint64
// The same chunk is often found several times (e.g. the live
// log and old copies in unallocated space). Write it once.
seen map[[sha256.Size]byte]bool
}
func newEvtxWriter(fd io.WriterAt) (*evtxWriter, error) {
self := &evtxWriter{
fd: fd,
seen: make(map[[sha256.Size]byte]bool),
next_number: 1,
}
// A valid header even if no chunks are written.
return self, self.writeHeader()
}
// Add the chunk to the file. Returns false if the chunk was skipped
// because it was already written or the file is full.
func (self *evtxWriter) WriteChunk(chunk *carvedChunk) (bool, error) {
key := sha256.Sum256(chunk.Data)
if self.seen[key] || self.chunks >= maxChunksPerFile {
return false, nil
}
data := repairChunk(chunk)
if data == nil {
return false, nil
}
first := binary.LittleEndian.Uint64(data[0x08:])
last := binary.LittleEndian.Uint64(data[0x10:])
binary.LittleEndian.PutUint64(data[0x08:], self.next_number)
binary.LittleEndian.PutUint64(data[0x10:], self.next_number+last-first)
setChunkHeaderChecksum(data)
offset := int64(fileHeaderSize) + int64(self.chunks)*evtx.EVTX_CHUNK_SIZE
_, err := self.fd.WriteAt(data, offset)
if err != nil {
return false, err
}
// When the output file is on the disk being carved, the scan
// may find the chunks we wrote. Do not write them again.
self.seen[key] = true
self.seen[sha256.Sum256(data)] = true
self.chunks++
self.next_number += last - first + 1
if chunk.Header.LastEventRecID >= self.next_record {
self.next_record = chunk.Header.LastEventRecID + 1
}
return true, nil
}
// Windows refuses to open a file if any chunk has a bad checksum. A
// damaged chunk is cut down to its leading intact records and its
// header and data checksum are rewritten to match. Returns a copy of
// the chunk, or nil if no record survives. The caller sets the header
// checksum.
func repairChunk(chunk *carvedChunk) []byte {
data := append([]byte{}, chunk.Data...)
if chunk.HeaderChecksumValid && chunk.DataChecksumValid {
return data
}
free_space := int(binary.LittleEndian.Uint32(data[chunkFreeSpaceOffset:]))
// Records follow each other from the end of the chunk header.
// Each starts with a signature and its size, and ends with a
// copy of the size.
offset := evtx.EVTX_CHUNK_HEADER_SIZE
last_offset := 0
var last_id uint64
count := uint64(0)
for offset+recordHeaderSize <= free_space {
if binary.LittleEndian.Uint32(data[offset:]) != recordMagic {
break
}
size := int(binary.LittleEndian.Uint32(data[offset+4:]))
if size < recordHeaderSize+4 || offset+size > free_space ||
int(binary.LittleEndian.Uint32(data[offset+size-4:])) != size {
break
}
last_offset = offset
last_id = binary.LittleEndian.Uint64(data[offset+8:])
count++
offset += size
}
if count == 0 {
return nil
}
// Drop everything after the last intact record.
for i := offset; i < len(data); i++ {
data[i] = 0
}
first_number := binary.LittleEndian.Uint64(data[0x08:])
binary.LittleEndian.PutUint64(data[0x10:], first_number+count-1)
binary.LittleEndian.PutUint64(data[0x20:], last_id)
binary.LittleEndian.PutUint32(data[0x2c:], uint32(last_offset))
binary.LittleEndian.PutUint32(data[chunkFreeSpaceOffset:], uint32(offset))
binary.LittleEndian.PutUint32(data[chunkDataChecksumOffset:],
crc32.ChecksumIEEE(data[evtx.EVTX_CHUNK_HEADER_SIZE:offset]))
return data
}
// The header checksum covers the first 120 bytes and the
// string/template tables, skipping the flags and the checksum.
func setChunkHeaderChecksum(data []byte) {
crc := crc32.NewIEEE()
_, _ = crc.Write(data[:chunkHeaderChecksumStart])
_, _ = crc.Write(data[chunkHeaderChecksumEnd:evtx.EVTX_CHUNK_HEADER_SIZE])
binary.LittleEndian.PutUint32(data[chunkHeaderChecksumStart+4:], crc.Sum32())
}
func (self *evtxWriter) Chunks() uint64 {
return self.chunks
}
// Write the final header. Must be called after the last chunk.
func (self *evtxWriter) Close() error {
return self.writeHeader()
}
func (self *evtxWriter) writeHeader() error {
header := make([]byte, fileHeaderSize)
copy(header, evtx.EVTX_HEADER_MAGIC)
last_chunk := uint64(0)
if self.chunks > 0 {
last_chunk = self.chunks - 1
}
binary.LittleEndian.PutUint64(header[0x08:], 0) // First chunk number
binary.LittleEndian.PutUint64(header[0x10:], last_chunk)
binary.LittleEndian.PutUint64(header[0x18:], self.next_record)
binary.LittleEndian.PutUint32(header[0x20:], fileHeaderUsedSize)
binary.LittleEndian.PutUint16(header[0x24:], 2) // Minor version
binary.LittleEndian.PutUint16(header[0x26:], 3) // Major version
binary.LittleEndian.PutUint16(header[0x28:], fileHeaderSize)
binary.LittleEndian.PutUint16(header[0x2a:], uint16(self.chunks))
// Flags are left clear: the file is not dirty and not full.
binary.LittleEndian.PutUint32(header[fileHeaderChecksumStart+4:],
crc32.ChecksumIEEE(header[:fileHeaderChecksumStart]))
_, err := self.fd.WriteAt(header, 0)
return err
}
+164
View File
@@ -0,0 +1,164 @@
package event_logs
import (
"bytes"
"encoding/binary"
"math/rand"
"os"
"testing"
"www.velocidex.com/golang/evtx"
"www.velocidex.com/golang/velociraptor/vtesting/assert"
)
// An in memory io.WriterAt.
type memWriter struct {
buf []byte
}
func (self *memWriter) WriteAt(p []byte, off int64) (int, error) {
if end := int(off) + len(p); end > len(self.buf) {
self.buf = append(self.buf, make([]byte, end-len(self.buf))...)
}
return copy(self.buf[off:], p), nil
}
func mustCheckChunk(t *testing.T, data []byte) *carvedChunk {
chunk, ok := checkChunk(0, data, len(data))
assert.True(t, ok)
return chunk
}
// Chunks of the second log in the testdata.
func otherLogChunks(t *testing.T) []*carvedChunk {
data, err := os.ReadFile("../../../artifacts/testdata/files/RDPAuth_Security.evtx")
assert.NoError(t, err)
var result []*carvedChunk
for off := 0x1000; off+evtx.EVTX_CHUNK_SIZE <= len(data); off += evtx.EVTX_CHUNK_SIZE {
chunk, ok := checkChunk(int64(off), data[off:off+evtx.EVTX_CHUNK_SIZE],
evtx.EVTX_CHUNK_SIZE)
if ok {
result = append(result, chunk)
}
}
assert.True(t, len(result) > 2)
return result
}
func recordIds(t *testing.T, chunk *carvedChunk) []uint64 {
records, err := parseCarvedChunk(chunk)
assert.NoError(t, err)
var result []uint64
for _, r := range records {
result = append(result, r.Header.RecordID)
}
return result
}
func TestEvtxWriter(t *testing.T) {
intact := mustCheckChunk(t, getTestChunk(t))
others := otherLogChunks(t)
// The records after the middle of the used space are overwritten.
damaged_data := append([]byte{}, others[1].Data...)
free_space := int(binary.LittleEndian.Uint32(damaged_data[chunkFreeSpaceOffset:]))
mid := evtx.EVTX_CHUNK_HEADER_SIZE + (free_space-evtx.EVTX_CHUNK_HEADER_SIZE)/2
copy(damaged_data[mid:free_space], noise(newRng(), free_space-mid))
damaged := mustCheckChunk(t, damaged_data)
assert.False(t, damaged.DataChecksumValid)
// Nothing survives when the first record is destroyed.
hopeless_data := append([]byte{}, others[2].Data...)
copy(hopeless_data[evtx.EVTX_CHUNK_HEADER_SIZE:], "XXXX")
hopeless := mustCheckChunk(t, hopeless_data)
out := &memWriter{}
writer, err := newEvtxWriter(out)
assert.NoError(t, err)
// Chunks from different logs, a duplicate and damaged chunks.
for _, c := range []*carvedChunk{others[0], intact, others[0], damaged, hopeless} {
_, err := writer.WriteChunk(c)
assert.NoError(t, err)
}
assert.NoError(t, writer.Close())
assert.Equal(t, uint64(3), writer.Chunks())
// The library reads the file back.
chunks, err := evtx.GetChunks(bytes.NewReader(out.buf))
assert.NoError(t, err)
assert.Equal(t, 3, len(chunks))
// Every written chunk has valid checksums and the record
// numbers in the chunk headers follow on from each other.
next := uint64(1)
var written []*carvedChunk
for i := 0; i < 3; i++ {
off := fileHeaderSize + i*evtx.EVTX_CHUNK_SIZE
c := mustCheckChunk(t, out.buf[off:off+evtx.EVTX_CHUNK_SIZE])
assert.True(t, c.HeaderChecksumValid, "chunk %v", i)
assert.True(t, c.DataChecksumValid, "chunk %v", i)
assert.Equal(t, next, c.Header.FirstEventRecNumber, "chunk %v", i)
next = c.Header.LastEventRecNumber + 1
written = append(written, c)
}
// Event record ids are kept as they were.
assert.Equal(t, recordIds(t, others[0]), recordIds(t, written[0]))
assert.Equal(t, recordIds(t, intact), recordIds(t, written[1]))
// Only the intact leading records of the damaged chunk are kept.
repaired := recordIds(t, written[2])
original := recordIds(t, others[1])
assert.True(t, len(repaired) > 0)
assert.True(t, len(repaired) < len(original))
assert.Equal(t, original[:len(repaired)], repaired)
// The file header describes the chunks.
header := out.buf[:fileHeaderSize]
assert.Equal(t, evtx.EVTX_HEADER_MAGIC, string(header[:8]))
assert.Equal(t, uint16(3), binary.LittleEndian.Uint16(header[0x2a:]))
assert.Equal(t, uint64(2), binary.LittleEndian.Uint64(header[0x10:]))
}
// The output file may be on the disk being carved, so chunks we
// wrote can be found again by the scan.
func TestEvtxWriterSkipsOwnOutput(t *testing.T) {
others := otherLogChunks(t)
out := &memWriter{}
writer, err := newEvtxWriter(out)
assert.NoError(t, err)
// Written out of order so the second chunk is renumbered.
for _, c := range []*carvedChunk{others[1], others[0]} {
ok, err := writer.WriteChunk(c)
assert.NoError(t, err)
assert.True(t, ok)
}
// The second chunk was renumbered when written, so it differs
// from the original.
off := fileHeaderSize + evtx.EVTX_CHUNK_SIZE
found_again := mustCheckChunk(t, append([]byte{}, out.buf[off:off+evtx.EVTX_CHUNK_SIZE]...))
assert.NotEqual(t, others[0].Data, found_again.Data)
ok, err := writer.WriteChunk(found_again)
assert.NoError(t, err)
assert.False(t, ok)
assert.Equal(t, uint64(2), writer.Chunks())
}
func TestEvtxWriterEmpty(t *testing.T) {
out := &memWriter{}
writer, err := newEvtxWriter(out)
assert.NoError(t, err)
assert.NoError(t, writer.Close())
assert.Equal(t, fileHeaderSize, len(out.buf))
assert.Equal(t, evtx.EVTX_HEADER_MAGIC, string(out.buf[:8]))
}
func newRng() *rand.Rand {
return rand.New(rand.NewSource(11))
}
+24 -1
View File
@@ -207,6 +207,8 @@ func (self *AccessorReader) _Close() error {
self.cancel = nil
reader := self.reader
paged_reader := self.paged_reader
self.reader = nil
self.paged_reader = nil
self.last_opened = time.Time{}
@@ -222,6 +224,14 @@ func (self *AccessorReader) _Close() error {
reader.Close()
}
if paged_reader != nil {
// Report paged_reader efficiency
/*
stats := paged_reader.Stats()
fmt.Printf("%s: %#v\n", self.File.String(), stats)
*/
}
return nil
}
@@ -250,8 +260,21 @@ func (self *AccessorReader) ReadAt(buf []byte, offset int64) (int, error) {
lru_size = 100
}
// Pagesize is a sliding scale to maximize efficiency over
// memory use. If the lru is large, we use small pages, but if
// the lru is small, we use larger pages.
page_size := int64(1024 * 8)
if lru_size < 100 {
// Target < 100Mb
page_size = 1024 * 1024
} else if lru_size < 10 {
// Target < 100Mb
page_size = 10 * 1024 * 1024
}
paged_reader, err := ntfs.NewPagedReader(
utils.MakeReaderAtter(reader), 1024*8, lru_size)
utils.MakeReaderAtter(reader), page_size, lru_size)
if err != nil {
reader.Close()
self.mu.Unlock()