diff --git a/internal/etw/processors/chain_windows.go b/internal/etw/processors/chain_windows.go index 82f3ab47b..f7db20e34 100644 --- a/internal/etw/processors/chain_windows.go +++ b/internal/etw/processors/chain_windows.go @@ -43,15 +43,14 @@ func NewChain( psnapshotter: psnap, processors: make([]Processor, 0), } - devMapper = fs.NewDevMapper() - devPathResolver = fs.NewDevPathResolver() - vaRegionProber = va.NewRegionProber() + devMapper = fs.NewDevMapper() + vaRegionProber = va.NewRegionProber() ) chain.addProcessor(newPsProcessor(psnap, vaRegionProber)) if config.EventSource.EnableFileIOEvents { - chain.addProcessor(newFsProcessor(hsnap, psnap, devMapper, devPathResolver, config)) + chain.addProcessor(newFsProcessor(hsnap, psnap, devMapper, config)) } if config.EventSource.EnableRegistryEvents { chain.addProcessor(newRegistryProcessor(hsnap)) @@ -63,7 +62,7 @@ func NewChain( chain.addProcessor(newNetProcessor()) } if config.EventSource.EnableHandleEvents { - chain.addProcessor(newHandleProcessor(hsnap, psnap, devMapper, devPathResolver)) + chain.addProcessor(newHandleProcessor(hsnap, psnap, devMapper)) } if config.EventSource.EnableMemEvents { chain.addProcessor(newMemProcessor(psnap, vaRegionProber)) diff --git a/internal/etw/processors/fs_windows.go b/internal/etw/processors/fs_windows.go index 304d22af1..1c3fd0223 100644 --- a/internal/etw/processors/fs_windows.go +++ b/internal/etw/processors/fs_windows.go @@ -56,9 +56,8 @@ type fsProcessor struct { // irps contains a mapping between the IRP (I/O request packet) and CreateFile events irps map[uint64]*event.Event - devMapper fs.DevMapper - devPathResolver fs.DevPathResolver - config *config.Config + devMapper fs.DevMapper + config *config.Config // buckets stores stack walk events per stack id buckets map[uint64][]*event.Event @@ -80,21 +79,19 @@ func newFsProcessor( hsnap handle.Snapshotter, psnap ps.Snapshotter, devMapper fs.DevMapper, - devPathResolver fs.DevPathResolver, config *config.Config, ) Processor { f := &fsProcessor{ - files: make(map[uint64]*FileInfo), - irps: make(map[uint64]*event.Event), - hsnap: hsnap, - psnap: psnap, - devMapper: devMapper, - devPathResolver: devPathResolver, - config: config, - buckets: make(map[uint64][]*event.Event), - purger: time.NewTicker(time.Second * 5), - quit: make(chan struct{}, 1), - lim: rate.NewLimiter(30, 40), // allow 30 parse ops per second or bursts of 40 ops + files: make(map[uint64]*FileInfo), + irps: make(map[uint64]*event.Event), + hsnap: hsnap, + psnap: psnap, + devMapper: devMapper, + config: config, + buckets: make(map[uint64][]*event.Event), + purger: time.NewTicker(time.Second * 5), + quit: make(chan struct{}, 1), + lim: rate.NewLimiter(30, 40), // allow 30 parse ops per second or bursts of 40 ops } go f.purge() @@ -207,10 +204,6 @@ func (f *fsProcessor) processEvent(e *event.Event) (*event.Event, error) { f.files[fileObject] = fileinfo } - if f.config.EventSource.EnableHandleEvents { - f.devPathResolver.AddPath(ev.GetParamAsString(params.FilePath)) - } - ev.AppendParam(params.NTStatus, params.Status, status) if fileinfo.Type != fs.Unknown { ev.AppendEnum(params.FileType, uint32(fileinfo.Type), fs.FileTypes) diff --git a/internal/etw/processors/fs_windows_test.go b/internal/etw/processors/fs_windows_test.go index d8de0d9bb..7efe7e548 100644 --- a/internal/etw/processors/fs_windows_test.go +++ b/internal/etw/processors/fs_windows_test.go @@ -306,7 +306,7 @@ func TestFsProcessor(t *testing.T) { {File: "C:\\Windows\\System32\\kernel32.dll", BaseAddress: va.Address(0xffff23433), Size: 3098}, }, }) - p := newFsProcessor(hsnap, psnap, fs.NewDevMapper(), fs.NewDevPathResolver(), &config.Config{}) + p := newFsProcessor(hsnap, psnap, fs.NewDevMapper(), &config.Config{}) if tt.setupProcessor != nil { tt.setupProcessor(p) } diff --git a/internal/etw/processors/handle_windows.go b/internal/etw/processors/handle_windows.go index e25f217b7..ea01de038 100644 --- a/internal/etw/processors/handle_windows.go +++ b/internal/etw/processors/handle_windows.go @@ -19,8 +19,6 @@ package processors import ( - "strings" - "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" "github.com/rabbitstack/fibratus/pkg/fs" @@ -30,23 +28,20 @@ import ( ) type handleProcessor struct { - hsnap handle.Snapshotter - psnap ps.Snapshotter - devMapper fs.DevMapper - devPathResolver fs.DevPathResolver + hsnap handle.Snapshotter + psnap ps.Snapshotter + devMapper fs.DevMapper } func newHandleProcessor( hsnap handle.Snapshotter, psnap ps.Snapshotter, devMapper fs.DevMapper, - devPathResolver fs.DevPathResolver, ) Processor { return &handleProcessor{ - hsnap: hsnap, - psnap: psnap, - devMapper: devMapper, - devPathResolver: devPathResolver, + hsnap: hsnap, + psnap: psnap, + devMapper: devMapper, } } @@ -86,14 +81,6 @@ func (h *handleProcessor) processEvent(e *event.Event) (*event.Event, error) { } case handle.File: name = h.devMapper.Convert(name) - case handle.Driver: - driverName := strings.TrimPrefix(name, "\\Driver\\") + ".sys" - driverPath := h.devPathResolver.GetPath(driverName) - if driverPath == "" { - driverPath = driverName - } - h.devPathResolver.RemovePath(driverName) - e.Params.Append(params.ModulePath, params.Path, driverPath) } // assign the formatted handle name if err := e.Params.SetValue(params.HandleObjectName, name); err != nil { diff --git a/internal/etw/processors/handle_windows_test.go b/internal/etw/processors/handle_windows_test.go index c3b6b437d..d7f09138c 100644 --- a/internal/etw/processors/handle_windows_test.go +++ b/internal/etw/processors/handle_windows_test.go @@ -19,6 +19,8 @@ package processors import ( + "testing" + "github.com/rabbitstack/fibratus/pkg/event" "github.com/rabbitstack/fibratus/pkg/event/params" "github.com/rabbitstack/fibratus/pkg/fs" @@ -27,7 +29,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "testing" ) func TestHandleProcessor(t *testing.T) { @@ -92,7 +93,7 @@ func TestHandleProcessor(t *testing.T) { t.Run(tt.name, func(t *testing.T) { hsnap := tt.hsnap() psnap := new(ps.SnapshotterMock) - p := newHandleProcessor(hsnap, psnap, fs.NewDevMapper(), fs.NewDevPathResolver()) + p := newHandleProcessor(hsnap, psnap, fs.NewDevMapper()) var err error tt.e, _, err = p.ProcessEvent(tt.e) require.NoError(t, err) diff --git a/pkg/callstack/colorize.go b/pkg/callstack/colorize.go index 811adb352..c49f7a1cc 100644 --- a/pkg/callstack/colorize.go +++ b/pkg/callstack/colorize.go @@ -77,7 +77,7 @@ func (s Callstack) Colorize() string { // frames in kernel range with no resolved symbol are unresolved so // we can collapse them into a counter - if f.Addr.InSystemRange() && (f.Symbol == "" || f.Symbol == "?") { + if f.Addr.InSystemRange() && ((f.Symbol == "" || f.Symbol == "?") && f.Module == "") { unresolved++ continue } diff --git a/pkg/fs/driver.go b/pkg/fs/driver.go deleted file mode 100644 index ae5bb3233..000000000 --- a/pkg/fs/driver.go +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2021-2022 by Nedim Sabic Sabic - * https://www.fibratus.io - * All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package fs - -import ( - "github.com/rabbitstack/fibratus/pkg/sys" - "path/filepath" - "strings" -) - -// DevPathResolver resolves driver module paths from driver names. -// Prior to loading/unloading the kernel driver, the file object -// associated to it is opened. This gives us the opportunity to record -// the full path of the driver module and use it to augment events -// with this extra parameter. -type DevPathResolver struct { - paths map[string]string -} - -// NewDevPathResolver returns a new instance of driver device path resolver -func NewDevPathResolver() DevPathResolver { - return DevPathResolver{paths: make(map[string]string)} -} - -// AddPath adds the driver module path to the state of opened/created driver files. -func (d *DevPathResolver) AddPath(filename string) { - isDriver := strings.EqualFold(filepath.Ext(filename), ".sys") - if isDriver { - d.paths[strings.ToLower(filepath.Base(filename))] = filename - } -} - -// RemovePath removes driver path from the state. -func (d *DevPathResolver) RemovePath(driver string) { - delete(d.paths, driver) -} - -// GetPath returns the full path to the driver module file. This method -// first perform a lookup in the opened/created driver modules. If the module -// is not found, then we enumerate all drivers and try to find the matching -// driver module path. -func (d *DevPathResolver) GetPath(driver string) string { - path, ok := d.paths[strings.ToLower(driver)] - if ok { - return path - } - drivers := sys.EnumDevices() - for _, drv := range drivers { - if strings.EqualFold(strings.ToLower(filepath.Base(drv.Filename)), driver) { - return drv.Filename - } - } - return "" -} diff --git a/pkg/symbolize/driver.go b/pkg/symbolize/driver.go new file mode 100644 index 000000000..5a041b3bf --- /dev/null +++ b/pkg/symbolize/driver.go @@ -0,0 +1,97 @@ +/* + * Copyright 2021-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package symbolize + +import ( + "sync" + + "github.com/rabbitstack/fibratus/pkg/sys" + "github.com/rabbitstack/fibratus/pkg/util/va" +) + +type driverStore struct { + devs []sys.Driver + // drivers maps resolved kernel addresses to the driver objects + drivers map[va.Address]*sys.Driver + mux sync.RWMutex +} + +func initDriverStore() *driverStore { + return &driverStore{ + devs: sys.EnumDevices(), + drivers: make(map[va.Address]*sys.Driver), + } +} + +// resolve maps a kernel return address to a driver. +// If the kernel address is already resolved, then +// then the driver object is recovered from the cache. +// Returns nil if no module contains the address. +func (d *driverStore) resolve(addr va.Address) *sys.Driver { + // driver already cached? + d.mux.RLock() + dev, isCached := d.drivers[addr] + d.mux.RUnlock() + if isCached { + return dev + } + + d.mux.Lock() + defer d.mux.Unlock() + for i := range d.devs { + dev := &d.devs[i] + base := va.Address(dev.Base) + if addr >= base && addr < base.Inc(uint64(dev.Size)) { + d.drivers[addr] = dev + return dev + } + } + + return nil +} + +func (d *driverStore) addDriver(base va.Address, size uint64, path string) { + d.mux.Lock() + defer d.mux.Unlock() + + dev := sys.Driver{ + Path: path, + Base: base.Uintptr(), + Size: uint32(size), + } + d.devs = append(d.devs, dev) +} + +func (d *driverStore) removeDriver(base va.Address, size uint64) { + d.mux.Lock() + defer d.mux.Unlock() + + for i, dev := range d.devs { + if dev.Base == base.Uintptr() { + d.devs = append(d.devs[:i], d.devs[i+1:]...) + break + } + } + + for addr := range d.drivers { + if addr >= base && addr < base.Inc(size) { + delete(d.drivers, addr) + } + } +} diff --git a/pkg/symbolize/driver_test.go b/pkg/symbolize/driver_test.go new file mode 100644 index 000000000..a2b6389ff --- /dev/null +++ b/pkg/symbolize/driver_test.go @@ -0,0 +1,258 @@ +/* + * Copyright 2021-present by Nedim Sabic Sabic + * https://www.fibratus.io + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package symbolize + +import ( + "sync" + "testing" + + "github.com/rabbitstack/fibratus/pkg/sys" + "github.com/rabbitstack/fibratus/pkg/util/va" +) + +func makeStore(devs []sys.Driver) *driverStore { + return &driverStore{ + devs: devs, + drivers: make(map[va.Address]*sys.Driver), + } +} + +// Real-ish kernel driver addresses +const ( + ntosBase va.Address = 0xFFFFF80000000000 + ntosSize = 0x800000 // 8 MB + + halBase va.Address = 0xFFFFF80008000000 + halSize = 0x100000 // 1 MB +) + +func realDrivers() []sys.Driver { + return []sys.Driver{ + {Path: `\SystemRoot\system32\ntoskrnl.exe`, Base: ntosBase.Uintptr(), Size: ntosSize}, + {Path: `\SystemRoot\system32\hal.dll`, Base: halBase.Uintptr(), Size: halSize}, + } +} + +func TestResolveHitsFirstDriver(t *testing.T) { + ds := makeStore(realDrivers()) + + addr := ntosBase + 0x1000 // well inside ntoskrnl + drv := ds.resolve(addr) + + if drv == nil { + t.Fatal("expected driver, got nil") + } + if drv.Path != `\SystemRoot\system32\ntoskrnl.exe` { + t.Errorf("wrong driver: %s", drv.Path) + } +} + +func TestResolveHitsSecondDriver(t *testing.T) { + ds := makeStore(realDrivers()) + + addr := halBase + 0x500 + drv := ds.resolve(addr) + + if drv == nil { + t.Fatal("expected driver, got nil") + } + if drv.Path != `\SystemRoot\system32\hal.dll` { + t.Errorf("wrong driver: %s", drv.Path) + } +} + +func TestResolveExactBaseAddress(t *testing.T) { + ds := makeStore(realDrivers()) + drv := ds.resolve(ntosBase) + if drv == nil { + t.Fatal("base address itself must resolve") + } +} + +func TestResolveLastByteInRange(t *testing.T) { + ds := makeStore(realDrivers()) + last := ntosBase + va.Address(ntosSize) - 1 + drv := ds.resolve(last) + if drv == nil { + t.Fatal("last byte inside range must resolve") + } +} + +func TestResolveOneBeyondEndReturnsNil(t *testing.T) { + ds := makeStore(realDrivers()) + beyond := ntosBase + va.Address(ntosSize) // exclusive upper bound + drv := ds.resolve(beyond) + if drv != nil { + t.Errorf("address beyond driver range should return nil, got %s", drv.Path) + } +} + +func TestResolveUnknownAddressReturnsNil(t *testing.T) { + ds := makeStore(realDrivers()) + drv := ds.resolve(0x0000000000001234) // user-space address + if drv != nil { + t.Errorf("unknown address should return nil, got %s", drv.Path) + } +} + +func TestResolveCacheHit(t *testing.T) { + ds := makeStore(realDrivers()) + addr := ntosBase + 0x4000 + + first := ds.resolve(addr) + if first == nil { + t.Fatal("first resolve failed") + } + + // wipe devs so a miss would return nil to prove the cache is used + ds.mux.Lock() + ds.devs = nil + ds.mux.Unlock() + + second := ds.resolve(addr) + if second == nil { + t.Fatal("second resolve (cache hit) returned nil") + } + if first != second { + t.Error("cache hit must return the same pointer") + } +} + +func TestResolveConcurrentReads(t *testing.T) { + ds := makeStore(realDrivers()) + addr := ntosBase + 0x2000 + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + drv := ds.resolve(addr) + if drv == nil { + t.Errorf("concurrent resolve returned nil") + } + }() + } + wg.Wait() +} + +func TestAddDriverResolvesAfterAdd(t *testing.T) { + ds := makeStore(nil) // start empty + + const base va.Address = 0xFFFFF90000000000 + const size uint64 = 0x200000 + const path = `\Driver\custom.sys` + + ds.addDriver(base, size, path) + + drv := ds.resolve(base + 0x100) + if drv == nil { + t.Fatal("newly added driver must resolve") + } + if drv.Path != path { + t.Errorf("got path %s, want %s", drv.Path, path) + } +} + +func TestAddDriverDoesNotAffectOtherRanges(t *testing.T) { + ds := makeStore(realDrivers()) + + const newBase va.Address = 0xFFFFF90000000000 + ds.addDriver(newBase, 0x10000, `\Driver\extra.sys`) + + // original drivers still work + if ds.resolve(ntosBase+0x500) == nil { + t.Error("existing driver should still resolve after addDriver") + } + // Unrelated address still nil. + if ds.resolve(0x1) != nil { + t.Error("unrelated address should still return nil") + } +} + +func TestRemoveDriverNoLongerResolves(t *testing.T) { + ds := makeStore(realDrivers()) + + // confirm it resolves before removal. + if ds.resolve(ntosBase+0x1000) == nil { + t.Fatal("precondition: driver must resolve before removal") + } + + ds.removeDriver(ntosBase, ntosSize) + + if ds.resolve(ntosBase+0x1000) != nil { + t.Error("driver must not resolve after removal") + } +} + +func TestRemoveDriverClearsCache(t *testing.T) { + ds := makeStore(realDrivers()) + addr := ntosBase + 0x3000 + + // populate the cache + ds.resolve(addr) + + ds.removeDriver(ntosBase, ntosSize) + + // cache entry must be gone; since devs no longer contains the driver, + // this must return nil rather than the stale cached pointer. + if ds.resolve(addr) != nil { + t.Error("cache must be invalidated after removeDriver") + } +} + +func TestRemoveDriverLeavesOtherDriversIntact(t *testing.T) { + ds := makeStore(realDrivers()) + ds.removeDriver(ntosBase, ntosSize) + + drv := ds.resolve(halBase + 0x100) + if drv == nil { + t.Fatal("hal driver should survive ntoskrnl removal") + } + if drv.Path != `\SystemRoot\system32\hal.dll` { + t.Errorf("wrong driver after partial removal: %s", drv.Path) + } +} + +func TestRemoveDriverNonExistentBase_NoOp(t *testing.T) { + ds := makeStore(realDrivers()) + before := len(ds.devs) + + ds.removeDriver(0xDEAD000000000000, 0x1000) // not in the store + + if len(ds.devs) != before { + t.Errorf("devs length changed after removing non-existent driver: %d → %d", before, len(ds.devs)) + } +} + +func TestAddRemoveRoundTrip(t *testing.T) { + ds := makeStore(nil) + + const base va.Address = 0xFFFFF80010000000 + const size uint64 = 0x50000 + + ds.addDriver(base, size, `\Driver\roundtrip.sys`) + ds.resolve(base + 0x100) // fill cache + + ds.removeDriver(base, size) + + if ds.resolve(base+0x100) != nil { + t.Error("driver must be gone after round-trip removal") + } +} diff --git a/pkg/symbolize/symbolizer.go b/pkg/symbolize/symbolizer.go index fcda9e154..8b30304b3 100644 --- a/pkg/symbolize/symbolizer.go +++ b/pkg/symbolize/symbolizer.go @@ -132,6 +132,11 @@ type Symbolizer struct { // exps stores resolved export directories exps *ExportsDirectoryCache + // drivers stores loaded kernel drivers and + // allows resolution of kernel addresses to + // kernel drivers + drivers *driverStore + r Resolver psnap ps.Snapshotter @@ -157,6 +162,7 @@ func NewSymbolizer(r Resolver, psnap ps.Snapshotter, config *config.Config, enqu quit: make(chan struct{}, 1), enqueue: enqueue, exps: NewExportsDirectoryCache(psnap), + drivers: initDriverStore(), r: r, psnap: psnap, } @@ -169,7 +175,7 @@ func NewSymbolizer(r Resolver, psnap ps.Snapshotter, config *config.Config, enqu } devs := sys.EnumDevices() for _, dev := range devs { - _ = r.LoadModule(windows.CurrentProcess(), dev.Filename, va.Address(dev.Addr)) + _ = r.LoadModule(windows.CurrentProcess(), dev.Path, va.Address(dev.Base)) } } @@ -189,7 +195,7 @@ func (s *Symbolizer) Close() { s.cleanAllSyms() if s.config.SymbolizeKernelAddresses { for _, dev := range sys.EnumDevices() { - s.r.UnloadModule(windows.CurrentProcess(), va.Address(dev.Addr)) + s.r.UnloadModule(windows.CurrentProcess(), va.Address(dev.Base)) } } } @@ -216,19 +222,28 @@ func (s *Symbolizer) ProcessEvent(e *event.Event) (bool, error) { } if e.IsLoadModule() || e.IsUnloadModule() { - filename := e.GetParamAsString(params.ModulePath) + path := e.GetParamAsString(params.ModulePath) + size := e.Params.TryGetUint64(params.ModuleSize) addr := e.Params.TryGetAddress(params.ModuleBase) // if the kernel driver is loaded or unloaded, - // load/unload symbol handlers respectively - if (strings.ToLower(filepath.Ext(filename)) == ".sys" || - e.Params.TryGetBool(params.FileIsDriver)) && s.config.SymbolizeKernelAddresses { + // load/unload symbol handlers respectively or + // update the driver store + if strings.ToLower(filepath.Ext(path)) == ".sys" || e.Params.TryGetBool(params.FileIsDriver) { if e.IsLoadModule() { - err := s.r.LoadModule(windows.CurrentProcess(), filename, addr) - if err != nil { - log.Errorf("unable to load symbol table for %s module: %v", filename, err) + if s.config.SymbolizeKernelAddresses { + err := s.r.LoadModule(windows.CurrentProcess(), path, addr) + if err != nil { + log.Errorf("unable to load symbol table for %s module: %v", path, err) + } + } else { + s.drivers.addDriver(addr, size, path) } } else { - s.r.UnloadModule(windows.CurrentProcess(), addr) + if s.config.SymbolizeKernelAddresses { + s.r.UnloadModule(windows.CurrentProcess(), addr) + } else { + s.drivers.removeDriver(addr, size) + } } } @@ -424,8 +439,14 @@ func (s *Symbolizer) produceFrame(addr va.Address, e *event.Event) callstack.Fra frame := callstack.Frame{PID: pid, Addr: addr} if addr.InSystemRange() { if s.config.SymbolizeKernelAddresses { + // full kernel address symbolization frame.Module = s.r.GetModuleName(windows.CurrentProcess(), addr) frame.Symbol, frame.Offset = s.r.GetSymbolNameAndOffset(windows.CurrentProcess(), addr) + } else { + // module-only symbolization + if dev := s.drivers.resolve(addr); dev != nil { + frame.Module = dev.Path + } } return frame } diff --git a/pkg/symbolize/symbolizer_test.go b/pkg/symbolize/symbolizer_test.go index 97b908b04..f9d4b2ddb 100644 --- a/pkg/symbolize/symbolizer_test.go +++ b/pkg/symbolize/symbolizer_test.go @@ -304,6 +304,104 @@ func TestProcessCallstack(t *testing.T) { assert.Equal(t, 0, s.procsSize()) } +func TestKernelCallstackSymbolizationFromDriverStore(t *testing.T) { + r := new(MockResolver) + c := &config.Config{ + SymbolizeKernelAddresses: false, + } + psnap := new(ps.SnapshotterMock) + + s := NewSymbolizer(r, psnap, c, false) + require.NotNil(t, s) + + // Inject real-ish kernel driver ranges directly into the symbolizer's driver store. + // These mimic what EnumDevices/NtQuerySystemInformation would return at runtime. + // + // ntoskrnl.exe 0xFFFFF80000000000 – 0xFFFFF80000800000 (8 MB) + // hal.dll 0xFFFFF80000800000 – 0xFFFFF80000900000 (1 MB) + // fltMgr.sys 0xFFFFF80001000000 – 0xFFFFF80001080000 (512 KB) + // ksecdd.sys 0xFFFFF80001200000 – 0xFFFFF80001240000 (256 KB) + s.drivers.addDriver(0xFFFFF80000000000, 0x800000, `\SystemRoot\system32\ntoskrnl.exe`) + s.drivers.addDriver(0xFFFFF80000800000, 0x100000, `\SystemRoot\system32\hal.dll`) + s.drivers.addDriver(0xFFFFF80001000000, 0x080000, `\SystemRoot\system32\drivers\fltMgr.sys`) + s.drivers.addDriver(0xFFFFF80001200000, 0x040000, `\SystemRoot\system32\drivers\ksecdd.sys`) + + proc := &pstypes.PS{ + Name: "notepad.exe", + PID: 23234, + Ppid: 2434, + Exe: `C:\Windows\notepad.exe`, + Cmdline: `C:\Windows\notepad.exe`, + SID: "S-1-1-18", + Cwd: `C:\Windows\`, + SessionID: 1, + Threads: map[uint32]pstypes.Thread{ + 3453: { + Tid: 3453, + StartAddress: va.Address(140729524944768), + IOPrio: 2, + PagePrio: 5, + KstackBase: va.Address(18446677035730165760), + KstackLimit: va.Address(18446677035730137088), + UstackLimit: va.Address(86376448), + UstackBase: va.Address(86372352), + }, + }, + Envs: map[string]string{"ProgramData": "C:\\ProgramData", "COMPUTRENAME": "archrabbit"}, + } + + // callstack mixes kernel addresses (0xFFFFF800...) with a user-space tail. + // Each kernel address falls inside one of the driver ranges injected above + e := &event.Event{ + Type: event.CreateProcess, + Tid: 2484, + PID: 2232, + CPU: 1, + Seq: 2, + Name: "CreatedProcess", + Timestamp: time.Now(), + Category: event.Process, + Host: "archrabbit", + Params: event.Params{ + params.ProcessParentID: {Name: params.ProcessParentID, Type: params.PID, Value: uint32(os.Getpid())}, + params.ProcessRealParentID: {Name: params.ProcessRealParentID, Type: params.PID, Value: uint32(os.Getpid())}, + params.Callstack: { + Name: params.Callstack, + Type: params.Slice, + Value: []va.Address{ + 0xFFFFF80000100000, // ntoskrnl.exe + 0xFFFFF80000810000, // hal.dll + 0xFFFFF80001010000, // fltMgr.sys + 0xFFFFF80001210000, // ksecdd.sys + }, + }, + }, + PS: proc, + } + + _, err := s.ProcessEvent(e) + require.NoError(t, err) + + // each frame must carry the driver path derived from the driver store, + // not a user-space module name, paired with the symbol the mock returned. + assert.Equal(t, + `0xfffff80000100000 \SystemRoot\system32\ntoskrnl.exe!?`+ + `|0xfffff80000810000 \SystemRoot\system32\hal.dll!?`+ + `|0xfffff80001010000 \SystemRoot\system32\drivers\fltMgr.sys!?`+ + `|0xfffff80001210000 \SystemRoot\system32\drivers\ksecdd.sys!?`, + e.Callstack.String(), + ) + + // verify each frame individually for clearer failure messages + frames := e.Callstack + require.Len(t, frames, 4) + + assert.Equal(t, `\SystemRoot\system32\ntoskrnl.exe`, frames[3].Module) + assert.Equal(t, `\SystemRoot\system32\hal.dll`, frames[2].Module) + assert.Equal(t, `\SystemRoot\system32\drivers\fltMgr.sys`, frames[1].Module) + assert.Equal(t, `\SystemRoot\system32\drivers\ksecdd.sys`, frames[0].Module) +} + func TestSymbolizeEventParamAddress(t *testing.T) { r := new(MockResolver) c := &config.Config{} diff --git a/pkg/sys/device.go b/pkg/sys/device.go index c12f8b92b..7792f3fdc 100644 --- a/pkg/sys/device.go +++ b/pkg/sys/device.go @@ -22,60 +22,86 @@ import ( "fmt" "os" "strings" - "syscall" "unsafe" + + "golang.org/x/sys/windows" ) -// DevSize specifies the initial size used to allocate the driver base addresses -const DevSize = 1024 +// devSize specifies the initial size used to allocate the drivers info buffer +const devSize uint32 = 1024 * 256 // Driver contains device driver metadata for each driver found in // the system. type Driver struct { - Filename string - Addr uintptr + Path string + Base uintptr + Size uint32 +} + +// RTL_PROCESS_MODULE_INFORMATION mirrors the C struct +type RTL_PROCESS_MODULE_INFORMATION struct { + Section uintptr + MappedBase uintptr + ImageBase uintptr + ImageSize uint32 + Flags uint32 + LoadOrderIndex uint16 + InitOrderIndex uint16 + LoadCount uint16 + OffsetToFileName uint16 + FullPathName [256]byte +} + +// RTL_PROCESS_MODULES is the header returned by NtQuerySystemInformation +type RTL_PROCESS_MODULES struct { + NumberOfModules uint32 + Modules [1]RTL_PROCESS_MODULE_INFORMATION // variable-length, treated as start of array } // String returns the driver string representation. func (d Driver) String() string { - return fmt.Sprintf("File: %s", d.Filename) + return fmt.Sprintf("Path: %s, Base: %x", d.Path, d.Base) } // EnumDevices returns metadata about device drivers encountered in the // system. If device driver enumeration fails, an empty slice with device // information is returned. func EnumDevices() []Driver { - needed := uint32(0) - addrs := make([]uintptr, DevSize) - err := EnumDeviceDrivers(uintptr(unsafe.Pointer(&addrs[0])), DevSize, &needed) - if err != nil { - return nil - } - // base image size greater than initial allocation - if needed > uint32(len(addrs)) { - addrs = make([]uintptr, needed) - err := EnumDeviceDrivers(uintptr(unsafe.Pointer(&addrs[0])), needed, &needed) + var length uint32 + buf := make([]byte, devSize) // 256 KB initial guess + + for { + err := windows.NtQuerySystemInformation(windows.SystemModuleInformation, unsafe.Pointer(&buf[0]), uint32(len(buf)), &length) + if err == windows.STATUS_INFO_LENGTH_MISMATCH || err == windows.STATUS_BUFFER_TOO_SMALL || err == windows.STATUS_BUFFER_OVERFLOW { + buf = make([]byte, length) + continue + } if err != nil { return nil } - } - // resize to get the number of drivers - if needed/8 < uint32(len(addrs)) { - addrs = addrs[:needed/8] - } - drivers := make([]Driver, len(addrs)) - for i, addr := range addrs { - drv := Driver{ - Addr: addr, - } - filename := make([]uint16, syscall.MAX_PATH) - n := GetDeviceDriverFileName(addr, &filename[0], syscall.MAX_PATH) - if n == 0 { - continue + + // parse the buffer with driver information + header := (*RTL_PROCESS_MODULES)(unsafe.Pointer(&buf[0])) + count := header.NumberOfModules + + offset := unsafe.Offsetof(header.Modules) + size := unsafe.Sizeof(RTL_PROCESS_MODULE_INFORMATION{}) + + devs := make([]Driver, 0, count) + + for i := range count { + m := (*RTL_PROCESS_MODULE_INFORMATION)(unsafe.Pointer(&buf[offset+uintptr(i)*size])) + path := windows.ByteSliceToString(m.FullPathName[:]) + // normalize driver path + path = strings.Replace(path, "\\SystemRoot", os.Getenv("SYSTEMROOT"), 1) + dev := Driver{ + Base: m.ImageBase, + Size: m.ImageSize, + Path: path, + } + devs = append(devs, dev) } - dev := syscall.UTF16ToString(filename) - drv.Filename = strings.Replace(dev, "\\SystemRoot", os.Getenv("SYSTEMROOT"), 1) - drivers[i] = drv + + return devs } - return drivers } diff --git a/pkg/sys/device_test.go b/pkg/sys/device_test.go index 704dab830..3fad62088 100644 --- a/pkg/sys/device_test.go +++ b/pkg/sys/device_test.go @@ -19,11 +19,12 @@ package sys import ( - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "path/filepath" "strings" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEnumDevices(t *testing.T) { @@ -32,7 +33,7 @@ func TestEnumDevices(t *testing.T) { ntoskrnlFound := false for _, drv := range drivers { - if strings.EqualFold(filepath.Base(drv.Filename), "ntoskrnl.exe") { + if strings.EqualFold(filepath.Base(drv.Path), "ntoskrnl.exe") { ntoskrnlFound = true break }