Skip to content

Commit f1c6822

Browse files
misc0110misc0110
authored
Merge pull request #41 from d-we/master
Fix TLB invalidation for PID != 0
2 parents 07e58e6 + eee0160 commit f1c6822

File tree

6 files changed

+96
-16
lines changed

6 files changed

+96
-16
lines changed

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,8 @@ System Info | Descriptions
139139

140140
TLB/Barriers | Descriptions
141141
--------------------------------|---------------------------------------------
142-
`void `[`ptedit_invalidate_tlb`](#group__BARRIERS_1gad2d64fa589bc626ba41ccf18c60d159f)`(void * address)` | Invalidates the TLB for a given address on all CPUs.
142+
`void `[`ptedit_invalidate_tlb`](#group__BARRIERS_1gad2d64fa589bc626ba41ccf18c60d159f)`(void * address)` | Invalidates the TLB entry of current process for a given address on all CPUs.
143+
`void `[`ptedit_invalidate_tlb_pid`](#group__BARRIERS_1gad2d64fa589bc626ba41ccf18c60d159f)`(pid_t pid, void * address)` | Invalidates the TLB for a given PID and address on all CPUs.
143144
`void `[`ptedit_full_serializing_barrier`](#group__BARRIERS_1ga35efff6b34856596b467ef3a5075adc6)`()` | A full serializing barrier which stops everything.
144145

145146
Memory types (PATs/MAIRs) | Descriptions

‎module/pteditor.c‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ typedef struct {
134134
static bool device_busy = false;
135135
static bool mm_is_locked = false;
136136

137-
void (*invalidate_tlb)(unsigned long);
137+
void (*invalidate_tlb)(pid_t, void*);
138138
void (*flush_tlb_mm_range_func)(struct mm_struct*, unsigned long, unsigned long, unsigned int, bool);
139139
void (*native_write_cr4_func)(unsigned long);
140140
static struct mm_struct* get_mm(size_t);
@@ -203,8 +203,8 @@ _invalidate_tlb(void *addr) {
203203
}
204204

205205
static void
206-
invalidate_tlb_custom(unsigned long addr) {
207-
on_each_cpu(_invalidate_tlb, (void*) addr, 1);
206+
invalidate_tlb_custom(pid_t pid, void* addr) {
207+
on_each_cpu(_invalidate_tlb, addr, 1);
208208
}
209209

210210
#if defined(__aarch64__)
@@ -220,9 +220,9 @@ void _flush_tlb_page_smp(void* info) {
220220
#endif
221221

222222
static void
223-
invalidate_tlb_kernel(unsigned long addr) {
223+
invalidate_tlb_kernel(pid_t pid, void* addr) {
224224
#if defined(__i386__) || defined(__x86_64__)
225-
flush_tlb_mm_range_func(get_mm(task_pid_nr(current)), addr, addr + real_page_size, real_page_shift, false);
225+
flush_tlb_mm_range_func(get_mm(pid), (unsigned long) addr, (unsigned long) addr + real_page_size, real_page_shift, false);
226226
#elif defined(__aarch64__)
227227
struct vm_area_struct *vma = find_vma(current->mm, addr);
228228
tlb_page_t tlb_page;
@@ -421,7 +421,7 @@ static int update_vm(ptedit_entry_t* new_entry, int lock) {
421421
set_pte(old_entry.pte, native_make_pte(new_entry->pte));
422422
}
423423

424-
invalidate_tlb(addr);
424+
invalidate_tlb(old_entry.pid, (void*) addr);
425425

426426
/* Unlock mm */
427427
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
@@ -585,9 +585,20 @@ static long device_ioctl(struct file *file, unsigned int ioctl_num, unsigned lon
585585
}
586586
case PTEDITOR_IOCTL_CMD_GET_PAGESIZE:
587587
return real_page_size;
588+
case PTEDITOR_IOCTL_CMD_INVALIDATE_TLB_PID:
589+
{
590+
ptedit_invalidate_tlb_args_t args;
591+
(void)from_user(&args, (void*)ioctl_param, sizeof(args));
592+
invalidate_tlb(args.pid, args.address);
593+
return 0;
594+
}
588595
case PTEDITOR_IOCTL_CMD_INVALIDATE_TLB:
589-
invalidate_tlb(ioctl_param);
596+
{
597+
// this is implemented as its own call to stay backwards compatible
598+
// even in case a user uses the old ioctl calls
599+
invalidate_tlb(task_pid_nr(current), (void*) ioctl_param);
590600
return 0;
601+
}
591602
case PTEDITOR_IOCTL_CMD_GET_PAT:
592603
{
593604
#if defined(__i386__) || defined(__x86_64__)

‎module/pteditor.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,14 @@ typedef struct {
9090
size_t root;
9191
} ptedit_paging_t;
9292

93+
/**
94+
* Structure to hold the arguments for TLB invalidation
95+
*/
96+
typedef struct {
97+
pid_t pid;
98+
void* address;
99+
} ptedit_invalidate_tlb_args_t;
100+
93101
#define PTEDIT_VALID_MASK_PGD (1<<0)
94102
#define PTEDIT_VALID_MASK_P4D (1<<1)
95103
#define PTEDIT_VALID_MASK_PUD (1<<2)
@@ -140,6 +148,9 @@ typedef struct {
140148

141149
#define PTEDITOR_IOCTL_CMD_SWITCH_TLB_INVALIDATION \
142150
_IOR(PTEDITOR_IOCTL_MAGIC_NUMBER, 13, size_t)
151+
152+
#define PTEDITOR_IOCTL_CMD_INVALIDATE_TLB_PID \
153+
_IOR(PTEDITOR_IOCTL_MAGIC_NUMBER, 14, size_t)
143154
#else
144155
#define PTEDITOR_READ_PAGE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x801, METHOD_BUFFERED, FILE_ANY_ACCESS)
145156
#define PTEDITOR_WRITE_PAGE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_READ_DATA)

‎ptedit.c‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -275,20 +275,20 @@ ptedit_fnc void ptedit_update_user_ext(void* address, pid_t pid, ptedit_entry_t*
275275
pset(root + pgdi * ptedit_entry_size, vm->pgd);
276276
}
277277

278-
ptedit_invalidate_tlb(address);
278+
ptedit_invalidate_tlb_pid(pid, address);
279279
}
280280

281281
// ---------------------------------------------------------------------------
282282
static void ptedit_update_user(void* address, pid_t pid, ptedit_entry_t* vm) {
283283
ptedit_update_user_ext(address, pid, vm, ptedit_phys_write_pwrite);
284-
ptedit_invalidate_tlb(address);
284+
ptedit_invalidate_tlb_pid(pid, address);
285285
}
286286

287287

288288
// ---------------------------------------------------------------------------
289289
static void ptedit_update_user_map(void* address, pid_t pid, ptedit_entry_t* vm) {
290290
ptedit_update_user_ext(address, pid, vm, ptedit_phys_write_map);
291-
ptedit_invalidate_tlb(address);
291+
ptedit_invalidate_tlb_pid(pid, address);
292292
}
293293

294294
// ---------------------------------------------------------------------------
@@ -642,9 +642,24 @@ void ptedit_set_paging_root(pid_t pid, size_t root) {
642642
#endif
643643
}
644644

645+
// ---------------------------------------------------------------------------
646+
ptedit_fnc void ptedit_invalidate_tlb_pid(pid_t pid, void* address) {
647+
#if defined(LINUX)
648+
ptedit_invalidate_tlb_args_t args;
649+
args.pid = pid;
650+
args.address = address;
651+
ioctl(ptedit_fd, PTEDITOR_IOCTL_CMD_INVALIDATE_TLB_PID, (size_t)&args, pid);
652+
#else
653+
size_t vaddr = (size_t)address;
654+
DWORD returnLength;
655+
DeviceIoControl(ptedit_fd, PTEDITOR_FLUSH_TLB, (LPVOID)&vaddr, sizeof(vaddr), (LPVOID)&vaddr, sizeof(vaddr), &returnLength, 0);
656+
#endif
657+
}
645658

646659
// ---------------------------------------------------------------------------
647660
ptedit_fnc void ptedit_invalidate_tlb(void* address) {
661+
// we do not directly call ptedit_invalidate_tlb_pid to ensure that the old
662+
// API is still working (for backwards compatibility)
648663
#if defined(LINUX)
649664
ioctl(ptedit_fd, PTEDITOR_IOCTL_CMD_INVALIDATE_TLB, (size_t)address);
650665
#else

‎ptedit.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -632,13 +632,21 @@ ptedit_fnc void ptedit_set_paging_root(pid_t pid, size_t root);
632632
*/
633633

634634
/**
635-
* Invalidates the TLB for a given address on all CPUs.
635+
* Invalidates the TLB for a given address (belonging to the current process) on all CPUs.
636636
*
637637
* @param[in] address The address to invalidate
638638
*
639639
*/
640640
ptedit_fnc void ptedit_invalidate_tlb(void* address);
641641

642+
/**
643+
* Invalidates the TLB for a given address (belonging to the specified pid) on all CPUs.
644+
*
645+
* @param[in] address The address to invalidate
646+
*
647+
*/
648+
ptedit_fnc void ptedit_invalidate_tlb_pid(pid_t pid, void* address);
649+
642650
/**
643651
* Change the method used for flushing the TLB (either kernel or custom function)
644652
*

‎ptedit_header.h‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ typedef struct {
9292
size_t root;
9393
} ptedit_paging_t;
9494

95+
/**
96+
* Structure to hold the arguments for TLB invalidation
97+
*/
98+
typedef struct {
99+
pid_t pid;
100+
void* address;
101+
} ptedit_invalidate_tlb_args_t;
102+
95103
#define PTEDIT_VALID_MASK_PGD (1<<0)
96104
#define PTEDIT_VALID_MASK_P4D (1<<1)
97105
#define PTEDIT_VALID_MASK_PUD (1<<2)
@@ -142,6 +150,9 @@ typedef struct {
142150

143151
#define PTEDITOR_IOCTL_CMD_SWITCH_TLB_INVALIDATION \
144152
_IOR(PTEDITOR_IOCTL_MAGIC_NUMBER, 13, size_t)
153+
154+
#define PTEDITOR_IOCTL_CMD_INVALIDATE_TLB_PID \
155+
_IOR(PTEDITOR_IOCTL_MAGIC_NUMBER, 14, size_t)
145156
#else
146157
#define PTEDITOR_READ_PAGE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x801, METHOD_BUFFERED, FILE_ANY_ACCESS)
147158
#define PTEDITOR_WRITE_PAGE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_READ_DATA)
@@ -789,13 +800,21 @@ ptedit_fnc void ptedit_set_paging_root(pid_t pid, size_t root);
789800
*/
790801

791802
/**
792-
* Invalidates the TLB for a given address on all CPUs.
803+
* Invalidates the TLB for a given address (belonging to the current process) on all CPUs.
793804
*
794805
* @param[in] address The address to invalidate
795806
*
796807
*/
797808
ptedit_fnc void ptedit_invalidate_tlb(void* address);
798809

810+
/**
811+
* Invalidates the TLB for a given address (belonging to the specified pid) on all CPUs.
812+
*
813+
* @param[in] address The address to invalidate
814+
*
815+
*/
816+
ptedit_fnc void ptedit_invalidate_tlb_pid(pid_t pid, void* address);
817+
799818
/**
800819
* Change the method used for flushing the TLB (either kernel or custom function)
801820
*
@@ -1255,20 +1274,20 @@ ptedit_fnc void ptedit_update_user_ext(void* address, pid_t pid, ptedit_entry_t*
12551274
pset(root + pgdi * ptedit_entry_size, vm->pgd);
12561275
}
12571276

1258-
ptedit_invalidate_tlb(address);
1277+
ptedit_invalidate_tlb_pid(pid, address);
12591278
}
12601279

12611280
// ---------------------------------------------------------------------------
12621281
static void ptedit_update_user(void* address, pid_t pid, ptedit_entry_t* vm) {
12631282
ptedit_update_user_ext(address, pid, vm, ptedit_phys_write_pwrite);
1264-
ptedit_invalidate_tlb(address);
1283+
ptedit_invalidate_tlb_pid(pid, address);
12651284
}
12661285

12671286

12681287
// ---------------------------------------------------------------------------
12691288
static void ptedit_update_user_map(void* address, pid_t pid, ptedit_entry_t* vm) {
12701289
ptedit_update_user_ext(address, pid, vm, ptedit_phys_write_map);
1271-
ptedit_invalidate_tlb(address);
1290+
ptedit_invalidate_tlb_pid(pid, address);
12721291
}
12731292

12741293
// ---------------------------------------------------------------------------
@@ -1622,9 +1641,24 @@ void ptedit_set_paging_root(pid_t pid, size_t root) {
16221641
#endif
16231642
}
16241643

1644+
// ---------------------------------------------------------------------------
1645+
ptedit_fnc void ptedit_invalidate_tlb_pid(pid_t pid, void* address) {
1646+
#if defined(LINUX)
1647+
ptedit_invalidate_tlb_args_t args;
1648+
args.pid = pid;
1649+
args.address = address;
1650+
ioctl(ptedit_fd, PTEDITOR_IOCTL_CMD_INVALIDATE_TLB_PID, (size_t)&args, pid);
1651+
#else
1652+
size_t vaddr = (size_t)address;
1653+
DWORD returnLength;
1654+
DeviceIoControl(ptedit_fd, PTEDITOR_FLUSH_TLB, (LPVOID)&vaddr, sizeof(vaddr), (LPVOID)&vaddr, sizeof(vaddr), &returnLength, 0);
1655+
#endif
1656+
}
16251657

16261658
// ---------------------------------------------------------------------------
16271659
ptedit_fnc void ptedit_invalidate_tlb(void* address) {
1660+
// we do not directly call ptedit_invalidate_tlb_pid to ensure that the old
1661+
// API is still working (for backwards compatibility)
16281662
#if defined(LINUX)
16291663
ioctl(ptedit_fd, PTEDITOR_IOCTL_CMD_INVALIDATE_TLB, (size_t)address);
16301664
#else

0 commit comments

Comments
 (0)