Skip to content

Commit 7a6d819

Browse files
brandtbucherbrandtbucher
committed
Test the JIT stencils build process
1 parent be02e68 commit 7a6d819

11 files changed

+446
-7
lines changed

‎Lib/test/test_jit_stencils.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
import pathlib
3+
import shlex
4+
import sys
5+
import sysconfig
6+
import tempfile
7+
import test.support
8+
import unittest
9+
10+
import test.support.script_helper
11+
12+
13+
_CPYTHON = pathlib.Path(test.support.REPO_ROOT).resolve()
14+
_TOOLS_JIT = _CPYTHON / "Tools" / "jit"
15+
_TOOLS_JIT_TEST = _TOOLS_JIT / "test"
16+
_TOOLS_JIT_BUILD_PY = _TOOLS_JIT / "build.py"
17+
18+
@unittest.skipIf(test.support.Py_DEBUG, "XXX")
19+
@unittest.skipUnless(sys._jit.is_available(), "XXX")
20+
@unittest.skipIf(test.support.Py_GIL_DISABLED, "XXX")
21+
@unittest.skipUnless(sysconfig.is_python_build(), "XXX")
22+
class TestJITStencils(unittest.TestCase):
23+
24+
def test_jit_stencils(self):
25+
self.maxDiff = None
26+
found = False
27+
pyconfig_dir = pathlib.Path(sysconfig.get_config_h_filename()).parent
28+
with tempfile.TemporaryDirectory() as work:
29+
output_dir = pathlib.Path(work).resolve()
30+
for test_jit_stencils_h in sorted(_TOOLS_JIT_TEST.glob("test_jit_stencils-*.h")):
31+
target = test_jit_stencils_h.stem.removeprefix("test_jit_stencils-")
32+
jit_stencils_h = output_dir / f"jit_stencils-{target}.h"
33+
with self.subTest(target):
34+
# relative = jit_stencils_h.relative_to(_CPYTHON)
35+
result, args = test.support.script_helper.run_python_until_end(
36+
_TOOLS_JIT_BUILD_PY,
37+
"--input-file", _TOOLS_JIT_TEST / "test_executor_cases.c.h",
38+
"--output-dir", output_dir,
39+
"--pyconfig-dir", pyconfig_dir,
40+
target,
41+
__isolated=False
42+
)
43+
if result.rc:
44+
self.skipTest(shlex.join(map(str, args)))
45+
found = True
46+
expected = test_jit_stencils_h.read_text()
47+
actual = "".join(jit_stencils_h.read_text().splitlines(True)[3:])
48+
self.assertEqual(expected, actual)
49+
self.assertTrue(found, "No JIT stencil tests run!")

‎Tools/jit/_targets.py‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class _Target(typing.Generic[_S, _R]):
5151
verbose: bool = False
5252
cflags: str = ""
5353
known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
54+
input_file: pathlib.Path = PYTHON_EXECUTOR_CASES_C_H
5455
pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve()
5556

5657
def _get_nop(self) -> bytes:
@@ -68,7 +69,7 @@ def _compute_digest(self) -> str:
6869
hasher.update(self.debug.to_bytes())
6970
hasher.update(self.cflags.encode())
7071
# These dependencies are also reflected in _JITSources in regen.targets:
71-
hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
72+
hasher.update(self.input_file.read_bytes())
7273
hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes())
7374
for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)):
7475
for filename in filenames:
@@ -82,10 +83,16 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
8283
if output is not None:
8384
# Make sure that full paths don't leak out (for reproducibility):
8485
long, short = str(path), str(path.name)
85-
group.code.disassembly.extend(
86-
line.expandtabs().strip().replace(long, short)
87-
for line in output.splitlines()
88-
)
86+
lines = output.splitlines()
87+
started = False
88+
for line in lines:
89+
if not started:
90+
if "_JIT_ENTRY" not in line:
91+
continue
92+
started = True
93+
cleaned = line.replace(long, short).expandtabs().strip()
94+
if cleaned:
95+
group.code.disassembly.append(cleaned)
8996
args = [
9097
"--elf-output-style=JSON",
9198
"--expand-relocs",
@@ -181,10 +188,12 @@ async def _compile(
181188
return await self._parse(o)
182189

183190
async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
184-
generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text()
191+
generated_cases = self.input_file.read_text()
185192
cases_and_opnames = sorted(
186193
re.findall(
187-
r"\n {8}(case (\w+): \{\n.*?\n {8}\})", generated_cases, flags=re.DOTALL
194+
r"^ {8}(case (\w+): \{\n.*?\n {8}\})",
195+
generated_cases,
196+
flags=re.DOTALL | re.MULTILINE,
188197
)
189198
)
190199
tasks = []

‎Tools/jit/build.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222
parser.add_argument(
2323
"-f", "--force", action="store_true", help="force the entire JIT to be rebuilt"
2424
)
25+
parser.add_argument(
26+
"-i",
27+
"--input-file",
28+
help="where to find the generated executor cases",
29+
type=lambda p: pathlib.Path(p).resolve(),
30+
)
2531
parser.add_argument(
2632
"-o",
2733
"--output-dir",
@@ -48,6 +54,8 @@
4854
target.force = args.force
4955
target.verbose = args.verbose
5056
target.cflags = args.cflags
57+
if args.input_file is not None:
58+
target.input_file = args.input_file
5159
target.pyconfig_dir = args.pyconfig_dir
5260
target.build(
5361
comment=comment,

‎Tools/jit/test/test_executor_cases.c.h‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
case 0: {
2+
break;
3+
}
4+
5+
case 1: {
6+
if (CURRENT_OPARG()) {
7+
JUMP_TO_JUMP_TARGET();
8+
}
9+
break;
10+
}
11+
12+
case 2: {
13+
if (CURRENT_OPARG()) {
14+
JUMP_TO_ERROR();
15+
}
16+
break;
17+
}
18+
19+
case 3: {
20+
GOTO_TIER_ONE((void *)CURRENT_OPERAND0() + CURRENT_TARGET());
21+
break;
22+
}
23+
24+
case 4: {
25+
GOTO_TIER_TWO((void *)CURRENT_OPERAND1());
26+
break;
27+
}

‎Tools/jit/test/test_jit_stencils-aarch64-apple-darwin.h‎

Whitespace-only changes.

‎Tools/jit/test/test_jit_stencils-aarch64-pc-windows-msvc.h‎

Whitespace-only changes.

‎Tools/jit/test/test_jit_stencils-aarch64-unknown-linux-gnu.h‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
void
2+
emit_shim(
3+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
4+
const _PyUOpInstruction *instruction, jit_state *state)
5+
{
6+
// 0000000000000000 <_JIT_ENTRY>:
7+
// 0: 6db63bef stp d15, d14, [sp, #-0xa0]!
8+
// 4: a90857f6 stp x22, x21, [sp, #0x80]
9+
// 8: aa0103f5 mov x21, x1
10+
// c: aa0203f6 mov x22, x2
11+
// 10: a9094ff4 stp x20, x19, [sp, #0x90]
12+
// 14: aa0003f4 mov x20, x0
13+
// 18: 6d0133ed stp d13, d12, [sp, #0x10]
14+
// 1c: 6d022beb stp d11, d10, [sp, #0x20]
15+
// 20: 6d0323e9 stp d9, d8, [sp, #0x30]
16+
// 24: a9047bfd stp x29, x30, [sp, #0x40]
17+
// 28: 910103fd add x29, sp, #0x40
18+
// 2c: a9056ffc stp x28, x27, [sp, #0x50]
19+
// 30: a90667fa stp x26, x25, [sp, #0x60]
20+
// 34: a9075ff8 stp x24, x23, [sp, #0x70]
21+
// 38: 9400000c bl 0x68 <_JIT_ENTRY+0x68>
22+
// 3c: a9494ff4 ldp x20, x19, [sp, #0x90]
23+
// 40: a94857f6 ldp x22, x21, [sp, #0x80]
24+
// 44: a9475ff8 ldp x24, x23, [sp, #0x70]
25+
// 48: a94667fa ldp x26, x25, [sp, #0x60]
26+
// 4c: a9456ffc ldp x28, x27, [sp, #0x50]
27+
// 50: a9447bfd ldp x29, x30, [sp, #0x40]
28+
// 54: 6d4323e9 ldp d9, d8, [sp, #0x30]
29+
// 58: 6d422beb ldp d11, d10, [sp, #0x20]
30+
// 5c: 6d4133ed ldp d13, d12, [sp, #0x10]
31+
// 60: 6cca3bef ldp d15, d14, [sp], #0xa0
32+
// 64: d65f03c0 ret
33+
const unsigned char code_body[104] = {
34+
0xef, 0x3b, 0xb6, 0x6d, 0xf6, 0x57, 0x08, 0xa9,
35+
0xf5, 0x03, 0x01, 0xaa, 0xf6, 0x03, 0x02, 0xaa,
36+
0xf4, 0x4f, 0x09, 0xa9, 0xf4, 0x03, 0x00, 0xaa,
37+
0xed, 0x33, 0x01, 0x6d, 0xeb, 0x2b, 0x02, 0x6d,
38+
0xe9, 0x23, 0x03, 0x6d, 0xfd, 0x7b, 0x04, 0xa9,
39+
0xfd, 0x03, 0x01, 0x91, 0xfc, 0x6f, 0x05, 0xa9,
40+
0xfa, 0x67, 0x06, 0xa9, 0xf8, 0x5f, 0x07, 0xa9,
41+
0x0c, 0x00, 0x00, 0x94, 0xf4, 0x4f, 0x49, 0xa9,
42+
0xf6, 0x57, 0x48, 0xa9, 0xf8, 0x5f, 0x47, 0xa9,
43+
0xfa, 0x67, 0x46, 0xa9, 0xfc, 0x6f, 0x45, 0xa9,
44+
0xfd, 0x7b, 0x44, 0xa9, 0xe9, 0x23, 0x43, 0x6d,
45+
0xeb, 0x2b, 0x42, 0x6d, 0xed, 0x33, 0x41, 0x6d,
46+
0xef, 0x3b, 0xca, 0x6c, 0xc0, 0x03, 0x5f, 0xd6,
47+
};
48+
memcpy(code, code_body, sizeof(code_body));
49+
}
50+
51+
void
52+
emit_0(
53+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
54+
const _PyUOpInstruction *instruction, jit_state *state)
55+
{
56+
}
57+
58+
void
59+
emit_1(
60+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
61+
const _PyUOpInstruction *instruction, jit_state *state)
62+
{
63+
// 0000000000000000 <_JIT_ENTRY>:
64+
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
65+
// 0000000000000000: R_AARCH64_ADR_GOT_PAGE _JIT_OPARG
66+
// 4: f9400108 ldr x8, [x8]
67+
// 0000000000000004: R_AARCH64_LD64_GOT_LO12_NC _JIT_OPARG
68+
// 8: 72003d1f tst w8, #0xffff
69+
// c: 54000040 b.eq 0x14 <_JIT_ENTRY+0x14>
70+
// 10: 14000000 b 0x10 <_JIT_ENTRY+0x10>
71+
// 0000000000000010: R_AARCH64_JUMP26 _JIT_JUMP_TARGET
72+
const unsigned char code_body[20] = {
73+
0x08, 0x00, 0x00, 0x90, 0x08, 0x01, 0x40, 0xf9,
74+
0x1f, 0x3d, 0x00, 0x72, 0x40, 0x00, 0x00, 0x54,
75+
0x00, 0x00, 0x00, 0x14,
76+
};
77+
// 0: OPARG
78+
patch_64(data + 0x0, instruction->oparg);
79+
memcpy(code, code_body, sizeof(code_body));
80+
patch_aarch64_33rx(code + 0x0, (uintptr_t)data);
81+
patch_aarch64_26r(code + 0x10, state->instruction_starts[instruction->jump_target]);
82+
}
83+
84+
void
85+
emit_2(
86+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
87+
const _PyUOpInstruction *instruction, jit_state *state)
88+
{
89+
// 0000000000000000 <_JIT_ENTRY>:
90+
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
91+
// 0000000000000000: R_AARCH64_ADR_GOT_PAGE _JIT_OPARG
92+
// 4: f9400108 ldr x8, [x8]
93+
// 0000000000000004: R_AARCH64_LD64_GOT_LO12_NC _JIT_OPARG
94+
// 8: 72003d1f tst w8, #0xffff
95+
// c: 54000040 b.eq 0x14 <_JIT_ENTRY+0x14>
96+
// 10: 14000000 b 0x10 <_JIT_ENTRY+0x10>
97+
// 0000000000000010: R_AARCH64_JUMP26 _JIT_ERROR_TARGET
98+
const unsigned char code_body[20] = {
99+
0x08, 0x00, 0x00, 0x90, 0x08, 0x01, 0x40, 0xf9,
100+
0x1f, 0x3d, 0x00, 0x72, 0x40, 0x00, 0x00, 0x54,
101+
0x00, 0x00, 0x00, 0x14,
102+
};
103+
// 0: OPARG
104+
patch_64(data + 0x0, instruction->oparg);
105+
memcpy(code, code_body, sizeof(code_body));
106+
patch_aarch64_33rx(code + 0x0, (uintptr_t)data);
107+
patch_aarch64_26r(code + 0x10, state->instruction_starts[instruction->error_target]);
108+
}
109+
110+
void
111+
emit_3(
112+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
113+
const _PyUOpInstruction *instruction, jit_state *state)
114+
{
115+
// 0000000000000000 <_JIT_ENTRY>:
116+
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
117+
// 0000000000000000: R_AARCH64_ADR_GOT_PAGE _JIT_TARGET
118+
// 4: 90000009 adrp x9, 0x0 <_JIT_ENTRY>
119+
// 0000000000000004: R_AARCH64_ADR_GOT_PAGE _JIT_OPERAND0
120+
// 8: f9400108 ldr x8, [x8]
121+
// 0000000000000008: R_AARCH64_LD64_GOT_LO12_NC _JIT_TARGET
122+
// c: f9400129 ldr x9, [x9]
123+
// 000000000000000c: R_AARCH64_LD64_GOT_LO12_NC _JIT_OPERAND0
124+
// 10: f9008adf str xzr, [x22, #0x110]
125+
// 14: f9002295 str x21, [x20, #0x40]
126+
// 18: 8b284120 add x0, x9, w8, uxtw
127+
// 1c: d65f03c0 ret
128+
const unsigned char code_body[32] = {
129+
0x08, 0x00, 0x00, 0x90, 0x09, 0x00, 0x00, 0x90,
130+
0x08, 0x01, 0x40, 0xf9, 0x29, 0x01, 0x40, 0xf9,
131+
0xdf, 0x8a, 0x00, 0xf9, 0x95, 0x22, 0x00, 0xf9,
132+
0x20, 0x41, 0x28, 0x8b, 0xc0, 0x03, 0x5f, 0xd6,
133+
};
134+
// 0: TARGET
135+
// 8: OPERAND0
136+
patch_64(data + 0x0, instruction->target);
137+
patch_64(data + 0x8, instruction->operand0);
138+
memcpy(code, code_body, sizeof(code_body));
139+
patch_aarch64_21rx(code + 0x0, (uintptr_t)data);
140+
patch_aarch64_21rx(code + 0x4, (uintptr_t)data + 0x8);
141+
patch_aarch64_12x(code + 0x8, (uintptr_t)data);
142+
patch_aarch64_12x(code + 0xc, (uintptr_t)data + 0x8);
143+
}
144+
145+
void
146+
emit_4(
147+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
148+
const _PyUOpInstruction *instruction, jit_state *state)
149+
{
150+
// 0000000000000000 <_JIT_ENTRY>:
151+
// 0: 90000008 adrp x8, 0x0 <_JIT_ENTRY>
152+
// 0000000000000000: R_AARCH64_ADR_GOT_PAGE _JIT_OPERAND1
153+
// 4: f9400108 ldr x8, [x8]
154+
// 0000000000000004: R_AARCH64_LD64_GOT_LO12_NC _JIT_OPERAND1
155+
// 8: f9403d00 ldr x0, [x8, #0x78]
156+
// c: f9008ac8 str x8, [x22, #0x110]
157+
// 10: d61f0000 br x0
158+
const unsigned char code_body[20] = {
159+
0x08, 0x00, 0x00, 0x90, 0x08, 0x01, 0x40, 0xf9,
160+
0x00, 0x3d, 0x40, 0xf9, 0xc8, 0x8a, 0x00, 0xf9,
161+
0x00, 0x00, 0x1f, 0xd6,
162+
};
163+
// 0: OPERAND1
164+
patch_64(data + 0x0, instruction->operand1);
165+
memcpy(code, code_body, sizeof(code_body));
166+
patch_aarch64_33rx(code + 0x0, (uintptr_t)data);
167+
}
168+
169+
static_assert(SYMBOL_MASK_WORDS >= 1, "SYMBOL_MASK_WORDS too small");
170+
171+
typedef struct {
172+
void (*emit)(
173+
unsigned char *code, unsigned char *data, _PyExecutorObject *executor,
174+
const _PyUOpInstruction *instruction, jit_state *state);
175+
size_t code_size;
176+
size_t data_size;
177+
symbol_mask trampoline_mask;
178+
} StencilGroup;
179+
180+
static const StencilGroup shim = {emit_shim, 104, 0, {0}};
181+
182+
static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {
183+
[0] = {emit_0, 0, 0, {0}},
184+
[1] = {emit_1, 20, 8, {0}},
185+
[2] = {emit_2, 20, 8, {0}},
186+
[3] = {emit_3, 32, 16, {0}},
187+
[4] = {emit_4, 20, 8, {0}},
188+
};
189+
190+
static const void * const symbols_map[1] = {
191+
0
192+
};

‎Tools/jit/test/test_jit_stencils-i686-pc-windows-msvc.h‎

Whitespace-only changes.

‎Tools/jit/test/test_jit_stencils-x86_64-apple-darwin.h‎

Whitespace-only changes.

‎Tools/jit/test/test_jit_stencils-x86_64-pc-windows-msvc.h‎

Whitespace-only changes.

0 commit comments

Comments
 (0)