Skip to content

Commit 9c13626

Browse files
BazookaMusicBazookaMusic
committed
remove guardrails sanitizer for now
1 parent 34da804 commit 9c13626

4 files changed

Lines changed: 113 additions & 84 deletions

File tree

‎javascript/ql/lib/ext/openai.model.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,3 @@ extensions:
2121
- ["@openai/agents", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"]
2222
- ["@openai/guardrails", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"]
2323
- ["@openai/guardrails", "Member[GuardrailAgent].Member[create].Argument[2]", "system-prompt-injection"]
24-
- ["openai.Client", "Member[responses].Member[create].Argument[0].Member[input]", "user-prompt-injection"]
25-
- ["openai.Client", "Member[completions].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"]
26-
- ["openai.Client", "Member[images].Member[generate,edit].Argument[0].Member[prompt]", "user-prompt-injection"]
27-
- ["openai.Client", "Member[embeddings].Member[create].Argument[0].Member[input]", "user-prompt-injection"]
28-
- ["openai.Client", "Member[audio].Member[transcriptions,translations].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"]

‎javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll‎

Lines changed: 71 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,6 @@ private predicate isSystemOrDevMessage(API::Node msg) {
1616
msg.getMember("role").asSink().mayHaveStringValue(["system", "developer", "assistant"])
1717
}
1818

19-
module OpenAIGuardrails {
20-
/** Gets a reference to the `GuardrailsOpenAI` class. */
21-
API::Node classRef() {
22-
result = API::moduleImport("@openai/guardrails")
23-
}
24-
25-
API::Node getSanitizerNode() {
26-
// checkPlainText(userInput, bundle) or runGuardrails(userInput, bundle)
27-
result = classRef().getMember(["checkPlainText", "runGuardrails"])
28-
}
29-
}
30-
3119
module OpenAI {
3220
/** Gets a reference to all OpenAI client instances. */
3321
private API::Node allClients() {
@@ -43,6 +31,33 @@ module OpenAI {
4331
.getPromised()
4432
}
4533

34+
/** Gets a guarded client that is clearly configured without input guardrails. */
35+
private API::Node unprotectedGuardedClient() {
36+
exists(API::Node createCall |
37+
createCall =
38+
API::moduleImport("@openai/guardrails")
39+
.getMember(["GuardrailsOpenAI", "GuardrailsAzureOpenAI"])
40+
.getMember("create") and
41+
result = createCall.getReturn().getPromised() and
42+
exists(createCall.getParameter(0).getMember("version")) and
43+
not exists(
44+
createCall.getParameter(0).getMember("input").getMember("guardrails").getArrayElement()
45+
) and
46+
not exists(
47+
createCall.getParameter(0).getMember("pre_flight").getMember("guardrails").getArrayElement()
48+
)
49+
)
50+
}
51+
52+
/** Gets a reference to all clients without input guardrails. */
53+
private API::Node clientsNoGuardrails() {
54+
result = API::moduleImport("openai").getInstance()
55+
or
56+
result = API::moduleImport("openai").getMember(["OpenAI", "AzureOpenAI"]).getInstance()
57+
or
58+
result = unprotectedGuardedClient()
59+
}
60+
4661
/**
4762
* Gets role-filtered system/developer/assistant message sinks.
4863
* These require checking a sibling `role` property and cannot be expressed in MaD.
@@ -100,10 +115,18 @@ module OpenAI {
100115
* These require checking a sibling `role` property and cannot be expressed in MaD.
101116
*/
102117
API::Node getUserPromptNode() {
118+
// responses.create({ input: "string" })
119+
result =
120+
clientsNoGuardrails()
121+
.getMember("responses")
122+
.getMember("create")
123+
.getParameter(0)
124+
.getMember("input")
125+
or
103126
// responses.create({ input: [{ role: "user", content: ... }] })
104127
exists(API::Node msg |
105128
msg =
106-
allClients()
129+
clientsNoGuardrails()
107130
.getMember("responses")
108131
.getMember("create")
109132
.getParameter(0)
@@ -117,7 +140,7 @@ module OpenAI {
117140
// chat.completions.create({ messages: [{ role: "user", content: ... }] })
118141
exists(API::Node msg, API::Node content |
119142
msg =
120-
allClients()
143+
clientsNoGuardrails()
121144
.getMember("chat")
122145
.getMember("completions")
123146
.getMember("create")
@@ -132,10 +155,34 @@ module OpenAI {
132155
result = content.getArrayElement().getMember("text")
133156
)
134157
or
158+
// Legacy completions API: completions.create({ prompt: ... })
159+
result =
160+
clientsNoGuardrails()
161+
.getMember("completions")
162+
.getMember("create")
163+
.getParameter(0)
164+
.getMember("prompt")
165+
or
166+
// images.generate({ prompt: ... }) and images.edit({ prompt: ... })
167+
result =
168+
clientsNoGuardrails()
169+
.getMember("images")
170+
.getMember(["generate", "edit"])
171+
.getParameter(0)
172+
.getMember("prompt")
173+
or
174+
// embeddings.create({ input: ... })
175+
result =
176+
clientsNoGuardrails()
177+
.getMember("embeddings")
178+
.getMember("create")
179+
.getParameter(0)
180+
.getMember("input")
181+
or
135182
// beta.threads.messages.create(threadId, { role: "user", content: ... })
136183
exists(API::Node msg |
137184
msg =
138-
allClients()
185+
clientsNoGuardrails()
139186
.getMember("beta")
140187
.getMember("threads")
141188
.getMember("messages")
@@ -145,6 +192,15 @@ module OpenAI {
145192
|
146193
result = msg.getMember("content")
147194
)
195+
or
196+
// audio.transcriptions/translations.create({ prompt: ... })
197+
result =
198+
clientsNoGuardrails()
199+
.getMember("audio")
200+
.getMember(["transcriptions", "translations"])
201+
.getMember("create")
202+
.getParameter(0)
203+
.getMember("prompt")
148204
}
149205
}
150206

‎javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected‎

Lines changed: 42 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,20 @@ edges
99
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:51:13:51:21 | userInput | provenance | |
1010
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:58:13:58:21 | userInput | provenance | |
1111
| gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:8:9:8:17 | userInput | provenance | |
12-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:24:12:24:20 | userInput | provenance | |
13-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:33:18:33:26 | userInput | provenance | |
14-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:44:18:44:26 | userInput | provenance | |
15-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:58:19:58:27 | userInput | provenance | |
16-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:68:13:68:21 | userInput | provenance | |
17-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:73:13:73:21 | userInput | provenance | |
18-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:77:13:77:21 | userInput | provenance | |
19-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:83:12:83:20 | userInput | provenance | |
20-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:90:13:90:21 | userInput | provenance | |
21-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:96:13:96:21 | userInput | provenance | |
22-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:102:14:102:22 | userInput | provenance | |
23-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:108:12:108:20 | userInput | provenance | |
24-
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:155:12:155:20 | userInput | provenance | |
25-
| openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:16:9:16:17 | userInput | provenance | |
12+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:22:12:22:20 | userInput | provenance | |
13+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:31:18:31:26 | userInput | provenance | |
14+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:42:18:42:26 | userInput | provenance | |
15+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:56:19:56:27 | userInput | provenance | |
16+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:66:13:66:21 | userInput | provenance | |
17+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:71:13:71:21 | userInput | provenance | |
18+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:75:13:75:21 | userInput | provenance | |
19+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:81:12:81:20 | userInput | provenance | |
20+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:88:13:88:21 | userInput | provenance | |
21+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:13:94:21 | userInput | provenance | |
22+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:14:100:22 | userInput | provenance | |
23+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:106:12:106:20 | userInput | provenance | |
24+
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:153:12:153:20 | userInput | provenance | |
25+
| openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:14:9:14:17 | userInput | provenance | |
2626
nodes
2727
| anthropic_user_test.js:8:9:8:17 | userInput | semmle.label | userInput |
2828
| anthropic_user_test.js:8:21:8:39 | req.query.userInput | semmle.label | req.query.userInput |
@@ -36,21 +36,21 @@ nodes
3636
| gemini_user_test.js:44:13:44:21 | userInput | semmle.label | userInput |
3737
| gemini_user_test.js:51:13:51:21 | userInput | semmle.label | userInput |
3838
| gemini_user_test.js:58:13:58:21 | userInput | semmle.label | userInput |
39-
| openai_user_test.js:16:9:16:17 | userInput | semmle.label | userInput |
40-
| openai_user_test.js:16:21:16:39 | req.query.userInput | semmle.label | req.query.userInput |
41-
| openai_user_test.js:24:12:24:20 | userInput | semmle.label | userInput |
42-
| openai_user_test.js:33:18:33:26 | userInput | semmle.label | userInput |
43-
| openai_user_test.js:44:18:44:26 | userInput | semmle.label | userInput |
44-
| openai_user_test.js:58:19:58:27 | userInput | semmle.label | userInput |
45-
| openai_user_test.js:68:13:68:21 | userInput | semmle.label | userInput |
46-
| openai_user_test.js:73:13:73:21 | userInput | semmle.label | userInput |
47-
| openai_user_test.js:77:13:77:21 | userInput | semmle.label | userInput |
48-
| openai_user_test.js:83:12:83:20 | userInput | semmle.label | userInput |
49-
| openai_user_test.js:90:13:90:21 | userInput | semmle.label | userInput |
50-
| openai_user_test.js:96:13:96:21 | userInput | semmle.label | userInput |
51-
| openai_user_test.js:102:14:102:22 | userInput | semmle.label | userInput |
52-
| openai_user_test.js:108:12:108:20 | userInput | semmle.label | userInput |
53-
| openai_user_test.js:155:12:155:20 | userInput | semmle.label | userInput |
39+
| openai_user_test.js:14:9:14:17 | userInput | semmle.label | userInput |
40+
| openai_user_test.js:14:21:14:39 | req.query.userInput | semmle.label | req.query.userInput |
41+
| openai_user_test.js:22:12:22:20 | userInput | semmle.label | userInput |
42+
| openai_user_test.js:31:18:31:26 | userInput | semmle.label | userInput |
43+
| openai_user_test.js:42:18:42:26 | userInput | semmle.label | userInput |
44+
| openai_user_test.js:56:19:56:27 | userInput | semmle.label | userInput |
45+
| openai_user_test.js:66:13:66:21 | userInput | semmle.label | userInput |
46+
| openai_user_test.js:71:13:71:21 | userInput | semmle.label | userInput |
47+
| openai_user_test.js:75:13:75:21 | userInput | semmle.label | userInput |
48+
| openai_user_test.js:81:12:81:20 | userInput | semmle.label | userInput |
49+
| openai_user_test.js:88:13:88:21 | userInput | semmle.label | userInput |
50+
| openai_user_test.js:94:13:94:21 | userInput | semmle.label | userInput |
51+
| openai_user_test.js:100:14:100:22 | userInput | semmle.label | userInput |
52+
| openai_user_test.js:106:12:106:20 | userInput | semmle.label | userInput |
53+
| openai_user_test.js:153:12:153:20 | userInput | semmle.label | userInput |
5454
subpaths
5555
#select
5656
| anthropic_user_test.js:18:18:18:26 | userInput | anthropic_user_test.js:8:21:8:39 | req.query.userInput | anthropic_user_test.js:18:18:18:26 | userInput | This prompt construction depends on a $@. | anthropic_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
@@ -61,16 +61,16 @@ subpaths
6161
| gemini_user_test.js:44:13:44:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:44:13:44:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
6262
| gemini_user_test.js:51:13:51:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:51:13:51:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
6363
| gemini_user_test.js:58:13:58:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:58:13:58:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
64-
| openai_user_test.js:24:12:24:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:24:12:24:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
65-
| openai_user_test.js:33:18:33:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:33:18:33:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
66-
| openai_user_test.js:44:18:44:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:44:18:44:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
67-
| openai_user_test.js:58:19:58:27 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:58:19:58:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
68-
| openai_user_test.js:68:13:68:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:68:13:68:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
69-
| openai_user_test.js:73:13:73:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:73:13:73:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
70-
| openai_user_test.js:77:13:77:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:77:13:77:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
71-
| openai_user_test.js:83:12:83:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:83:12:83:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
72-
| openai_user_test.js:90:13:90:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:90:13:90:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
73-
| openai_user_test.js:96:13:96:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:96:13:96:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
74-
| openai_user_test.js:102:14:102:22 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:102:14:102:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
75-
| openai_user_test.js:108:12:108:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:108:12:108:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
76-
| openai_user_test.js:155:12:155:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:155:12:155:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
64+
| openai_user_test.js:22:12:22:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:22:12:22:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
65+
| openai_user_test.js:31:18:31:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:31:18:31:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
66+
| openai_user_test.js:42:18:42:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:42:18:42:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
67+
| openai_user_test.js:56:19:56:27 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:56:19:56:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
68+
| openai_user_test.js:66:13:66:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:66:13:66:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
69+
| openai_user_test.js:71:13:71:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:71:13:71:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
70+
| openai_user_test.js:75:13:75:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:75:13:75:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
71+
| openai_user_test.js:81:12:81:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:81:12:81:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
72+
| openai_user_test.js:88:13:88:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:88:13:88:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
73+
| openai_user_test.js:94:13:94:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:13:94:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
74+
| openai_user_test.js:100:14:100:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:14:100:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
75+
| openai_user_test.js:106:12:106:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:106:12:106:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
76+
| openai_user_test.js:153:12:153:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:153:12:153:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |

0 commit comments

Comments
 (0)