Skip to content

Commit 385e14c

Browse files
hvitvedhvitved
committed
Shared: Post-processing query for inline test expectations
1 parent d56f0c5 commit 385e14c

File tree

1 file changed

+264
-9
lines changed

1 file changed

+264
-9
lines changed

‎shared/util/codeql/util/test/InlineExpectationsTest.qll‎

Lines changed: 264 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,30 @@ module Make<InlineExpectationsTestSig Impl> {
134134
* predicate for an active test will be ignored. This makes it possible to write multiple tests in
135135
* different `.ql` files that all query the same source code.
136136
*/
137+
bindingset[result]
137138
string getARelevantTag();
138139

140+
/**
141+
* Holds if expected tag `expectedTag` matches actual tag `actualTag`.
142+
*
143+
* This is normally defined as `expectedTag = actualTag`.
144+
*/
145+
bindingset[expectedTag, actualTag]
146+
default predicate tagMatches(string expectedTag, string actualTag) { expectedTag = actualTag }
147+
148+
bindingset[expectedTag]
149+
default predicate tagIsOptional(string expectedTag) { none() }
150+
151+
/**
152+
* Holds if expected tag `expectedTag` matches actual tag `actualTag`.
153+
*
154+
* This is normally defined as `expectedTag = actualTag`.
155+
*/
156+
bindingset[expectedValue, actualValue]
157+
default predicate valueMatches(string expectedValue, string actualValue) {
158+
expectedValue = actualValue
159+
}
160+
139161
/**
140162
* Returns the actual results of the query that is being tested. Each result consist of the
141163
* following values:
@@ -200,13 +222,13 @@ module Make<InlineExpectationsTestSig Impl> {
200222
not exists(ActualTestResult actualResult | expectation.matchesActualResult(actualResult)) and
201223
expectation.getTag() = TestImpl::getARelevantTag() and
202224
element = expectation and
203-
(
204-
expectation instanceof GoodTestExpectation and
205-
message = "Missing result: " + expectation.getExpectationText()
206-
or
207-
expectation instanceof FalsePositiveTestExpectation and
208-
message = "Fixed spurious result: " + expectation.getExpectationText()
209-
)
225+
not expectation.isOptional()
226+
|
227+
expectation instanceof GoodTestExpectation and
228+
message = "Missing result: " + expectation.getExpectationText()
229+
or
230+
expectation instanceof FalsePositiveTestExpectation and
231+
message = "Fixed spurious result: " + expectation.getExpectationText()
210232
)
211233
or
212234
exists(InvalidTestExpectation expectation |
@@ -311,9 +333,11 @@ module Make<InlineExpectationsTestSig Impl> {
311333

312334
predicate matchesActualResult(ActualTestResult actualResult) {
313335
onSameLine(pragma[only_bind_into](this), actualResult) and
314-
this.getTag() = actualResult.getTag() and
315-
this.getValue() = actualResult.getValue()
336+
TestImpl::tagMatches(this.getTag(), actualResult.getTag()) and
337+
TestImpl::valueMatches(this.getValue(), actualResult.getValue())
316338
}
339+
340+
predicate isOptional() { TestImpl::tagIsOptional(tag) }
317341
}
318342

319343
// Note: These next three classes correspond to all the possible values of type `TColumn`.
@@ -337,6 +361,15 @@ module Make<InlineExpectationsTestSig Impl> {
337361
string getExpectation() { result = expectation }
338362
}
339363

364+
ValidTestExpectation getAMatchingExpectation(
365+
Impl::Location location, string element, string tag, string val, boolean optional
366+
) {
367+
exists(ActualTestResult actualResult |
368+
result.matchesActualResult(actualResult) and
369+
actualResult = TActualResult(location, element, tag, val, optional)
370+
)
371+
}
372+
340373
query predicate testFailures(FailureLocatable element, string message) {
341374
hasFailureMessage(element, message)
342375
}
@@ -385,6 +418,7 @@ module Make<InlineExpectationsTestSig Impl> {
385418
* ```
386419
*/
387420
module MergeTests<TestSig TestImpl1, TestSig TestImpl2> implements TestSig {
421+
bindingset[result]
388422
string getARelevantTag() {
389423
result = TestImpl1::getARelevantTag() or result = TestImpl2::getARelevantTag()
390424
}
@@ -408,6 +442,7 @@ module Make<InlineExpectationsTestSig Impl> {
408442
module MergeTests3<TestSig TestImpl1, TestSig TestImpl2, TestSig TestImpl3> implements TestSig {
409443
private module M = MergeTests<MergeTests<TestImpl1, TestImpl2>, TestImpl3>;
410444

445+
bindingset[result]
411446
string getARelevantTag() { result = M::getARelevantTag() }
412447

413448
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -427,6 +462,7 @@ module Make<InlineExpectationsTestSig Impl> {
427462
{
428463
private module M = MergeTests<MergeTests3<TestImpl1, TestImpl2, TestImpl3>, TestImpl4>;
429464

465+
bindingset[result]
430466
string getARelevantTag() { result = M::getARelevantTag() }
431467

432468
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -448,6 +484,7 @@ module Make<InlineExpectationsTestSig Impl> {
448484
private module M =
449485
MergeTests<MergeTests4<TestImpl1, TestImpl2, TestImpl3, TestImpl4>, TestImpl5>;
450486

487+
bindingset[result]
451488
string getARelevantTag() { result = M::getARelevantTag() }
452489

453490
predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -590,3 +627,221 @@ private string expectationPattern() {
590627
result = tags + "(?:=" + value + ")?"
591628
)
592629
}
630+
631+
/**
632+
* Provides logic for creating a `@kind test-postprocess` query that checks
633+
* inline test expectations using `$ Alert` markers.
634+
*/
635+
module TestPostProcessing {
636+
external predicate queryResults(string relation, int row, int column, string data);
637+
638+
external predicate queryRelations(string relation);
639+
640+
external predicate queryMetadata(string key, string value);
641+
642+
private string getQueryId() { queryMetadata("id", result) }
643+
644+
private string getQueryKind() { queryMetadata("kind", result) }
645+
646+
signature module InputSig<InlineExpectationsTestSig Input> {
647+
string getRelativeUrl(Input::Location location);
648+
}
649+
650+
module Make<InlineExpectationsTestSig Input, InputSig<Input> Input2> {
651+
private import InlineExpectationsTest as InlineExpectationsTest
652+
private import InlineExpectationsTest::Make<Input>
653+
654+
/**
655+
* Gets the tag to be used for the path-problem source at result row `row`.
656+
*
657+
* This is either `Source` or `Alert`, depending on whether the location
658+
* of the source matches the location of the alert.
659+
*/
660+
private string getSourceTag(int row) {
661+
getQueryKind() = "path-problem" and
662+
exists(string loc | queryResults("#select", row, 2, loc) |
663+
if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Source"
664+
)
665+
}
666+
667+
/**
668+
* Gets the tag to be used for the path-problem sink at result row `row`.
669+
*
670+
* This is either `Sink` or `Alert`, depending on whether the location
671+
* of the sink matches the location of the alert.
672+
*/
673+
private string getSinkTag(int row) {
674+
getQueryKind() = "path-problem" and
675+
exists(string loc | queryResults("#select", row, 4, loc) |
676+
if queryResults("#select", row, 0, loc) then result = "Alert" else result = "Sink"
677+
)
678+
}
679+
680+
/**
681+
* A configuration for matching `// $ Source=foo` comments against actual
682+
* path-problem sources.
683+
*/
684+
private module PathProblemSourceTestInput implements TestSig {
685+
string getARelevantTag() { result = getSourceTag(_) }
686+
687+
bindingset[expectedValue, actualValue]
688+
predicate valueMatches(string expectedValue, string actualValue) {
689+
exists(expectedValue) and
690+
actualValue = ""
691+
}
692+
693+
additional predicate hasPathProblemSource(
694+
int row, Input::Location location, string element, string tag, string value
695+
) {
696+
getQueryKind() = "path-problem" and
697+
exists(string loc |
698+
queryResults("#select", row, 2, loc) and
699+
queryResults("#select", row, 3, element) and
700+
tag = getSourceTag(row) and
701+
value = "" and
702+
Input2::getRelativeUrl(location) = loc
703+
)
704+
}
705+
706+
predicate hasActualResult(Input::Location location, string element, string tag, string value) {
707+
hasPathProblemSource(_, location, element, tag, value)
708+
}
709+
}
710+
711+
private module PathProblemSourceTest = MakeTest<PathProblemSourceTestInput>;
712+
713+
private module TestInput implements TestSig {
714+
bindingset[result]
715+
string getARelevantTag() { any() }
716+
717+
private string getTagRegex() {
718+
exists(string sourceSinkTags |
719+
getQueryKind() = "problem" and
720+
sourceSinkTags = ""
721+
or
722+
sourceSinkTags = "|" + getSourceTag(_) + "|" + getSinkTag(_)
723+
|
724+
result = "(Alert" + sourceSinkTags + ")(\\[(.*)\\])?"
725+
)
726+
}
727+
728+
bindingset[expectedTag, actualTag]
729+
predicate tagMatches(string expectedTag, string actualTag) {
730+
actualTag = expectedTag.regexpCapture(getTagRegex(), 1) and
731+
(
732+
getQueryId() = expectedTag.regexpCapture(getTagRegex(), 3)
733+
or
734+
not exists(expectedTag.regexpCapture(getTagRegex(), 3))
735+
)
736+
}
737+
738+
bindingset[expectedTag]
739+
predicate tagIsOptional(string expectedTag) {
740+
not expectedTag.regexpMatch(getTagRegex())
741+
or
742+
exists(string queryId |
743+
queryId = expectedTag.regexpCapture(getTagRegex(), 3) and
744+
queryId != getQueryId()
745+
)
746+
}
747+
748+
bindingset[expectedValue, actualValue]
749+
predicate valueMatches(string expectedValue, string actualValue) {
750+
expectedValue = actualValue
751+
or
752+
actualValue = ""
753+
}
754+
755+
private predicate hasPathProblemSource = PathProblemSourceTestInput::hasPathProblemSource/5;
756+
757+
/**
758+
* Gets the expected sink value for result row `row`. This value must
759+
* match the value at the corresponding path-problem source.
760+
*/
761+
private string getSinkValue(int row) {
762+
exists(Input::Location location, string element, string tag, string val |
763+
hasPathProblemSource(row, location, element, tag, val) and
764+
result =
765+
PathProblemSourceTest::getAMatchingExpectation(location, element, tag, val, false)
766+
.getValue()
767+
)
768+
}
769+
770+
private predicate hasPathProblemSink(
771+
int row, Input::Location location, string element, string tag, string value
772+
) {
773+
getQueryKind() = "path-problem" and
774+
exists(string loc |
775+
queryResults("#select", row, 4, loc) and
776+
queryResults("#select", row, 5, element) and
777+
tag = getSinkTag(row) and
778+
Input2::getRelativeUrl(location) = loc
779+
|
780+
not exists(getSinkValue(row)) and value = ""
781+
or
782+
value = getSinkValue(row)
783+
)
784+
}
785+
786+
private predicate hasAlert(Input::Location location, string element, string tag, string value) {
787+
getQueryKind() = ["problem", "path-problem"] and
788+
exists(int row, string loc |
789+
queryResults("#select", row, 0, loc) and
790+
queryResults("#select", row, 2, element) and
791+
tag = "Alert" and
792+
value = "" and
793+
Input2::getRelativeUrl(location) = loc and
794+
not hasPathProblemSource(row, location, _, _, _) and
795+
not hasPathProblemSink(row, location, _, _, _)
796+
)
797+
}
798+
799+
predicate hasActualResult(Input::Location location, string element, string tag, string value) {
800+
hasAlert(location, element, tag, value)
801+
or
802+
hasPathProblemSource(_, location, element, tag, value)
803+
or
804+
hasPathProblemSink(_, location, element, tag, value)
805+
}
806+
}
807+
808+
private module Test = MakeTest<TestInput>;
809+
810+
private newtype TTestFailure =
811+
MkTestFailure(Test::FailureLocatable f, string message) { Test::testFailures(f, message) }
812+
813+
private predicate rankedTestFailures(int i, MkTestFailure f) {
814+
f =
815+
rank[i](MkTestFailure f0, Test::FailureLocatable fl, string message, string filename,
816+
int startLine, int startColumn, int endLine, int endColumn |
817+
f0 = MkTestFailure(fl, message) and
818+
fl.getLocation().hasLocationInfo(filename, startLine, startColumn, endLine, endColumn)
819+
|
820+
f0 order by filename, startLine, startColumn, endLine, endColumn, message
821+
)
822+
}
823+
824+
query predicate results(string relation, int row, int column, string data) {
825+
queryResults(relation, row, column, data)
826+
or
827+
exists(MkTestFailure f, Test::FailureLocatable fl, string message |
828+
relation = "testFailures" and
829+
rankedTestFailures(row, f) and
830+
f = MkTestFailure(fl, message)
831+
|
832+
column = 0 and data = Input2::getRelativeUrl(fl.getLocation())
833+
or
834+
column = 1 and data = fl.toString()
835+
or
836+
column = 2 and data = message
837+
)
838+
}
839+
840+
query predicate resultRelations(string relation) {
841+
queryRelations(relation)
842+
or
843+
Test::testFailures(_, _) and
844+
relation = "testFailures"
845+
}
846+
}
847+
}

0 commit comments

Comments
 (0)