From 4403f45f1b4e433e3511b9aec9448ac8c3402e81 Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Fri, 3 Oct 2025 17:48:56 +0200 Subject: [PATCH 1/9] Make image reference tests work on CI (WIP) --- reference/Project.toml | 3 +- reference/compare.jl | 70 +++++++++++++++++++++-------------------- reference/generate.jl | 11 +++++++ reference/references.jl | 4 +-- 4 files changed, 51 insertions(+), 37 deletions(-) create mode 100644 reference/generate.jl diff --git a/reference/Project.toml b/reference/Project.toml index 02a4cf5..5a9fa70 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -1,7 +1,8 @@ [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" MathTeXEngine = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" +Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" diff --git a/reference/compare.jl b/reference/compare.jl index ed7e1d2..e1fac0a 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -1,50 +1,52 @@ using CairoMakie using FileIO -using Git using MathTeXEngine +using Tar using Test +using TOML include("references.jl") -const git = Git.git() - -begin - readchomp(`$git fetch`) - master_id = readchomp(`$git rev-parse --short master`) - current_id = readchomp(`$git rev-parse --short HEAD`) - is_clean = isempty(readchomp(`$git status -s`)) - current_branch = readchomp(`$git rev-parse --abbrev-ref HEAD`) - - @info "Reference test started on branch $current_branch" - - if !is_clean - @warn "Using dirty commit for comparison" - current_id *= "-dirty" - end - - rm("reference/$current_id", recursive = true, force = true) - generate("reference/$current_id") - - if current_branch == "master" - @info "Reference test started on master branch, nothing to compare" - return - end +function last_major_version() + path = joinpath(@__DIR__, "..", "Project.toml") + version = VersionNumber(TOML.parse(String(read(path)))["version"]) + return "v" * string(VersionNumber(version.major, version.minor)) +end - if !isdir("reference/$master_id") - @warn "No reference available for master commit $master_id, aborting" - @test_broken false - return +function download_refimages(tag = last_major_version()) + url = "https://github.com/user-attachments/files/22683734/reference_images.tgz" + images_tar = joinpath(@__DIR__, "reference_images.tar") + images = joinpath(@__DIR__, "reference_images") + if isfile(images_tar) + if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0"))) + @info "$images_tar already exists, skipping download as requested" + else + rm(images_tar) + end end + !isfile(images_tar) && download(url, images_tar) + isdir(images) && rm(images, recursive = true, force = true) + Tar.extract(images_tar, images) + return images +end - @info "Comparing reference on master $master_id with current commit $current_id on branch $current_branch" +begin + @info "Reference test started" + reference_images = download_refimages() + comparison_images = joinpath(@__DIR__, "comparison_images") + + rm(comparison_images, recursive = true, force = true) + generate(comparison_images) + # Compare - rm("reference/comparisons", recursive = true, force = true) - path = mkpath("reference/comparisons") + reference_comparison_images = joinpath(@__DIR__, "reference_comparison_images") + rm(reference_comparison_images, recursive = true, force = true) + path = mkpath(reference_comparison_images) for group in keys(inputs) - refimg = load("reference/$master_id/$group.png") - img = load("reference/$current_id/$group.png") + refimg = load(joinpath(reference_images, "$group.png")) + img = load(joinpath(comparison_images, "$group.png")) if img != refimg @info "Saving the reference comparison for '$group'." @@ -59,6 +61,6 @@ begin save(joinpath(path, "$group.png"), fig) end - @test img == refimg + @show @test img == refimg end end diff --git a/reference/generate.jl b/reference/generate.jl new file mode 100644 index 0000000..de3fb4a --- /dev/null +++ b/reference/generate.jl @@ -0,0 +1,11 @@ +using Tar + +include("references.jl") + +begin + @info "Generating reference images" + reference_images = joinpath(@__DIR__, "reference_images") + rm(reference_images, recursive = true, force = true) + generate(reference_images) + Tar.create(reference_images, "reference_images.tar") +end diff --git a/reference/references.jl b/reference/references.jl index b3bf399..1ccd984 100644 --- a/reference/references.jl +++ b/reference/references.jl @@ -49,7 +49,7 @@ inputs["integrals"] = [ L"\int \int \int" ] -input["linebreaks"] = [ +inputs["linebreaks"] = [ L"we clearly see $x = 22$\\and $y > x^2$" ] @@ -125,4 +125,4 @@ function single_figure(exprs) end end return fig, failures -end \ No newline at end of file +end From 93eb6cc05e2a08cdd0e043cfa1ada34fca1fe4b3 Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 08:52:18 +0200 Subject: [PATCH 2/9] Initial attempt with CI --- .github/workflows/reference_tests.yml | 48 +++++++++++++++++++ reference/Project.toml | 3 ++ reference/README.md | 7 +-- reference/compare.jl | 11 +++-- reference/generate.jl | 3 +- .../MathTeXEngineReferenceTests.jl} | 6 +++ reference/test/runtests.jl | 1 + 7 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/reference_tests.yml rename reference/{references.jl => src/MathTeXEngineReferenceTests.jl} (97%) create mode 100644 reference/test/runtests.jl diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml new file mode 100644 index 0000000..36f690e --- /dev/null +++ b/.github/workflows/reference_tests.yml @@ -0,0 +1,48 @@ +name: ReferenceTests +on: + push: + pull_request: +jobs: + reftest: + name: Reference Test Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - name: Checkout + uses: actions/checkout@v5 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v2 + - name: Install Julia dependencies + run: > + julia --project=reference -e 'using Pkg; Pkg.resolve(); Pkg.instantiate()' + - name: Run the reference tests + continue-on-error: true + run: > + julia --project=reference -e 'using Pkg; Pkg.test(coverage=true)' + && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV + - name: Upload test Artifacts + uses: actions/upload-artifact@v4 + with: + name: ReferenceImages_MathTeXEngine_${{ matrix.version }} + path: ./reference/reference_comparison_images/ + - name: Fail after artifacts if tests failed + if: ${{ env.TESTS_SUCCESSFUL != 'true' }} + run: exit 1 + - uses: julia-actions/julia-processcoverage@v1 + with: + directories: reference/src + - uses: codecov/codecov-action@v5 + with: + file: lcov.info diff --git a/reference/Project.toml b/reference/Project.toml index 5a9fa70..26da24e 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -1,3 +1,6 @@ +name = "MathTeXEngineReferenceTests" +uuid = "2f3f2751-15d8-4cee-a4c9-fb36372069b7" + [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" diff --git a/reference/README.md b/reference/README.md index 099004e..0b5fdf9 100644 --- a/reference/README.md +++ b/reference/README.md @@ -1,8 +1,5 @@ # Reference test -To run the reference tests, simply run `compare.jl` twice. +To run the reference tests, simply run `compare.jl` or run the tests for `MathTeXEngineReferenceTests`. This will download the reference files corresponding to the latest release. -First on the master branch, it will create the reference images. - -Then on the current branch to test, it will generate the corresponding -reference images and test they are the same. \ No newline at end of file +To generate new reference images, run `generate.jl`. This will generate `reference_images.tar`. diff --git a/reference/compare.jl b/reference/compare.jl index e1fac0a..ec2e188 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -5,7 +5,7 @@ using Tar using Test using TOML -include("references.jl") +using MathTeXEngineReferenceTests function last_major_version() path = joinpath(@__DIR__, "..", "Project.toml") @@ -30,12 +30,14 @@ function download_refimages(tag = last_major_version()) return images end -begin +@testset "Reference images" begin @info "Reference test started" + @info "Downloading reference images" reference_images = download_refimages() - comparison_images = joinpath(@__DIR__, "comparison_images") + @info "Generating comparison images" + comparison_images = joinpath(@__DIR__, "comparison_images") rm(comparison_images, recursive = true, force = true) generate(comparison_images) @@ -44,6 +46,7 @@ begin rm(reference_comparison_images, recursive = true, force = true) path = mkpath(reference_comparison_images) + @info "Comparing images" for group in keys(inputs) refimg = load(joinpath(reference_images, "$group.png")) img = load(joinpath(comparison_images, "$group.png")) @@ -61,6 +64,6 @@ begin save(joinpath(path, "$group.png"), fig) end - @show @test img == refimg + @test img == refimg end end diff --git a/reference/generate.jl b/reference/generate.jl index de3fb4a..57be12d 100644 --- a/reference/generate.jl +++ b/reference/generate.jl @@ -1,6 +1,5 @@ using Tar - -include("references.jl") +using MathTeXEngineReferenceTests begin @info "Generating reference images" diff --git a/reference/references.jl b/reference/src/MathTeXEngineReferenceTests.jl similarity index 97% rename from reference/references.jl rename to reference/src/MathTeXEngineReferenceTests.jl index 1ccd984..167f180 100644 --- a/reference/references.jl +++ b/reference/src/MathTeXEngineReferenceTests.jl @@ -1,6 +1,10 @@ +module MathTeXEngineReferenceTests + using CairoMakie using MathTeXEngine +export inputs, generate + inputs = Dict() inputs["accents"] = [ @@ -126,3 +130,5 @@ function single_figure(exprs) end return fig, failures end + +end # module diff --git a/reference/test/runtests.jl b/reference/test/runtests.jl new file mode 100644 index 0000000..f4f2c3b --- /dev/null +++ b/reference/test/runtests.jl @@ -0,0 +1 @@ +include("../compare.jl") From 49cfc8b75c73e3893287bb66af88591435bc87bc Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 08:55:19 +0200 Subject: [PATCH 3/9] Needs Pkg update? --- .github/workflows/reference_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml index 36f690e..f9489df 100644 --- a/.github/workflows/reference_tests.yml +++ b/.github/workflows/reference_tests.yml @@ -26,7 +26,7 @@ jobs: - uses: julia-actions/cache@v2 - name: Install Julia dependencies run: > - julia --project=reference -e 'using Pkg; Pkg.resolve(); Pkg.instantiate()' + julia --project=reference -e 'using Pkg; Pkg.update(); Pkg.resolve(); Pkg.instantiate()' - name: Run the reference tests continue-on-error: true run: > From dbb3eb010c9e110376c73ddb63e52a07439df023 Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 11:01:44 +0100 Subject: [PATCH 4/9] Ensure that MathTeXEngine gets dev-ed --- .github/workflows/reference_tests.yml | 7 ++++--- reference/Project.toml | 3 --- reference/compare.jl | 2 +- reference/generate.jl | 3 ++- .../{src/MathTeXEngineReferenceTests.jl => references.jl} | 6 ------ reference/test/runtests.jl | 1 - 6 files changed, 7 insertions(+), 15 deletions(-) rename reference/{src/MathTeXEngineReferenceTests.jl => references.jl} (97%) delete mode 100644 reference/test/runtests.jl diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml index f9489df..eed33ee 100644 --- a/.github/workflows/reference_tests.yml +++ b/.github/workflows/reference_tests.yml @@ -26,11 +26,11 @@ jobs: - uses: julia-actions/cache@v2 - name: Install Julia dependencies run: > - julia --project=reference -e 'using Pkg; Pkg.update(); Pkg.resolve(); Pkg.instantiate()' + julia --project=reference -e 'using Pkg; Pkg.Registry.update(); Pkg.develop(path="."); Pkg.resolve(); Pkg.instantiate();' - name: Run the reference tests continue-on-error: true run: > - julia --project=reference -e 'using Pkg; Pkg.test(coverage=true)' + julia --project=reference reference/compare.jl && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - name: Upload test Artifacts uses: actions/upload-artifact@v4 @@ -45,4 +45,5 @@ jobs: directories: reference/src - uses: codecov/codecov-action@v5 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/reference/Project.toml b/reference/Project.toml index 26da24e..5a9fa70 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -1,6 +1,3 @@ -name = "MathTeXEngineReferenceTests" -uuid = "2f3f2751-15d8-4cee-a4c9-fb36372069b7" - [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" diff --git a/reference/compare.jl b/reference/compare.jl index ec2e188..2914f9e 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -5,7 +5,7 @@ using Tar using Test using TOML -using MathTeXEngineReferenceTests +include("references.jl") function last_major_version() path = joinpath(@__DIR__, "..", "Project.toml") diff --git a/reference/generate.jl b/reference/generate.jl index 57be12d..de3fb4a 100644 --- a/reference/generate.jl +++ b/reference/generate.jl @@ -1,5 +1,6 @@ using Tar -using MathTeXEngineReferenceTests + +include("references.jl") begin @info "Generating reference images" diff --git a/reference/src/MathTeXEngineReferenceTests.jl b/reference/references.jl similarity index 97% rename from reference/src/MathTeXEngineReferenceTests.jl rename to reference/references.jl index 167f180..1ccd984 100644 --- a/reference/src/MathTeXEngineReferenceTests.jl +++ b/reference/references.jl @@ -1,10 +1,6 @@ -module MathTeXEngineReferenceTests - using CairoMakie using MathTeXEngine -export inputs, generate - inputs = Dict() inputs["accents"] = [ @@ -130,5 +126,3 @@ function single_figure(exprs) end return fig, failures end - -end # module diff --git a/reference/test/runtests.jl b/reference/test/runtests.jl deleted file mode 100644 index f4f2c3b..0000000 --- a/reference/test/runtests.jl +++ /dev/null @@ -1 +0,0 @@ -include("../compare.jl") From 4ff2bfc3035b693bdee4d864805281de063fbc76 Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 12:19:20 +0100 Subject: [PATCH 5/9] Introduce error to test that dev-ed version is used --- .github/workflows/reference_tests.yml | 2 -- src/engine/layout.jl | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml index eed33ee..76266e7 100644 --- a/.github/workflows/reference_tests.yml +++ b/.github/workflows/reference_tests.yml @@ -41,8 +41,6 @@ jobs: if: ${{ env.TESTS_SUCCESSFUL != 'true' }} run: exit 1 - uses: julia-actions/julia-processcoverage@v1 - with: - directories: reference/src - uses: codecov/codecov-action@v5 with: files: lcov.info diff --git a/src/engine/layout.jl b/src/engine/layout.jl index d30c340..c52aaa0 100644 --- a/src/engine/layout.jl +++ b/src/engine/layout.jl @@ -4,7 +4,7 @@ middle of the xheight. """ function y_for_centered(font_family, elem) h = inkheight(elem) - return h/2 + xheight(font_family)/2 + return h + xheight(font_family)/2 end function argument_as_string(arg) @@ -348,4 +348,4 @@ function generate_tex_elements(str, font_family=FontFamily()) end layout = tex_layout(expr, font_family) return unravel(layout) -end \ No newline at end of file +end From 7abdc5e3ae761801c188130065361755e5096bc8 Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 13:31:32 +0100 Subject: [PATCH 6/9] Use a change that generates an error --- src/engine/layout.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/layout.jl b/src/engine/layout.jl index c52aaa0..5fdd2a3 100644 --- a/src/engine/layout.jl +++ b/src/engine/layout.jl @@ -4,7 +4,7 @@ middle of the xheight. """ function y_for_centered(font_family, elem) h = inkheight(elem) - return h + xheight(font_family)/2 + return h/2 + xheight(font_family)/2 end function argument_as_string(arg) @@ -25,7 +25,7 @@ function tex_layout(expr, state) font_family = state.font_family head = expr.head args = [expr.args...] - shrink = 0.6 + shrink = 0.1 try if isleaf(expr) # :char, :delimiter, :digit, :punctuation, :symbol From 8a37df76247e4accd06e41fb345714e537d966be Mon Sep 17 00:00:00 2001 From: "David A. W. Barton" Date: Sat, 4 Oct 2025 13:43:03 +0100 Subject: [PATCH 7/9] Revert intentional error and add code coverage --- .github/workflows/reference_tests.yml | 3 ++- src/engine/layout.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml index 76266e7..5eaaacb 100644 --- a/.github/workflows/reference_tests.yml +++ b/.github/workflows/reference_tests.yml @@ -1,6 +1,7 @@ name: ReferenceTests on: push: + branches: [master] pull_request: jobs: reftest: @@ -30,7 +31,7 @@ jobs: - name: Run the reference tests continue-on-error: true run: > - julia --project=reference reference/compare.jl + julia --project=reference --code-coverage reference/compare.jl && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV - name: Upload test Artifacts uses: actions/upload-artifact@v4 diff --git a/src/engine/layout.jl b/src/engine/layout.jl index 5fdd2a3..43bcac6 100644 --- a/src/engine/layout.jl +++ b/src/engine/layout.jl @@ -25,7 +25,7 @@ function tex_layout(expr, state) font_family = state.font_family head = expr.head args = [expr.args...] - shrink = 0.1 + shrink = 0.6 try if isleaf(expr) # :char, :delimiter, :digit, :punctuation, :symbol From 0a7cbc1b8d4920afa5bf6cb3aef06c8f6ff626e2 Mon Sep 17 00:00:00 2001 From: David Barton Date: Sat, 4 Oct 2025 14:48:29 +0100 Subject: [PATCH 8/9] Remove mention of running tests for MathTeXEngineReferenceTests [skipCI] --- reference/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/README.md b/reference/README.md index 0b5fdf9..bda0682 100644 --- a/reference/README.md +++ b/reference/README.md @@ -1,5 +1,5 @@ # Reference test -To run the reference tests, simply run `compare.jl` or run the tests for `MathTeXEngineReferenceTests`. This will download the reference files corresponding to the latest release. +To run the reference tests, simply run `compare.jl`. This will download the reference files corresponding to the latest release. To generate new reference images, run `generate.jl`. This will generate `reference_images.tar`. From 5944f1a7b810ed79bab9d773f12d6fbb5b9cf1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Tue, 25 Nov 2025 18:07:41 +0100 Subject: [PATCH 9/9] Automatise finding the latest refimages tag --- .gitignore | 3 ++- reference/Project.toml | 3 ++- reference/compare.jl | 15 +++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index ceecd32..87b959c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ Manifest.toml reference/**/*.png -experimental/ \ No newline at end of file +experimental/ +*.tar \ No newline at end of file diff --git a/reference/Project.toml b/reference/Project.toml index 5a9fa70..b1f0a18 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -1,8 +1,9 @@ [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" MathTeXEngine = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" diff --git a/reference/compare.jl b/reference/compare.jl index 2914f9e..6308b74 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -1,5 +1,6 @@ using CairoMakie using FileIO +using Git using MathTeXEngine using Tar using Test @@ -7,14 +8,16 @@ using TOML include("references.jl") -function last_major_version() - path = joinpath(@__DIR__, "..", "Project.toml") - version = VersionNumber(TOML.parse(String(read(path)))["version"]) - return "v" * string(VersionNumber(version.major, version.minor)) +function latest_refimages_tag() + buff = IOBuffer() + run(pipeline(git(["tag", "--list"]) ; stdout = buff)) + tags = split(String(take!(buff))) + i = findlast(s -> (length(s) >= 9 && s[1:9] == "refimages"), tags) + return tags[i] end -function download_refimages(tag = last_major_version()) - url = "https://github.com/user-attachments/files/22683734/reference_images.tgz" +function download_refimages(tag = latest_refimages_tag()) + url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar" images_tar = joinpath(@__DIR__, "reference_images.tar") images = joinpath(@__DIR__, "reference_images") if isfile(images_tar)