diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml new file mode 100644 index 0000000..5eaaacb --- /dev/null +++ b/.github/workflows/reference_tests.yml @@ -0,0 +1,48 @@ +name: ReferenceTests +on: + push: + branches: [master] + pull_request: +jobs: + reftest: + name: Reference Test Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.10' + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - name: Checkout + uses: actions/checkout@v5 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v2 + - name: Install Julia dependencies + run: > + julia --project=reference -e 'using Pkg; Pkg.Registry.update(); Pkg.develop(path="."); Pkg.resolve(); Pkg.instantiate();' + - name: Run the reference tests + continue-on-error: true + run: > + julia --project=reference --code-coverage reference/compare.jl + && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV + - name: Upload test Artifacts + uses: actions/upload-artifact@v4 + with: + name: ReferenceImages_MathTeXEngine_${{ matrix.version }} + path: ./reference/reference_comparison_images/ + - name: Fail after artifacts if tests failed + if: ${{ env.TESTS_SUCCESSFUL != 'true' }} + run: exit 1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v5 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index ceecd32..87b959c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ Manifest.toml reference/**/*.png -experimental/ \ No newline at end of file +experimental/ +*.tar \ No newline at end of file diff --git a/reference/Project.toml b/reference/Project.toml index 02a4cf5..b1f0a18 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -4,4 +4,6 @@ FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" MathTeXEngine = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/reference/README.md b/reference/README.md index 099004e..bda0682 100644 --- a/reference/README.md +++ b/reference/README.md @@ -1,8 +1,5 @@ # Reference test -To run the reference tests, simply run `compare.jl` twice. +To run the reference tests, simply run `compare.jl`. This will download the reference files corresponding to the latest release. -First on the master branch, it will create the reference images. - -Then on the current branch to test, it will generate the corresponding -reference images and test they are the same. \ No newline at end of file +To generate new reference images, run `generate.jl`. This will generate `reference_images.tar`. diff --git a/reference/compare.jl b/reference/compare.jl index ed7e1d2..6308b74 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -2,49 +2,57 @@ using CairoMakie using FileIO using Git using MathTeXEngine +using Tar using Test +using TOML include("references.jl") -const git = Git.git() - -begin - readchomp(`$git fetch`) - master_id = readchomp(`$git rev-parse --short master`) - current_id = readchomp(`$git rev-parse --short HEAD`) - is_clean = isempty(readchomp(`$git status -s`)) - current_branch = readchomp(`$git rev-parse --abbrev-ref HEAD`) - - @info "Reference test started on branch $current_branch" - - if !is_clean - @warn "Using dirty commit for comparison" - current_id *= "-dirty" - end - - rm("reference/$current_id", recursive = true, force = true) - generate("reference/$current_id") - - if current_branch == "master" - @info "Reference test started on master branch, nothing to compare" - return - end +function latest_refimages_tag() + buff = IOBuffer() + run(pipeline(git(["tag", "--list"]) ; stdout = buff)) + tags = split(String(take!(buff))) + i = findlast(s -> (length(s) >= 9 && s[1:9] == "refimages"), tags) + return tags[i] +end - if !isdir("reference/$master_id") - @warn "No reference available for master commit $master_id, aborting" - @test_broken false - return +function download_refimages(tag = latest_refimages_tag()) + url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar" + images_tar = joinpath(@__DIR__, "reference_images.tar") + images = joinpath(@__DIR__, "reference_images") + if isfile(images_tar) + if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0"))) + @info "$images_tar already exists, skipping download as requested" + else + rm(images_tar) + end end + !isfile(images_tar) && download(url, images_tar) + isdir(images) && rm(images, recursive = true, force = true) + Tar.extract(images_tar, images) + return images +end - @info "Comparing reference on master $master_id with current commit $current_id on branch $current_branch" +@testset "Reference images" begin + @info "Reference test started" + @info "Downloading reference images" + reference_images = download_refimages() + + @info "Generating comparison images" + comparison_images = joinpath(@__DIR__, "comparison_images") + rm(comparison_images, recursive = true, force = true) + generate(comparison_images) + # Compare - rm("reference/comparisons", recursive = true, force = true) - path = mkpath("reference/comparisons") + reference_comparison_images = joinpath(@__DIR__, "reference_comparison_images") + rm(reference_comparison_images, recursive = true, force = true) + path = mkpath(reference_comparison_images) + @info "Comparing images" for group in keys(inputs) - refimg = load("reference/$master_id/$group.png") - img = load("reference/$current_id/$group.png") + refimg = load(joinpath(reference_images, "$group.png")) + img = load(joinpath(comparison_images, "$group.png")) if img != refimg @info "Saving the reference comparison for '$group'." diff --git a/reference/generate.jl b/reference/generate.jl new file mode 100644 index 0000000..de3fb4a --- /dev/null +++ b/reference/generate.jl @@ -0,0 +1,11 @@ +using Tar + +include("references.jl") + +begin + @info "Generating reference images" + reference_images = joinpath(@__DIR__, "reference_images") + rm(reference_images, recursive = true, force = true) + generate(reference_images) + Tar.create(reference_images, "reference_images.tar") +end diff --git a/reference/references.jl b/reference/references.jl index b3bf399..1ccd984 100644 --- a/reference/references.jl +++ b/reference/references.jl @@ -49,7 +49,7 @@ inputs["integrals"] = [ L"\int \int \int" ] -input["linebreaks"] = [ +inputs["linebreaks"] = [ L"we clearly see $x = 22$\\and $y > x^2$" ] @@ -125,4 +125,4 @@ function single_figure(exprs) end end return fig, failures -end \ No newline at end of file +end diff --git a/src/engine/layout.jl b/src/engine/layout.jl index d30c340..43bcac6 100644 --- a/src/engine/layout.jl +++ b/src/engine/layout.jl @@ -348,4 +348,4 @@ function generate_tex_elements(str, font_family=FontFamily()) end layout = tex_layout(expr, font_family) return unravel(layout) -end \ No newline at end of file +end