Make image reference tests work on CI

.github/workflows/reference_tests.yml

@@ -0,0 +1,48 @@
+name: ReferenceTests
+on:
+  push:
+    branches: [master]
+  pull_request:
+jobs:
+  reftest:
+    name: Reference Test Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.10'
+          - 'nightly'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/cache@v2
+      - name: Install Julia dependencies
+        run: >
+          julia --project=reference -e 'using Pkg; Pkg.Registry.update(); Pkg.develop(path="."); Pkg.resolve(); Pkg.instantiate();'
+      - name: Run the reference tests
+        continue-on-error: true
+        run: >
+          julia --project=reference --code-coverage reference/compare.jl
+          && echo "TESTS_SUCCESSFUL=true" >> $GITHUB_ENV
+      - name: Upload test Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ReferenceImages_MathTeXEngine_${{ matrix.version }}
+          path: ./reference/reference_comparison_images/
+      - name: Fail after artifacts if tests failed
+        if: ${{ env.TESTS_SUCCESSFUL != 'true' }}
+        run: exit 1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v5
+        with:
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -1,3 +1,4 @@
 Manifest.toml
 reference/**/*.png
-experimental/
+experimental/
+*.tar

reference/Project.toml

@@ -4,4 +4,6 @@ FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
 MathTeXEngine = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

reference/README.md

@@ -1,8 +1,5 @@
 # Reference test
 
-To run the reference tests, simply run `compare.jl` twice.
+To run the reference tests, simply run `compare.jl`. This will download the reference files corresponding to the latest release.
 
-First on the master branch, it will create the reference images.
-
-Then on the current branch to test, it will generate the corresponding
-reference images and test they are the same.
+To generate new reference images, run `generate.jl`. This will generate `reference_images.tar`.

reference/compare.jl

@@ -2,49 +2,57 @@ using CairoMakie
 using FileIO
 using Git
 using MathTeXEngine
+using Tar
 using Test
+using TOML
 
 include("references.jl")
 
-const git = Git.git()
-
-begin
-    readchomp(`$git fetch`)
-    master_id = readchomp(`$git rev-parse --short master`)
-    current_id = readchomp(`$git rev-parse --short HEAD`)
-    is_clean = isempty(readchomp(`$git status -s`))
-    current_branch = readchomp(`$git rev-parse --abbrev-ref HEAD`)
-
-    @info "Reference test started on branch $current_branch"
-
-    if !is_clean
-        @warn "Using dirty commit for comparison"
-        current_id *= "-dirty"
-    end
-
-    rm("reference/$current_id", recursive = true, force = true)
-    generate("reference/$current_id")
-
-    if current_branch == "master"
-        @info "Reference test started on master branch, nothing to compare"
-        return
-    end
+function latest_refimages_tag()
+    buff = IOBuffer()
+    run(pipeline(git(["tag", "--list"]) ; stdout = buff))
+    tags = split(String(take!(buff)))
+    i = findlast(s -> (length(s) >= 9 && s[1:9] == "refimages"), tags)
+    return tags[i]
+end
 
-    if !isdir("reference/$master_id")
-        @warn "No reference available for master commit $master_id, aborting"
-        @test_broken false
-        return
+function download_refimages(tag = latest_refimages_tag())
+    url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar"
+    images_tar = joinpath(@__DIR__, "reference_images.tar")
+    images = joinpath(@__DIR__, "reference_images")
+    if isfile(images_tar)
+        if Bool(parse(Int, get(ENV, "REUSE_IMAGES_TAR", "0")))
+            @info "$images_tar already exists, skipping download as requested"
+        else
+            rm(images_tar)
+        end
     end
+    !isfile(images_tar) && download(url, images_tar)
+    isdir(images) && rm(images, recursive = true, force = true)
+    Tar.extract(images_tar, images)
+    return images
+end
 
-    @info "Comparing reference on master $master_id with current commit $current_id on branch $current_branch"
+@testset "Reference images" begin
+    @info "Reference test started"
 
+    @info "Downloading reference images"
+    reference_images = download_refimages()
+
+    @info "Generating comparison images"
+    comparison_images = joinpath(@__DIR__, "comparison_images")
+    rm(comparison_images, recursive = true, force = true)
+    generate(comparison_images)
+
     # Compare
-    rm("reference/comparisons", recursive = true, force = true)
-    path = mkpath("reference/comparisons")
+    reference_comparison_images = joinpath(@__DIR__, "reference_comparison_images")
+    rm(reference_comparison_images, recursive = true, force = true)
+    path = mkpath(reference_comparison_images)
 
+    @info "Comparing images"
     for group in keys(inputs)
-        refimg = load("reference/$master_id/$group.png")
-        img = load("reference/$current_id/$group.png")
+        refimg = load(joinpath(reference_images, "$group.png"))
+        img = load(joinpath(comparison_images, "$group.png"))
 
         if img != refimg
             @info "Saving the reference comparison for '$group'."

reference/generate.jl

@@ -0,0 +1,11 @@
+using Tar
+
+include("references.jl")
+
+begin
+    @info "Generating reference images"
+    reference_images = joinpath(@__DIR__, "reference_images")
+    rm(reference_images, recursive = true, force = true)
+    generate(reference_images)
+    Tar.create(reference_images, "reference_images.tar")
+end

reference/references.jl

@@ -49,7 +49,7 @@ inputs["integrals"] = [
     L"\int \int \int"
 ]
 
-input["linebreaks"] = [
+inputs["linebreaks"] = [
     L"we clearly see $x = 22$\\and $y > x^2$"
 ]
 
@@ -125,4 +125,4 @@ function single_figure(exprs)
         end 
     end
     return fig, failures
-end
+end

src/engine/layout.jl

@@ -348,4 +348,4 @@ function generate_tex_elements(str, font_family=FontFamily())
     end
     layout = tex_layout(expr, font_family)
     return unravel(layout)
-end
+end