From a6a419b93fb8df9e5ccce69f92fb99a39824e963 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Thu, 26 Aug 2021 16:01:53 +0100 Subject: [PATCH 01/19] added j2k compression option with fixes psnr of 45 --- isyntax2raw/__init__.py | 5 ++++- register_j2k.py | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 register_j2k.py diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index f128f1f..78788b2 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -19,6 +19,8 @@ import softwarerenderbackend import zarr +from register_j2k import j2k + from datetime import datetime from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait from threading import BoundedSemaphore @@ -516,7 +518,8 @@ def create_tile_directory(self, series, resolution, width, height): self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), shape=(1, 1, 3, height, width), - chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' + chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B', + compressor=j2k(psnr=45) ) def make_planar(self, pixels, tile_width, tile_height): diff --git a/register_j2k.py b/register_j2k.py new file mode 100644 index 0000000..711b279 --- /dev/null +++ b/register_j2k.py @@ -0,0 +1,45 @@ +from numcodecs.abc import Codec +from numcodecs.compat import ensure_ndarray, ensure_bytes +from numcodecs.compat import ndarray_copy +from numcodecs.registry import register_codec +import glymur +import tempfile + +class j2k(Codec): + """Codec providing j2k compression via Glymur. + Parameters + ---------- + psnr : int + Compression peak signal noise ratio. + """ + + codec_id = "j2k" + + def __init__(self, psnr=50): + self.psnr = psnr + assert (self.psnr > 0 and self.psnr <= 100 + and isinstance(self.psnr, int)) + super().__init__() + + def encode(self, buf): + bufa = ensure_ndarray(buf) + tmp = tempfile.NamedTemporaryFile() + buff = glymur.Jp2k(tmp.name, shape=bufa.shape) + buff._write(bufa, psnr=[30], numres=1) + f = open(tmp.name, 'rb') + array = f.read() + return array + + def decode(self, buf, out=None): + buf = ensure_bytes(buf) + if out is not None: + out = ensure_bytes(out) + tmp = tempfile.NamedTemporaryFile(delete=False) + with open(tmp.name, "wb") as fb: + fb.write(buf) + jp2 = glymur.Jp2k(tmp.name) + fullres = jp2[:] + tiled = fullres + return ndarray_copy(tiled, out) + +register_codec(j2k) \ No newline at end of file From 65de27b312b4742adf1ba39c3e4d1c60f7cdf288 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Fri, 27 Aug 2021 10:40:28 +0100 Subject: [PATCH 02/19] merged register_j2k.py to __init__.py --- isyntax2raw/__init__.py | 47 ++++++++++++++++++++++++++++++++++++++++- register_j2k.py | 45 --------------------------------------- 2 files changed, 46 insertions(+), 46 deletions(-) delete mode 100644 register_j2k.py diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 78788b2..dddc36c 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -19,7 +19,12 @@ import softwarerenderbackend import zarr -from register_j2k import j2k +from numcodecs.abc import Codec +from numcodecs.compat import ensure_ndarray, ensure_bytes +from numcodecs.compat import ndarray_copy +from numcodecs.registry import register_codec +import glymur +import tempfile from datetime import datetime from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait @@ -708,3 +713,43 @@ def create_patch_list( # order to identify the patches returned asynchronously patch_ids.append((x, y)) return patches, patch_ids + + +class j2k(Codec): + """Codec providing j2k compression via Glymur. + Parameters + ---------- + psnr : int + Compression peak signal noise ratio. + """ + + codec_id = "j2k" + + def __init__(self, psnr=50): + self.psnr = psnr + assert (self.psnr > 0 and self.psnr <= 100 + and isinstance(self.psnr, int)) + super().__init__() + + def encode(self, buf): + bufa = ensure_ndarray(buf) + tmp = tempfile.NamedTemporaryFile() + buff = glymur.Jp2k(tmp.name, shape=bufa.shape) + buff._write(bufa, psnr=[30], numres=1) + f = open(tmp.name, 'rb') + array = f.read() + return array + + def decode(self, buf, out=None): + buf = ensure_bytes(buf) + if out is not None: + out = ensure_bytes(out) + tmp = tempfile.NamedTemporaryFile(delete=False) + with open(tmp.name, "wb") as fb: + fb.write(buf) + jp2 = glymur.Jp2k(tmp.name) + fullres = jp2[:] + tiled = fullres + return ndarray_copy(tiled, out) + +register_codec(j2k) \ No newline at end of file diff --git a/register_j2k.py b/register_j2k.py deleted file mode 100644 index 711b279..0000000 --- a/register_j2k.py +++ /dev/null @@ -1,45 +0,0 @@ -from numcodecs.abc import Codec -from numcodecs.compat import ensure_ndarray, ensure_bytes -from numcodecs.compat import ndarray_copy -from numcodecs.registry import register_codec -import glymur -import tempfile - -class j2k(Codec): - """Codec providing j2k compression via Glymur. - Parameters - ---------- - psnr : int - Compression peak signal noise ratio. - """ - - codec_id = "j2k" - - def __init__(self, psnr=50): - self.psnr = psnr - assert (self.psnr > 0 and self.psnr <= 100 - and isinstance(self.psnr, int)) - super().__init__() - - def encode(self, buf): - bufa = ensure_ndarray(buf) - tmp = tempfile.NamedTemporaryFile() - buff = glymur.Jp2k(tmp.name, shape=bufa.shape) - buff._write(bufa, psnr=[30], numres=1) - f = open(tmp.name, 'rb') - array = f.read() - return array - - def decode(self, buf, out=None): - buf = ensure_bytes(buf) - if out is not None: - out = ensure_bytes(out) - tmp = tempfile.NamedTemporaryFile(delete=False) - with open(tmp.name, "wb") as fb: - fb.write(buf) - jp2 = glymur.Jp2k(tmp.name) - fullres = jp2[:] - tiled = fullres - return ndarray_copy(tiled, out) - -register_codec(j2k) \ No newline at end of file From 822b097e21125c257591165b699540aa2ea29288 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Thu, 2 Sep 2021 12:50:56 +0100 Subject: [PATCH 03/19] added psnr option, fixed bugs --- isyntax2raw/__init__.py | 9 ++++++--- isyntax2raw/cli/isyntax2raw.py | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index dddc36c..ca46b61 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -88,7 +88,7 @@ class WriteTiles(object): def __init__( self, tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, input_path, output_path, psnr ): self.tile_width = tile_width self.tile_height = tile_height @@ -99,6 +99,7 @@ def __init__( self.nested = nested self.input_path = input_path self.slide_directory = output_path + self.psnr = psnr render_context = softwarerendercontext.SoftwareRenderContext() render_backend = softwarerenderbackend.SoftwareRenderBackend() @@ -524,7 +525,7 @@ def create_tile_directory(self, series, resolution, width, height): "%s/%s" % (str(series), str(resolution)), shape=(1, 1, 3, height, width), chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B', - compressor=j2k(psnr=45) + compressor=j2k(self.psnr) ) def make_planar(self, pixels, tile_width, tile_height): @@ -732,10 +733,11 @@ def __init__(self, psnr=50): super().__init__() def encode(self, buf): + buf = np.squeeze(buf) bufa = ensure_ndarray(buf) tmp = tempfile.NamedTemporaryFile() buff = glymur.Jp2k(tmp.name, shape=bufa.shape) - buff._write(bufa, psnr=[30], numres=1) + buff._write(bufa, psnr=[self.psnr], numres=1) f = open(tmp.name, 'rb') array = f.read() return array @@ -750,6 +752,7 @@ def decode(self, buf, out=None): jp2 = glymur.Jp2k(tmp.name) fullres = jp2[:] tiled = fullres + print('Decoded Array Shape:',tiled.shape) return ndarray_copy(tiled, out) register_codec(j2k) \ No newline at end of file diff --git a/isyntax2raw/cli/isyntax2raw.py b/isyntax2raw/cli/isyntax2raw.py index 56a3152..dc215ca 100644 --- a/isyntax2raw/cli/isyntax2raw.py +++ b/isyntax2raw/cli/isyntax2raw.py @@ -64,16 +64,20 @@ def cli(): "--debug", is_flag=True, help="enable debugging", ) +@click.option( + "--psnr", default=50, show_default=True, + help="JPEG-2000 compression PSNR" +) @click.argument("input_path") @click.argument("output_path") def write_tiles( tile_width, tile_height, resolutions, max_workers, batch_size, - fill_color, nested, debug, input_path, output_path + fill_color, nested, debug, input_path, output_path, psnr ): setup_logging(debug) with WriteTiles( tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, input_path, output_path, psnr ) as wt: wt.write_metadata() wt.write_label_image() From 0b3b4e7601b41512e82ba1f138193fa6d04c2294 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Thu, 2 Sep 2021 12:58:55 +0100 Subject: [PATCH 04/19] flake8 --- isyntax2raw/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index ca46b61..b39eb71 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -752,7 +752,7 @@ def decode(self, buf, out=None): jp2 = glymur.Jp2k(tmp.name) fullres = jp2[:] tiled = fullres - print('Decoded Array Shape:',tiled.shape) return ndarray_copy(tiled, out) -register_codec(j2k) \ No newline at end of file + +register_codec(j2k) From 99a58607d31e1b278722e0d6202e949fa651b0a4 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Mon, 6 Sep 2021 14:27:38 +0100 Subject: [PATCH 05/19] added glymur to setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index da228cc..22971d6 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ def read(fname): 'numpy==1.17.3', 'zarr==2.8.1', 'kajiki==0.8.2', + 'Glymur==0.9.3', 'fsspec>=0.9.0', ], tests_require=[ From 98968cc883d35ff1cde2ccbab17686049236d4ed Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Fri, 10 Sep 2021 14:06:24 +0100 Subject: [PATCH 06/19] Correct dimension order vs OME-NGFF 0.2 --- isyntax2raw/__init__.py | 8 ++++---- isyntax2raw/resources/ome_template.xml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index f128f1f..d42c0c8 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -493,7 +493,7 @@ def write_image_type(self, image_type, series): for channel in range(0, 3): band = np.array(img.getdata(band=channel)) band.shape = (height, width) - tile[0, 0, channel] = band + tile[0, channel, 0] = band log.info("wrote %s image" % image_type) @@ -512,10 +512,10 @@ def create_tile_directory(self, series, resolution, width, height): # important to explicitly set the chunk size to 1 for non-XY dims # setting to None may cause all planes to be chunked together - # ordering is TZCYX and hard-coded since Z and T are not present + # ordering is TCZYX and hard-coded since Z and T are not present self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), - shape=(1, 1, 3, height, width), + shape=(1, 3, 1, height, width), chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' ) @@ -551,7 +551,7 @@ def write_tile( # disk (not interleaved RGB) pixels = self.make_planar(pixels, tile_width, tile_height) z = self.zarr_group["0/%d" % resolution] - z[0, 0, :, y_start:y_end, x_start:x_end] = pixels + z[0, :, 0, y_start:y_end, x_start:x_end] = pixels except Exception: log.error( "Failed to write tile [:, %d:%d, %d:%d]" % ( diff --git a/isyntax2raw/resources/ome_template.xml b/isyntax2raw/resources/ome_template.xml index b63a79b..53a84c0 100644 --- a/isyntax2raw/resources/ome_template.xml +++ b/isyntax2raw/resources/ome_template.xml @@ -7,7 +7,7 @@ ${image['description']} - ${image['acquisitionDate']} - @@ -41,7 +41,7 @@ ${image['acquisitionDate']} - Date: Fri, 10 Sep 2021 14:54:50 +0100 Subject: [PATCH 07/19] Add OME-NGFF 0.2 multiscales metadata --- isyntax2raw/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index d42c0c8..ab1ad92 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -388,6 +388,18 @@ def wait_any(self, regions): else: return self.pixel_engine.wait_any(regions) + def write_image_metadata(self, resolutions, series): + multiscales = [{ + 'metadata': { + 'method': 'pixelengine', + 'version': str(self.pixel_engine.version) + }, + 'version': '0.2', + 'datasets': [{'path': str(v)} for v in resolutions] + }] + z = self.zarr_group["%d" % series] + z.attrs['multiscales'] = multiscales + def write_metadata_json(self, metadata_file): '''write metadata to a JSON file''' @@ -494,6 +506,7 @@ def write_image_type(self, image_type, series): band = np.array(img.getdata(band=channel)) band.shape = (height, width) tile[0, channel, 0] = band + self.write_image_metadata(range(1), series) log.info("wrote %s image" % image_type) @@ -656,6 +669,7 @@ def write_tile( x_start, y_start, width, height )) wait(jobs, return_when=ALL_COMPLETED) + self.write_image_metadata(resolutions, 0) def create_patch_list( self, dim_ranges, tiles, tile_size, tile_directory From 8bd764a18e8b3e858465150c9873ae3d8d76b415 Mon Sep 17 00:00:00 2001 From: Muhanad Zahra Date: Tue, 14 Sep 2021 16:25:53 +0100 Subject: [PATCH 08/19] added thread locking --- isyntax2raw/__init__.py | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 63bf488..4db0859 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -12,6 +12,7 @@ import logging import math import os +import threading import numpy as np import pixelengine @@ -21,7 +22,6 @@ from numcodecs.abc import Codec from numcodecs.compat import ensure_ndarray, ensure_bytes -from numcodecs.compat import ndarray_copy from numcodecs.registry import register_codec import glymur import tempfile @@ -536,14 +536,9 @@ def create_tile_directory(self, series, resolution, width, height): # ordering is TCZYX and hard-coded since Z and T are not present self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), -<<<<<<< HEAD - shape=(1, 1, 3, height, width), + shape=(1, 3, 1, height, width), chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B', compressor=j2k(self.psnr) -======= - shape=(1, 3, 1, height, width), - chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' ->>>>>>> ome-ngff-v2 ) def make_planar(self, pixels, tile_width, tile_height): @@ -752,26 +747,31 @@ def __init__(self, psnr=50): super().__init__() def encode(self, buf): - buf = np.squeeze(buf) - bufa = ensure_ndarray(buf) - tmp = tempfile.NamedTemporaryFile() - buff = glymur.Jp2k(tmp.name, shape=bufa.shape) - buff._write(bufa, psnr=[self.psnr], numres=1) - f = open(tmp.name, 'rb') - array = f.read() + print("Locking") + with lock: + print("Locked") + buf = np.squeeze(buf) + bufa = ensure_ndarray(buf) + tmp = tempfile.NamedTemporaryFile() + buff = glymur.Jp2k(tmp.name, shape=bufa.shape) + buff._write(bufa, psnr=[self.psnr], numres=1) + f = open(tmp.name, 'rb') + array = f.read() return array def decode(self, buf, out=None): - buf = ensure_bytes(buf) - if out is not None: - out = ensure_bytes(out) - tmp = tempfile.NamedTemporaryFile(delete=False) - with open(tmp.name, "wb") as fb: - fb.write(buf) - jp2 = glymur.Jp2k(tmp.name) - fullres = jp2[:] - tiled = fullres - return ndarray_copy(tiled, out) - - + with lock: + buf = ensure_bytes(buf) + if out is not None: + out = ensure_bytes(out) + tmp = tempfile.NamedTemporaryFile() + with open(tmp.name, "wb") as fb: + fb.write(buf) + jp2 = glymur.Jp2k(tmp.name) + fullres = jp2[:] + tiled = fullres + return tiled + + +lock = threading.Lock() register_codec(j2k) From c1bdd98cae7cc567181b48badd32bb16436cc64f Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Thu, 16 Sep 2021 11:42:55 +0100 Subject: [PATCH 09/19] Fix some logic errors Firstly, `NamedTemporaryFile` cannot be opened more than once on Windows; the implementation was effectively broken completely on the platform. Consequently, temporary file usage has been moved to `mkstemp()` and handled accordingly. Furthermore, the `out` argument to the JPEG-2000 codec `Codec.decode()` implementation was not being handled correctly causing visual corruption with certain workflows (noteably napari). As of this commit glymur 0.9.4 is required and the constructor based API is now in use. --- isyntax2raw/__init__.py | 50 ++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 4db0859..6f11fab 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -21,7 +21,11 @@ import zarr from numcodecs.abc import Codec -from numcodecs.compat import ensure_ndarray, ensure_bytes +from numcodecs.compat import \ + ensure_bytes, \ + ensure_contiguous_ndarray, \ + ensure_ndarray, \ + ndarray_copy from numcodecs.registry import register_codec import glymur import tempfile @@ -747,30 +751,36 @@ def __init__(self, psnr=50): super().__init__() def encode(self, buf): - print("Locking") - with lock: - print("Locked") - buf = np.squeeze(buf) - bufa = ensure_ndarray(buf) - tmp = tempfile.NamedTemporaryFile() - buff = glymur.Jp2k(tmp.name, shape=bufa.shape) - buff._write(bufa, psnr=[self.psnr], numres=1) - f = open(tmp.name, 'rb') - array = f.read() + buf = ensure_ndarray(np.squeeze(buf)) + fd, fname = tempfile.mkstemp() + try: + buff = glymur.Jp2k( + fname, psnr=[self.psnr], shape=buf.shape, numres=1 + ) + buff._write(buf) + with open(fname, 'rb') as f: + array = f.read() + finally: + os.close(fd) + os.remove(fname) return array def decode(self, buf, out=None): - with lock: + fd, fname = tempfile.mkstemp() + try: + with open(fname, 'wb') as f: + f.write(buf) buf = ensure_bytes(buf) + jp2 = glymur.Jp2k(fname) if out is not None: - out = ensure_bytes(out) - tmp = tempfile.NamedTemporaryFile() - with open(tmp.name, "wb") as fb: - fb.write(buf) - jp2 = glymur.Jp2k(tmp.name) - fullres = jp2[:] - tiled = fullres - return tiled + out_view = ensure_contiguous_ndarray(out) + ndarray_copy(jp2[:], out_view) + else: + out = jp2[:] + return out + finally: + os.close(fd) + os.remove(fname) lock = threading.Lock() From 32689d3c4904497960b64bb2e62682fa8c83b7b5 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 20 Sep 2021 10:18:51 +0100 Subject: [PATCH 10/19] Switch to imagecodecs --- isyntax2raw/__init__.py | 31 ++++++------------------------- setup.py | 2 +- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 6f11fab..06f207b 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -27,7 +27,7 @@ ensure_ndarray, \ ndarray_copy from numcodecs.registry import register_codec -import glymur +import imagecodecs import tempfile from datetime import datetime @@ -735,7 +735,7 @@ def create_patch_list( class j2k(Codec): - """Codec providing j2k compression via Glymur. + """Codec providing j2k compression via imagecodecs. Parameters ---------- psnr : int @@ -751,36 +751,17 @@ def __init__(self, psnr=50): super().__init__() def encode(self, buf): - buf = ensure_ndarray(np.squeeze(buf)) - fd, fname = tempfile.mkstemp() - try: - buff = glymur.Jp2k( - fname, psnr=[self.psnr], shape=buf.shape, numres=1 - ) - buff._write(buf) - with open(fname, 'rb') as f: - array = f.read() - finally: - os.close(fd) - os.remove(fname) - return array + return imagecodecs.jpeg2k_encode(np.squeeze(buf), level=self.psnr) def decode(self, buf, out=None): - fd, fname = tempfile.mkstemp() - try: - with open(fname, 'wb') as f: - f.write(buf) buf = ensure_bytes(buf) - jp2 = glymur.Jp2k(fname) + decoded = imagecodecs.jpeg2k_decode(buf) if out is not None: out_view = ensure_contiguous_ndarray(out) - ndarray_copy(jp2[:], out_view) + ndarray_copy(decoded, out_view) else: - out = jp2[:] + out = decoded return out - finally: - os.close(fd) - os.remove(fname) lock = threading.Lock() diff --git a/setup.py b/setup.py index 22971d6..0a9b1a1 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ def read(fname): 'numpy==1.17.3', 'zarr==2.8.1', 'kajiki==0.8.2', - 'Glymur==0.9.3', + 'imagecodecs==2021.8.26', 'fsspec>=0.9.0', ], tests_require=[ From 6285b03afcc7e20f0bad5ac858227543d6c5c62e Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 20 Sep 2021 16:05:55 +0100 Subject: [PATCH 11/19] Switch to interleaved --- isyntax2raw/__init__.py | 27 +++++++++----------------- isyntax2raw/resources/ome_template.xml | 6 +++--- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 06f207b..9569e3b 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -515,9 +515,9 @@ def write_image_type(self, image_type, series): tile = self.zarr_group["%d/0" % series] tile.attrs['image type'] = image_type for channel in range(0, 3): - band = np.array(img.getdata(band=channel)) - band.shape = (height, width) - tile[0, channel, 0] = band + data = np.array(img.getdata()) + data.shape = (height, width, 3) + tile[0, 0, :] = data self.write_image_metadata(range(1), series) log.info("wrote %s image" % image_type) @@ -537,22 +537,15 @@ def create_tile_directory(self, series, resolution, width, height): # important to explicitly set the chunk size to 1 for non-XY dims # setting to None may cause all planes to be chunked together - # ordering is TCZYX and hard-coded since Z and T are not present + # ordering is TZYXC (interleaved) and hard-coded since Z and T + # are not present self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), - shape=(1, 3, 1, height, width), - chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B', + shape=(1, 1, height, width, 3), + chunks=(1, 1, self.tile_height, self.tile_width, 3), dtype='B', compressor=j2k(self.psnr) ) - def make_planar(self, pixels, tile_width, tile_height): - r = pixels[0::3] - g = pixels[1::3] - b = pixels[2::3] - for v in (r, g, b): - v.shape = (tile_height, tile_width) - return np.array([r, g, b]) - def write_pyramid(self): '''write the slide's pyramid as a set of tiles''' pe_in = self.pixel_engine["in"] @@ -573,11 +566,9 @@ def write_tile( x_end = x_start + tile_width y_end = y_start + tile_height try: - # Zarr has a single n-dimensional array representation on - # disk (not interleaved RGB) - pixels = self.make_planar(pixels, tile_width, tile_height) + pixels.shape = (tile_height, tile_width, 3) z = self.zarr_group["0/%d" % resolution] - z[0, :, 0, y_start:y_end, x_start:x_end] = pixels + z[0, 0, y_start:y_end, x_start:x_end, :] = pixels except Exception: log.error( "Failed to write tile [:, %d:%d, %d:%d]" % ( diff --git a/isyntax2raw/resources/ome_template.xml b/isyntax2raw/resources/ome_template.xml index 53a84c0..5df7793 100644 --- a/isyntax2raw/resources/ome_template.xml +++ b/isyntax2raw/resources/ome_template.xml @@ -8,7 +8,7 @@ @@ -42,7 +42,7 @@ Date: Tue, 21 Sep 2021 11:27:48 +0100 Subject: [PATCH 12/19] removed unused imports --- isyntax2raw/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 9569e3b..4182638 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -12,7 +12,6 @@ import logging import math import os -import threading import numpy as np import pixelengine @@ -28,7 +27,6 @@ ndarray_copy from numcodecs.registry import register_codec import imagecodecs -import tempfile from datetime import datetime from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait @@ -755,5 +753,4 @@ def decode(self, buf, out=None): return out -lock = threading.Lock() register_codec(j2k) From f391e30ee0a0578dbd8799b98c110b2b4c249018 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 11:44:37 +0100 Subject: [PATCH 13/19] Fix flake8 errors --- isyntax2raw/__init__.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 4182638..f84420a 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -23,7 +23,6 @@ from numcodecs.compat import \ ensure_bytes, \ ensure_contiguous_ndarray, \ - ensure_ndarray, \ ndarray_copy from numcodecs.registry import register_codec import imagecodecs @@ -743,14 +742,14 @@ def encode(self, buf): return imagecodecs.jpeg2k_encode(np.squeeze(buf), level=self.psnr) def decode(self, buf, out=None): - buf = ensure_bytes(buf) - decoded = imagecodecs.jpeg2k_decode(buf) - if out is not None: - out_view = ensure_contiguous_ndarray(out) - ndarray_copy(decoded, out_view) - else: - out = decoded - return out + buf = ensure_bytes(buf) + decoded = imagecodecs.jpeg2k_decode(buf) + if out is not None: + out_view = ensure_contiguous_ndarray(out) + ndarray_copy(decoded, out_view) + else: + out = decoded + return out register_codec(j2k) From a053f15a9380cce504f6da1cd1a16dc42dd294e6 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 11:45:32 +0100 Subject: [PATCH 14/19] Reflect Python 3.7 requirement in setuptools and actions --- .github/workflows/workflow.yml | 1 - setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d3fa4be..b408776 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -14,7 +14,6 @@ jobs: fail-fast: false matrix: python-version: - - '3.6' - '3.7' - '3.8' - '3.9' diff --git a/setup.py b/setup.py index 0a9b1a1..a98ee97 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def read(fname): setup(name='isyntax2raw', version=version.getVersion(), - python_requires='>=3.6', + python_requires='>=3.7', description='iSyntax to raw format converter', long_description=read('README.md'), long_description_content_type='text/markdown', From 4303084f2cbc19d0d3100964549d6e996571c24f Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 11:59:07 +0100 Subject: [PATCH 15/19] Install an updated version of setuptools, wheel --- appveyor.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 71468ed..b4b72c9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,11 +1,9 @@ image: ubuntu1804 -install: - - sudo apt-get install -y python3-setuptools python3-wheel twine - build: off build_script: + - pip install --user -U setuptools wheel - python3 setup.py build test_script: From ad95779de9c8ff0fe550dce2c4399081df8b698c Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 12:05:47 +0100 Subject: [PATCH 16/19] Need base pip3 first --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b4b72c9..c76934e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,8 +2,11 @@ image: ubuntu1804 build: off +install: + - sudo apt-get install -y python3-pip + build_script: - - pip install --user -U setuptools wheel + - pip3 install --user -U pip setuptools wheel - python3 setup.py build test_script: From 89230c04ae867e76102e7384dd94c81eb93581b4 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 12:17:11 +0100 Subject: [PATCH 17/19] Adjust flake8 --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c76934e..d145b33 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,11 +6,11 @@ install: - sudo apt-get install -y python3-pip build_script: - - pip3 install --user -U pip setuptools wheel + - pip3 install --user -U pip setuptools wheel flake8 - python3 setup.py build test_script: - - python3 setup.py flake8 + - flake8 . after_test: - python3 setup.py bdist_wheel bdist_egg From c464132a0378dd5cfb932ff0f2edc4f520a2ef2d Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 12:23:36 +0100 Subject: [PATCH 18/19] Need PATH to flake8 --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index d145b33..db21a5b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,6 +7,7 @@ install: build_script: - pip3 install --user -U pip setuptools wheel flake8 + - export PATH="$(python3 -m site --user-base)/bin:${PATH}" - python3 setup.py build test_script: From d69107ff64ad39a94f8a2da178589e4f6fe08e76 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Mon, 25 Oct 2021 14:15:31 +0100 Subject: [PATCH 19/19] Do not build egg --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index db21a5b..e36e90e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ test_script: - flake8 . after_test: - - python3 setup.py bdist_wheel bdist_egg + - python3 setup.py bdist_wheel artifacts: - path: dist/*