Skip to content

Commit c2fc8e9

Browse files
committed
feat: add optional PLT and TLM markers during encoding
These can significantly improve decoding performance when pulling a small area out of a very large file, where the decoder supports it. OpenJPEG supports using the TLM marker as of uclouvain/openjpeg#1538
1 parent 39b7d5b commit c2fc8e9

File tree

4 files changed

+294
-2
lines changed

4 files changed

+294
-2
lines changed

lib/interface/encode.c

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ extern int EncodeArray(
3737
int use_mct,
3838
PyObject *compression_ratios,
3939
PyObject *signal_noise_ratios,
40-
int codec_format
40+
int codec_format,
41+
int add_tlm,
42+
int add_plt
4143
)
4244
{
4345
/* Encode a numpy ndarray using JPEG 2000.
@@ -64,6 +66,10 @@ extern int EncodeArray(
6466
The format of the encoded JPEG 2000 data, one of:
6567
* ``0`` - OPJ_CODEC_J2K : JPEG-2000 codestream
6668
* ``1`` - OPJ_CODEC_JP2 : JP2 file format
69+
add_tlm : int
70+
Add tile-part data length markers (TLM). Supported values 0-1.
71+
add_plt : int
72+
Add packet length tile-part header markers (PLT). Supported values 0-1.
6773
6874
Returns
6975
-------
@@ -411,6 +417,24 @@ extern int EncodeArray(
411417
goto failure;
412418
}
413419

420+
const char* extra_options[3] = { NULL, NULL, NULL };
421+
int extra_option_index = 0;
422+
if (add_plt) {
423+
extra_options[extra_option_index] = "PLT=YES";
424+
extra_option_index += 1;
425+
}
426+
if (add_tlm) {
427+
extra_options[extra_option_index] = "TLM=YES";
428+
extra_option_index += 1;
429+
}
430+
if (extra_option_index > 0) {
431+
if (! opj_encoder_set_extra_options(codec, extra_options)) {
432+
py_error("Failed to set extra options on the encoder");
433+
return_code = 28;
434+
goto failure;
435+
}
436+
}
437+
414438
/* Send info, warning, error message to Python logging */
415439
opj_set_info_handler(codec, info_callback, NULL);
416440
opj_set_warning_handler(codec, warning_callback, NULL);
@@ -490,7 +514,9 @@ extern int EncodeBuffer(
490514
unsigned int use_mct,
491515
PyObject *compression_ratios,
492516
PyObject *signal_noise_ratios,
493-
int codec_format
517+
int codec_format,
518+
int add_tlm,
519+
int add_plt
494520
)
495521
{
496522
/* Encode image data using JPEG 2000.
@@ -527,6 +553,10 @@ extern int EncodeBuffer(
527553
The format of the encoded JPEG 2000 data, one of:
528554
* ``0`` - OPJ_CODEC_J2K : JPEG-2000 codestream
529555
* ``1`` - OPJ_CODEC_JP2 : JP2 file format
556+
add_tlm : int
557+
Add tile-part data length markers (TLM). Supported values 0-1.
558+
add_plt : int
559+
Add packet length tile-part header markers (PLT). Supported values 0-1.
530560
531561
Returns
532562
-------
@@ -891,6 +921,24 @@ extern int EncodeBuffer(
891921
goto failure;
892922
}
893923

924+
const char* extra_options[3] = { NULL, NULL, NULL };
925+
int extra_option_index = 0;
926+
if (add_plt) {
927+
extra_options[extra_option_index] = "PLT=YES";
928+
extra_option_index += 1;
929+
}
930+
if (add_tlm) {
931+
extra_options[extra_option_index] = "TLM=YES";
932+
extra_option_index += 1;
933+
}
934+
if (extra_option_index > 0) {
935+
if (! opj_encoder_set_extra_options(codec, extra_options)) {
936+
py_error("Failed to set extra options on the encoder");
937+
return_code = 28;
938+
goto failure;
939+
}
940+
}
941+
894942
/* Send info, warning, error message to Python logging */
895943
opj_set_info_handler(codec, info_callback, NULL);
896944
opj_set_warning_handler(codec, warning_callback, NULL);

openjpeg/_openjpeg.pyx

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ cdef extern int EncodeArray(
3232
PyObject* compression_ratios,
3333
PyObject* signal_noise_ratios,
3434
int codec_format,
35+
bint add_tlm,
36+
bint add_plt,
3537
)
3638
cdef extern int EncodeBuffer(
3739
PyObject* src,
@@ -46,6 +48,8 @@ cdef extern int EncodeBuffer(
4648
PyObject* compression_ratios,
4749
PyObject* signal_noise_ratios,
4850
int codec_format,
51+
bint add_tlm,
52+
bint add_plt,
4953
)
5054

5155

@@ -213,6 +217,8 @@ def encode_array(
213217
List[float] compression_ratios,
214218
List[float] signal_noise_ratios,
215219
int codec_format,
220+
bint add_tlm,
221+
bint add_plt,
216222
) -> Tuple[int, bytes]:
217223
"""Return the JPEG 2000 compressed `arr`.
218224

@@ -239,6 +245,10 @@ def encode_array(
239245

240246
* ``0``: JPEG 2000 codestream only (default) (J2K/J2C format)
241247
* ``1``: A boxed JPEG 2000 codestream (JP2 format)
248+
add_tlm : bool
249+
If ``True`` then add tile-part length markers (TLM) to the codestream.
250+
add_plt : bool
251+
If ``True`` then add packet length tile-part header markers (PLT) to the codestream.
242252

243253
Returns
244254
-------
@@ -318,6 +328,8 @@ def encode_array(
318328
<PyObject *> compression_ratios,
319329
<PyObject *> signal_noise_ratios,
320330
codec_format,
331+
add_tlm,
332+
add_plt,
321333
)
322334
return return_code, dst.getvalue()
323335

@@ -334,6 +346,9 @@ def encode_buffer(
334346
List[float] compression_ratios,
335347
List[float] signal_noise_ratios,
336348
int codec_format,
349+
bint add_tlm,
350+
bint add_plt,
351+
337352
) -> Tuple[int, bytes]:
338353
"""Return the JPEG 2000 compressed `src`.
339354

@@ -375,6 +390,10 @@ def encode_buffer(
375390

376391
* ``0``: JPEG 2000 codestream only (default) (J2K/J2C format)
377392
* ``1``: A boxed JPEG 2000 codestream (JP2 format)
393+
add_tlm : bool
394+
If ``True`` then add tile-part length markers (TLM) to the codestream.
395+
add_plt : bool
396+
If ``True`` then add packet length tile-part header markers (PLT) to the codestream.
378397

379398
Returns
380399
-------
@@ -466,5 +485,7 @@ def encode_buffer(
466485
<PyObject *> compression_ratios,
467486
<PyObject *> signal_noise_ratios,
468487
codec_format,
488+
add_tlm,
489+
add_plt,
469490
)
470491
return return_code, dst.getvalue()

openjpeg/tests/test_encode.py

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,57 @@ def parse_j2k(buffer):
7070

7171
return param
7272

73+
def parse_codestream_markers(buffer):
74+
offset = 0
75+
markers = []
76+
while offset < len(buffer):
77+
symbol_code_bytes = buffer[offset: offset + 2]
78+
marker = unpack(">H", symbol_code_bytes)[0]
79+
offset += 2
80+
if marker == 0xFF4F:
81+
markers.append("SOC")
82+
elif marker == 0xFF51:
83+
markers.append("SIZ")
84+
length_bytes = buffer[offset: offset + 2]
85+
length = unpack(">H", length_bytes)[0]
86+
offset += length
87+
elif marker == 0xFF52:
88+
markers.append("COD")
89+
length_bytes = buffer[offset: offset + 2]
90+
length = unpack(">H", length_bytes)[0]
91+
offset += length
92+
elif marker == 0xFF55:
93+
markers.append("TLM")
94+
length_bytes = buffer[offset: offset + 2]
95+
length = unpack(">H", length_bytes)[0]
96+
offset += length
97+
elif marker == 0xFF58:
98+
markers.append("PLT")
99+
length_bytes = buffer[offset: offset + 2]
100+
length = unpack(">H", length_bytes)[0]
101+
offset += length
102+
elif marker == 0xFF5C:
103+
markers.append("QCD")
104+
length_bytes = buffer[offset: offset + 2]
105+
length = unpack(">H", length_bytes)[0]
106+
offset += length
107+
elif marker == 0xFF64:
108+
markers.append("COM")
109+
length_bytes = buffer[offset: offset + 2]
110+
length = unpack(">H", length_bytes)[0]
111+
offset += length
112+
elif marker == 0xFF90:
113+
markers.append("SOT")
114+
length_bytes = buffer[offset: offset + 2]
115+
length = unpack(">H", length_bytes)[0]
116+
offset += length
117+
elif marker == 0xFF93:
118+
markers.append("SOD")
119+
# If we get to here, we have the marker info we need
120+
break
121+
else:
122+
raise Exception(f"unexpected marker: 0x{marker:04X}")
123+
return markers
73124

74125
class TestEncode:
75126
"""Tests for encode_array()"""
@@ -701,6 +752,82 @@ def test_jp2(self):
701752
buffer = encode_array(arr, codec_format=1)
702753
assert buffer.startswith(b"\x00\x00\x00\x0c\x6a\x50\x20\x20\x0d\x0a\x87\x0a")
703754

755+
def test_no_tlm_or_plt_explicit(self):
756+
"""Test encoding with no TLM or PLT, explicitly disabled"""
757+
rows = 123
758+
cols = 234
759+
bit_depth = 8
760+
maximum = 2**bit_depth - 1
761+
dtype = f"u{math.ceil(bit_depth / 8)}"
762+
arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
763+
buffer = encode_array(arr, compression_ratios=[4, 2, 1], add_tlm=False, add_plt=False)
764+
out = decode(buffer)
765+
markers = parse_codestream_markers(buffer)
766+
assert "TLM" not in markers
767+
assert "PLT" not in markers
768+
assert np.allclose(arr, out, atol=5)
769+
770+
def test_no_tlm_or_plt_default(self):
771+
"""Test encoding with no TLM or PLT, default options"""
772+
rows = 123
773+
cols = 234
774+
bit_depth = 8
775+
maximum = 2**bit_depth - 1
776+
dtype = f"u{math.ceil(bit_depth / 8)}"
777+
arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
778+
buffer = encode_array(arr, compression_ratios=[4, 2, 1])
779+
out = decode(buffer)
780+
markers = parse_codestream_markers(buffer)
781+
assert "TLM" not in markers
782+
assert "PLT" not in markers
783+
assert np.allclose(arr, out, atol=5)
784+
785+
def test_tlm(self):
786+
"""Test encoding with TLM"""
787+
rows = 123
788+
cols = 234
789+
bit_depth = 8
790+
maximum = 2**bit_depth - 1
791+
dtype = f"u{math.ceil(bit_depth / 8)}"
792+
arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
793+
buffer = encode_array(arr, compression_ratios=[4, 2, 1], add_tlm=True)
794+
out = decode(buffer)
795+
markers = parse_codestream_markers(buffer)
796+
assert "TLM" in markers
797+
assert "PLT" not in markers
798+
assert np.allclose(arr, out, atol=5)
799+
800+
def test_plt(self):
801+
"""Test encoding with PLT"""
802+
rows = 123
803+
cols = 234
804+
bit_depth = 8
805+
maximum = 2**bit_depth - 1
806+
dtype = f"u{math.ceil(bit_depth / 8)}"
807+
arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
808+
buffer = encode_array(arr, compression_ratios=[4, 2, 1], add_plt=True)
809+
out = decode(buffer)
810+
markers = parse_codestream_markers(buffer)
811+
assert "TLM" not in markers
812+
assert "PLT" in markers
813+
assert np.allclose(arr, out, atol=5)
814+
815+
def test_tlm_and_plt(self):
816+
"""Test encoding with both TLM and PLT"""
817+
rows = 123
818+
cols = 234
819+
bit_depth = 8
820+
maximum = 2**bit_depth - 1
821+
dtype = f"u{math.ceil(bit_depth / 8)}"
822+
arr = np.random.randint(0, high=maximum + 1, size=(rows, cols), dtype=dtype)
823+
buffer = encode_array(arr, compression_ratios=[4, 2, 1], add_plt=True, add_tlm=True)
824+
out = decode(buffer)
825+
param = parse_j2k(buffer)
826+
assert param["precision"] == bit_depth
827+
markers = parse_codestream_markers(buffer)
828+
assert "TLM" in markers
829+
assert "PLT" in markers
830+
assert np.allclose(arr, out, atol=5)
704831

705832
class TestEncodeBuffer:
706833
"""Tests for _openjpeg.encode_buffer"""
@@ -2034,6 +2161,80 @@ def test_unused_bits_u1(self):
20342161
assert out.dtype.kind == "u"
20352162
assert np.array_equal(arr, out)
20362163

2164+
def test_no_tlm_or_plt_explicit(self):
2165+
"""Test encoding with no TLM and PLT, explicitly disabled"""
2166+
rows = 123
2167+
cols = 234
2168+
bit_depth = 8
2169+
arr = np.random.randint(0, high=256, size=(rows, cols), dtype="u1")
2170+
buffer = encode_buffer(arr.tobytes(), cols, rows, samples_per_pixel=1, bits_stored=bit_depth, is_signed=False, add_plt=False, add_tlm=False)
2171+
out = decode(buffer)
2172+
param = parse_j2k(buffer)
2173+
assert param["precision"] == bit_depth
2174+
markers = parse_codestream_markers(buffer)
2175+
assert "TLM" not in markers
2176+
assert "PLT" not in markers
2177+
assert np.allclose(arr, out, atol=5)
2178+
2179+
def test_no_tlm_or_plt_default(self):
2180+
"""Test encoding with no TLM and PLT, default options"""
2181+
rows = 123
2182+
cols = 234
2183+
bit_depth = 8
2184+
arr = np.random.randint(0, high=256, size=(rows, cols), dtype="u1")
2185+
buffer = encode_buffer(arr.tobytes(), cols, rows, samples_per_pixel=1, bits_stored=bit_depth, is_signed=False)
2186+
out = decode(buffer)
2187+
param = parse_j2k(buffer)
2188+
assert param["precision"] == bit_depth
2189+
markers = parse_codestream_markers(buffer)
2190+
assert "TLM" not in markers
2191+
assert "PLT" not in markers
2192+
assert np.allclose(arr, out, atol=5)
2193+
2194+
def test_tlm(self):
2195+
"""Test encoding with TLM"""
2196+
rows = 123
2197+
cols = 234
2198+
bit_depth = 8
2199+
arr = np.random.randint(0, high=256, size=(rows, cols), dtype="u1")
2200+
buffer = encode_buffer(arr.tobytes(), cols, rows, samples_per_pixel=1, bits_stored=bit_depth, is_signed=False, add_tlm=True)
2201+
out = decode(buffer)
2202+
param = parse_j2k(buffer)
2203+
assert param["precision"] == bit_depth
2204+
markers = parse_codestream_markers(buffer)
2205+
assert "TLM" in markers
2206+
assert "PLT" not in markers
2207+
assert np.allclose(arr, out, atol=5)
2208+
2209+
def test_plt(self):
2210+
"""Test encoding with PLT"""
2211+
rows = 123
2212+
cols = 234
2213+
bit_depth = 8
2214+
arr = np.random.randint(0, high=256, size=(rows, cols), dtype="u1")
2215+
buffer = encode_buffer(arr.tobytes(), cols, rows, samples_per_pixel=1, bits_stored=bit_depth, is_signed=False, add_plt=True)
2216+
out = decode(buffer)
2217+
param = parse_j2k(buffer)
2218+
assert param["precision"] == bit_depth
2219+
markers = parse_codestream_markers(buffer)
2220+
assert "TLM" not in markers
2221+
assert "PLT" in markers
2222+
assert np.allclose(arr, out, atol=5)
2223+
2224+
def test_tlm_and_plt(self):
2225+
"""Test encoding with both TLM and PLT"""
2226+
rows = 123
2227+
cols = 234
2228+
bit_depth = 8
2229+
arr = np.random.randint(0, high=256, size=(rows, cols), dtype="u1")
2230+
buffer = encode_buffer(arr.tobytes(), cols, rows, samples_per_pixel=1, bits_stored=bit_depth, is_signed=False, add_plt=True, add_tlm=True)
2231+
out = decode(buffer)
2232+
param = parse_j2k(buffer)
2233+
assert param["precision"] == bit_depth
2234+
markers = parse_codestream_markers(buffer)
2235+
assert "TLM" in markers
2236+
assert "PLT" in markers
2237+
assert np.allclose(arr, out, atol=5)
20372238

20382239
class TestEncodePixelData:
20392240
"""Tests for encode_pixel_data()"""

0 commit comments

Comments
 (0)