Skip to content

Add support for separator alignment in sequences #1594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
olabusayoT wants to merge 3 commits into from
Closed

Add support for separator alignment in sequences #1594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7111fa5
Add support for separator alignment in sequences
olabusayoT Nov 24, 2025
8323d7e
fixup! Add support for separator alignment in sequences
olabusayoT Dec 3, 2025
1b6f98d
fixup! MTA are alignments not lengths
olabusayoT Dec 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 75 additions & 21 deletions daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ import org.apache.daffodil.core.dsom.ElementBase
import org.apache.daffodil.core.dsom.ModelGroup
import org.apache.daffodil.core.dsom.QuasiElementDeclBase
import org.apache.daffodil.core.dsom.Root
import org.apache.daffodil.core.dsom.SequenceTermBase
import org.apache.daffodil.core.dsom.Term
import org.apache.daffodil.lib.exceptions.Assert
import org.apache.daffodil.lib.schema.annotation.props.gen.AlignmentKind
import org.apache.daffodil.lib.schema.annotation.props.gen.AlignmentUnits
import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind
import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits
import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition
import org.apache.daffodil.lib.util.Math

case class AlignmentMultipleOf(nBits: Long) {
Expand Down Expand Up @@ -65,7 +67,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
final lazy val isKnownToBeAligned: Boolean = LV(Symbol("isKnownToBeAligned")) {
if (!isRepresented || (alignmentKindDefaulted == AlignmentKind.Manual)) true
else {
val pa = priorAlignmentWithLeadingSkipApprox
val pa = priorAlignmentWithLeftFramingApprox
val aa = alignmentApprox
val res = (pa % aa) == 0
res
Expand All @@ -85,7 +87,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
else if (isKnownEncoding) {
if (knownEncodingAlignmentInBits == 1)
true
else if (priorAlignmentWithLeadingSkipApprox.nBits % knownEncodingAlignmentInBits == 0)
else if (priorAlignmentWithLeftFramingApprox.nBits % knownEncodingAlignmentInBits == 0)
true
else
false
Expand Down Expand Up @@ -134,11 +136,14 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
LengthExact(trailingSkipInBits)
}

// FIXME: DAFFODIL-2295
// Does not take into account that in a sequence, what may be prior may be a separator.
// The separator is text in some encoding, might not be the same as this term's encoding, and
// the alignment will be left where that text leaves it.
//
/**
* The priorAlignmentApprox doesn't need to take into account the separator
* alignment/length as that comes after the priorAlignmentApprox, but before
* the leadingSkip.
*
* TODO: DAFFODIL-3056 We need to consider the initiator MTA/length,
* as well as the prefix length
*/
private lazy val priorAlignmentApprox: AlignmentMultipleOf =
LV(Symbol("priorAlignmentApprox")) {
if (this.isInstanceOf[Root] || this.isInstanceOf[QuasiElementDeclBase]) {
Expand Down Expand Up @@ -201,7 +206,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
// Return 0 here, unordered alignment will be handled by unorderedSequenceSelfAlignment
AlignmentMultipleOf(0)
} else {
ps.endingAlignmentApprox
ps.endingAlignmentWithRightFramingApprox
}
eaa
}
Expand All @@ -228,14 +233,59 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
}
}.value

private lazy val priorAlignmentWithLeadingSkipApprox: AlignmentMultipleOf = {
priorAlignmentApprox + leadingSkipApprox
private lazy val separatorPrefixMTAApprox =
this.optLexicalParent match {
case Some(s: SequenceTermBase) if s.hasSeparator =>
import SeparatorPosition.*
s.separatorPosition match {
case Prefix | Infix => AlignmentMultipleOf(s.knownEncodingAlignmentInBits)
case Postfix => AlignmentMultipleOf(0)
}
case _ => AlignmentMultipleOf(0)
}

private lazy val separatorPostfixMTAApprox =
this.optLexicalParent match {
case Some(s: SequenceTermBase) if s.hasSeparator =>
import SeparatorPosition.*
s.separatorPosition match {
case Prefix | Infix => AlignmentMultipleOf(0)
case Postfix => AlignmentMultipleOf(s.knownEncodingAlignmentInBits)
}
case _ => AlignmentMultipleOf(0)
}

private lazy val separatorLengthApprox = this.optLexicalParent match {
case Some(s: SequenceTermBase) if s.hasSeparator =>
getEncodingLengthApprox(s)
case _ => LengthMultipleOf(0)
}

private def getEncodingLengthApprox(t: Term) = {
if (t.isKnownEncoding) {
val dfdlCharset = t.charsetEv.optConstant.get
val fixedWidth =
if (dfdlCharset.maybeFixedWidth.isDefined) dfdlCharset.maybeFixedWidth.get else 8
LengthMultipleOf(fixedWidth)
} else {
// runtime encoding, which must be a byte-length encoding
LengthMultipleOf(8)
}
}

private lazy val priorAlignmentWithLeftFramingApprox: AlignmentMultipleOf = {
val pawls =
(priorAlignmentApprox
* separatorPrefixMTAApprox)
+ separatorLengthApprox
+ leadingSkipApprox
pawls
}

protected lazy val contentStartAlignment: AlignmentMultipleOf = {
if (priorAlignmentWithLeadingSkipApprox % alignmentApprox == 0) {
if (priorAlignmentWithLeftFramingApprox % alignmentApprox == 0) {
// alignment won't be needed, continue using prior alignment as start alignment
priorAlignmentWithLeadingSkipApprox
priorAlignmentWithLeftFramingApprox
} else {
// alignment will be needed, it will be forced to be aligned to alignmentApprox
alignmentApprox
Expand Down Expand Up @@ -291,6 +341,18 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
}
}

/**
* The postfix separator MTA/length needs to be added after the trailing skip
*
* TODO: DAFFODIL-3057 needs to consider terminator MTA/length before trailingSkip
*/
protected lazy val endingAlignmentWithRightFramingApprox: AlignmentMultipleOf = {
val res = (endingAlignmentApprox
* separatorPostfixMTAApprox)
+ separatorLengthApprox
res
}

protected lazy val elementSpecifiedLengthApprox: LengthApprox = {
this match {
case eb: ElementBase => {
Expand Down Expand Up @@ -329,15 +391,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
}

private lazy val encodingLengthApprox: LengthApprox = {
if (isKnownEncoding) {
val dfdlCharset = charsetEv.optConstant.get
val fixedWidth =
if (dfdlCharset.maybeFixedWidth.isDefined) dfdlCharset.maybeFixedWidth.get else 8
LengthMultipleOf(fixedWidth)
} else {
// runtime encoding, which must be a byte-length encoding
LengthMultipleOf(8)
}
getEncodingLengthApprox(this)
}

protected lazy val isKnownToBeByteAlignedAndByteLength: Boolean = {
Expand Down
185 changes: 176 additions & 9 deletions daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,6 @@
separatorPosition="infix"
emptyElementParsePolicy="treatAsEmpty"/>

<!-- DAFFODIL-2217
These additional tests give another case where trailingEmptyStrict should be causing a
parse error, but it does not in a case where maxOccurs="3" but works correctly when maxOccurs="4".
-->

<!-- maxOccurs='3' fails to detect the trailing "/" i.e., this should fail with a parse error
due to trailingEmptyStrict -->

<xs:element name="file1">
<xs:complexType>
<xs:sequence dfdl:separator="/" dfdl:separatorPosition="infix"
Expand All @@ -160,7 +152,6 @@
</xs:complexType>
</xs:element>

<!-- same, but maxOccurs is 4. This works, as in detects the error. -->
<xs:element name="file2">
<xs:complexType>
<xs:sequence dfdl:separator="/" dfdl:separatorPosition="infix"
Expand Down Expand Up @@ -680,4 +671,180 @@
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>

<tdml:defineSchema name="s8" elementFormDefault="unqualified">

<xs:include schemaLocation="/org/apache/daffodil/xsd/DFDLGeneralFormatPortable.dfdl.xsd"/>
<dfdl:format
ref="ex:GeneralFormat"
representation="text"
lengthKind="delimited"
separatorPosition="infix"
lengthUnits="bits"
alignmentUnits="bits"
/>

<xs:element name="infix">
<xs:complexType>
<xs:sequence>
<xs:element name="e1" dfdl:lengthKind="implicit" dfdl:fillByte="E">
<xs:complexType>
<xs:sequence
dfdl:separator="/"
dfdl:separatorPosition="infix" dfdl:fillByte="S">
<xs:element name="a" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="A"/>
<xs:element name="b" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="B"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="prefix">
<xs:complexType>
<xs:sequence>
<xs:element name="e1" dfdl:lengthKind="implicit" dfdl:fillByte="E">
<xs:complexType>
<xs:sequence
dfdl:separator="/"
dfdl:separatorPosition="prefix" dfdl:fillByte="S">
<xs:element name="a" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="A"/>
<xs:element name="b" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="B"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="postfix">
<xs:complexType>
<xs:sequence>
<xs:element name="e1" dfdl:lengthKind="implicit" dfdl:fillByte="E">
<xs:complexType>
<xs:sequence
dfdl:separator="/"
dfdl:separatorPosition="postfix" dfdl:fillByte="S">
<xs:element name="a" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="A"/>
<xs:element name="b" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="B"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="postfix_2">
<xs:complexType>
<xs:sequence>
<xs:element name="e1" dfdl:lengthKind="implicit" dfdl:fillByte="E">
<xs:complexType>
<xs:sequence dfdl:separator="-" dfdl:separatorPosition="postfix" dfdl:fillByte="0">
<xs:element name="c" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="C"/>
<xs:sequence
dfdl:separator="/"
dfdl:separatorPosition="postfix" dfdl:fillByte="1">
<xs:element name="a" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="A"/>
<xs:element name="b" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="B"/>
</xs:sequence>
<xs:element name="d" type="xs:string"
dfdl:length="16"
dfdl:lengthKind="explicit" dfdl:alignment="16"
dfdl:fillByte="D"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</tdml:defineSchema>

<tdml:unparserTestCase name="test_sep_alignment_1" root="infix" model="s8" roundTrip="onePass">
<tdml:document>aa/Bbb</tdml:document>
<tdml:infoset>
<tdml:dfdlInfoset>
<ex:infix xmlns:ex="http://example.com">
<e1>
<a>aa</a>
<b>bb</b>
</e1>
</ex:infix>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:unparserTestCase>

<tdml:unparserTestCase name="test_sep_alignment_2" root="prefix" model="s8" roundTrip="onePass">
<tdml:document>/Aaa/Bbb</tdml:document>
<tdml:infoset>
<tdml:dfdlInfoset>
<ex:prefix xmlns:ex="http://example.com">
<e1>
<a>aa</a>
<b>bb</b>
</e1>
</ex:prefix>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:unparserTestCase>

<tdml:unparserTestCase name="test_sep_alignment_3" root="postfix" model="s8" roundTrip="onePass">
<tdml:document>aa/Bbb/</tdml:document>

<tdml:infoset>
<tdml:dfdlInfoset>
<ex:postfix xmlns:ex="http://example.com">
<e1>
<a>aa</a>
<b>bb</b>
</e1>
</ex:postfix>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:unparserTestCase>

<tdml:unparserTestCase name="test_sep_alignment_4" root="postfix_2" model="s8" roundTrip="onePass">
<tdml:document>cc-Aaa/Bbb/-dd-</tdml:document>

<tdml:infoset>
<tdml:dfdlInfoset>
<ex:postfix_2 xmlns:ex="http://example.com">
<e1>
<c>cc</c>
<a>aa</a>
<b>bb</b>
<d>dd</d>
</e1>
</ex:postfix_2>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:unparserTestCase>

</tdml:testSuite>
7 changes: 7 additions & 0 deletions daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,11 @@ class TestSepTests extends TdmlTests {

// DAFFODIL-2791
@Test def test_treatAsAbsent_occursIndex = test

// DAFFODIL-2295
@Test def test_sep_alignment_1 = test
@Test def test_sep_alignment_2 = test
@Test def test_sep_alignment_3 = test

@Test def test_sep_alignment_4 = test
}
Loading