diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala
index a6d52b8a5c..f2d3ceb4ed 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala
@@ -21,17 +21,28 @@ import org.apache.daffodil.core.dsom.ElementBase
import org.apache.daffodil.core.dsom.ModelGroup
import org.apache.daffodil.core.dsom.QuasiElementDeclBase
import org.apache.daffodil.core.dsom.Root
+import org.apache.daffodil.core.dsom.SequenceTermBase
import org.apache.daffodil.core.dsom.Term
import org.apache.daffodil.lib.exceptions.Assert
import org.apache.daffodil.lib.schema.annotation.props.gen.AlignmentKind
import org.apache.daffodil.lib.schema.annotation.props.gen.AlignmentUnits
import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind.Explicit
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind.Implicit
+import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind.Prefixed
import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits
+import org.apache.daffodil.lib.schema.annotation.props.gen.Representation
+import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition
import org.apache.daffodil.lib.util.Math
case class AlignmentMultipleOf(nBits: Long) {
+ // To combine alignments that could all happen at the same point in data, use *,
+ // To add new alignment to an existing approximate alignment, use +
def *(that: AlignmentMultipleOf) = AlignmentMultipleOf(Math.gcd(nBits, that.nBits))
def %(that: AlignmentMultipleOf) = nBits % that.nBits
+ def +(that: AlignmentMultipleOf) =
+ if (this.nBits % that.nBits == 0) this else that
+ // To combine AlignmentMultipleOfs and lengths, use +
def +(that: LengthApprox) = AlignmentMultipleOf(Math.gcd(nBits, nBits + that.nBits))
}
@@ -63,13 +74,15 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
* Hence we are guaranteed to be properly aligned.
*/
final lazy val isKnownToBeAligned: Boolean = LV(Symbol("isKnownToBeAligned")) {
- if (!isRepresented || (alignmentKindDefaulted == AlignmentKind.Manual)) true
- else {
- val pa = priorAlignmentWithLeadingSkipApprox
- val aa = alignmentApprox
- val res = (pa % aa) == 0
- res
- }
+ val res =
+ if (!isRepresented || (alignmentKindDefaulted == AlignmentKind.Manual)) true
+ else {
+ val pa = priorAlignmentWithLeadingSkipApprox
+ val aa = alignmentApprox
+ val res = (pa % aa) == 0
+ res
+ }
+ res
}.value
/**
@@ -81,33 +94,37 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
* considers the surrounding context meeting the alignment needs.
*/
final lazy val isKnownToBeTextAligned: Boolean = LV(Symbol("isKnownToBeTextAligned")) {
- if (alignmentKindDefaulted == AlignmentKind.Manual) true // manual alignment
- else if (isKnownEncoding) {
- if (knownEncodingAlignmentInBits == 1)
- true
- else if (priorAlignmentWithLeadingSkipApprox.nBits % knownEncodingAlignmentInBits == 0)
+ val res =
+ if (alignmentKindDefaulted == AlignmentKind.Manual) true // manual alignment
+ else if (isKnownEncoding) {
+ if (knownEncodingAlignmentInBits == 1)
+ true
+ else if (priorAlignmentWithLeadingSkipApprox.nBits % knownEncodingAlignmentInBits == 0)
+ true
+ else
+ false
+ } else if (schemaSet.root.isScannable)
true
else
false
- } else if (schemaSet.root.isScannable)
- true
- else
- false
+ res
}.value
final lazy val isDelimiterKnownToBeTextAligned: Boolean = {
- if (alignmentKindDefaulted == AlignmentKind.Manual) true // manual alignment
- else if (isKnownEncoding) {
- if (knownEncodingAlignmentInBits == 1)
- true
- else if (endingAlignmentApprox.nBits % knownEncodingAlignmentInBits == 0)
+ val res =
+ if (alignmentKindDefaulted == AlignmentKind.Manual) true // manual alignment
+ else if (isKnownEncoding) {
+ if (knownEncodingAlignmentInBits == 1)
+ true
+ else if (contentEndAlignment.nBits % knownEncodingAlignmentInBits == 0)
+ true
+ else
+ false
+ } else if (schemaSet.root.isScannable)
true
else
false
- } else if (schemaSet.root.isScannable)
- true
- else
- false
+ res
}
final lazy val hasNoSkipRegions = LV(Symbol("hasNoSkipRegions")) {
@@ -134,17 +151,17 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
LengthExact(trailingSkipInBits)
}
- // FIXME: DAFFODIL-2295
- // Does not take into account that in a sequence, what may be prior may be a separator.
- // The separator is text in some encoding, might not be the same as this term's encoding, and
- // the alignment will be left where that text leaves it.
- //
+ /**
+ * The priorAlignmentApprox doesn't need to take into account the separator
+ * alignment/length as that comes after the priorAlignmentApprox, but before
+ * the leadingSkip.
+ */
private lazy val priorAlignmentApprox: AlignmentMultipleOf =
LV(Symbol("priorAlignmentApprox")) {
if (this.isInstanceOf[Root] || this.isInstanceOf[QuasiElementDeclBase]) {
AlignmentMultipleOf(
0
- ) // root and quasi elements are aligned with anything // TODO: really? Why quasi-elements - they should have implicit alignment ?
+ ) // root and quasi elements are aligned with anything (quasi elements are always 1 aligned)
} else {
val priorSibs = potentialPriorTerms
val arraySelfAlignment =
@@ -201,7 +218,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
// Return 0 here, unordered alignment will be handled by unorderedSequenceSelfAlignment
AlignmentMultipleOf(0)
} else {
- ps.endingAlignmentApprox
+ ps.endingAlignmentWithRightFramingApprox
}
eaa
}
@@ -228,29 +245,162 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
}
}.value
+ private lazy val separatorPrefixMTAApprox =
+ this.optLexicalParent match {
+ case Some(s: SequenceTermBase) if s.hasSeparator =>
+ import SeparatorPosition.*
+ s.separatorPosition match {
+ case Prefix | Infix => AlignmentMultipleOf(s.knownEncodingAlignmentInBits)
+ case Postfix => AlignmentMultipleOf(1)
+ }
+ case _ => AlignmentMultipleOf(1)
+ }
+
+ private lazy val separatorPostfixMTAApprox =
+ this.optLexicalParent match {
+ case Some(s: SequenceTermBase) if s.hasSeparator =>
+ import SeparatorPosition.*
+ s.separatorPosition match {
+ case Prefix | Infix => AlignmentMultipleOf(1)
+ case Postfix => AlignmentMultipleOf(s.knownEncodingAlignmentInBits)
+ }
+ case _ => AlignmentMultipleOf(1)
+ }
+
+ private lazy val separatorLengthApprox = this.optLexicalParent match {
+ case Some(s: SequenceTermBase) if s.hasSeparator =>
+ getEncodingLengthApprox(s)
+ case _ => LengthExact(0)
+ }
+
+ private lazy val initiatorMTAApprox =
+ if (this.hasInitiator) {
+ AlignmentMultipleOf(this.knownEncodingAlignmentInBits)
+ } else {
+ AlignmentMultipleOf(1)
+ }
+
+ private lazy val initiatorLengthApprox = if (this.hasInitiator) {
+ getEncodingLengthApprox(this)
+ } else {
+ LengthExact(0)
+ }
+
+ private lazy val terminatorMTAApprox =
+ if (this.hasTerminator) {
+ AlignmentMultipleOf(this.knownEncodingAlignmentInBits)
+ } else {
+ AlignmentMultipleOf(1)
+ }
+
+ private lazy val terminatorLengthApprox = if (this.hasTerminator) {
+ getEncodingLengthApprox(this)
+ } else {
+ LengthExact(0)
+ }
+
+ private def getEncodingLengthApprox(t: Term) = {
+ if (t.isKnownEncoding) {
+ val dfdlCharset = t.charsetEv.optConstant.get
+ val fixedWidth =
+ if (dfdlCharset.maybeFixedWidth.isDefined) dfdlCharset.maybeFixedWidth.get else 8
+ LengthMultipleOf(fixedWidth)
+ } else {
+ // runtime encoding, which must be a byte-length encoding
+ LengthMultipleOf(8)
+ }
+ }
+
+ /**
+ * prior alignment with leading skip includes the prefix/infix separator MTA and length,
+ * any leading skips
+ */
private lazy val priorAlignmentWithLeadingSkipApprox: AlignmentMultipleOf = {
- priorAlignmentApprox + leadingSkipApprox
+ val priorAlignmentWithSeparatorApprox = priorAlignmentApprox
+ + separatorPrefixMTAApprox
+ + separatorLengthApprox
+ val leadingAlignmentApprox = (priorAlignmentWithSeparatorApprox
+ + leadingSkipApprox)
+ leadingAlignmentApprox
}
- protected lazy val contentStartAlignment: AlignmentMultipleOf = {
- if (priorAlignmentWithLeadingSkipApprox % alignmentApprox == 0) {
- // alignment won't be needed, continue using prior alignment as start alignment
- priorAlignmentWithLeadingSkipApprox
+ /**
+ * The length of the prefix length quasi element.
+ *
+ * This is only relevant for quasi elements, or prefixed terms.
+ */
+ protected lazy val prefixLengthElementLength: LengthApprox = {
+ this match {
+ case eb: ElementBase => {
+ if (eb.lengthKind == Prefixed) {
+ val prefixTypeElem = eb.prefixedLengthElementDecl
+ getElementLength(prefixTypeElem)
+ } else {
+ LengthExact(0)
+ }
+ }
+ case _ => LengthExact(0)
+ }
+ }
+
+ private def getElementLength(elem: ElementBase) = {
+ val currentLengthKind = elem.lengthKind
+ if (currentLengthKind == Explicit || currentLengthKind == Implicit) {
+ LengthExact(
+ elem.elementLengthInBitsEv.constValue.get
+ )
+ } else if (currentLengthKind == Prefixed) {
+ elem.prefixLengthElementLength
} else {
- // alignment will be needed, it will be forced to be aligned to alignmentApprox
- alignmentApprox
+ LengthExact(0)
}
}
- protected lazy val endingAlignmentApprox: AlignmentMultipleOf = {
+ /**
+ * The alignment of the term's value.
+ *
+ * This is only relevant for simple elements.
+ */
+ private lazy val valueMTAApprox = {
+ this match {
+ case eb: ElementBase if eb.isSimpleType && eb.representation == Representation.Text => {
+ AlignmentMultipleOf(eb.knownEncodingAlignmentInBits)
+ }
+ case _ => AlignmentMultipleOf(1)
+ }
+ }
+
+ /**
+ * This alignment is made up of the alignment of the term itself,
+ * the initiator MTA and length, the prefix length quasi
+ * element length, and the value MTA (we add it for optimization
+ * but is extremely difficult to test, as other things such
+ * as unparsers will provide MTA, even elementSpecifiedLengthApprox
+ * considers the encoding also).
+ */
+ protected lazy val contentStartAlignment: AlignmentMultipleOf = {
+ val leftFramingApprox = priorAlignmentWithLeadingSkipApprox
+ + alignmentApprox
+ + initiatorMTAApprox
+ + initiatorLengthApprox
+ val csa = (leftFramingApprox + prefixLengthElementLength) + valueMTAApprox
+ csa
+ }
+
+ /**
+ * Accounts for the start of the content start alignment and element length.
+ * This is used to determine the alignment before the terminator and trailing skip
+ */
+ protected lazy val contentEndAlignment: AlignmentMultipleOf = {
this match {
case eb: ElementBase => {
- if (eb.isComplexType && eb.lengthKind == LengthKind.Implicit) {
- eb.complexType.group.endingAlignmentApprox + trailingSkipApprox
+ val res = if (eb.isComplexType && eb.lengthKind == LengthKind.Implicit) {
+ eb.complexType.group.contentEndAlignment
} else {
// simple type or complex type with specified length
- contentStartAlignment + (elementSpecifiedLengthApprox + trailingSkipApprox)
+ contentStartAlignment + elementSpecifiedLengthApprox
}
+ res
}
case mg: ModelGroup => {
//
@@ -260,37 +410,59 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
//
val (lastChildren, couldBeLast) = mg.potentialLastChildren
val lastApproxesConsideringChildren: Seq[AlignmentMultipleOf] = lastChildren.map { lc =>
- //
- // for each possible last child, add its ending alignment
- // to our trailing skip to get where it would leave off were
- // it the actual last child.
- //
+ // for each possible last child, gather its endingAlignmentApprox
val lceaa = lc.endingAlignmentApprox
- val res = lceaa + trailingSkipApprox
- res
+ lceaa
}
val optApproxIfNoChildren =
//
// gather possibilities for this item itself
//
- if (couldBeLast)
+ if (couldBeLast) {
//
// if this model group could be last, then consider
// if none of its content was present.
- // We'd just have the contentStart, nothing, and the trailing Skip.
+ // We'd just have the contentStart
//
- Some(contentStartAlignment + trailingSkipApprox)
- else
+ val res = Some(contentStartAlignment)
+ res
+ } else
// can't be last, no possibilities to gather.
None
val lastApproxes = lastApproxesConsideringChildren ++ optApproxIfNoChildren
+ // take all the gathered possibilities to get where it would leave off were
+ // each the actual last child.
Assert.invariant(lastApproxes.nonEmpty)
- val res = lastApproxes.reduce { _ * _ }
+ val res = lastApproxes.reduce {
+ _ * _
+ }
res
}
}
}
+ /**
+ * Add the ending alignment of the term
+ * to terminator MTA/length and our trailing skip
+ */
+ protected lazy val endingAlignmentApprox: AlignmentMultipleOf = {
+ val res = this.contentEndAlignment
+ + terminatorMTAApprox
+ + terminatorLengthApprox
+ + trailingSkipApprox
+ res
+ }
+
+ /**
+ * The postfix separator MTA/length needs to be added after the trailing skip
+ */
+ protected lazy val endingAlignmentWithRightFramingApprox: AlignmentMultipleOf = {
+ val res = endingAlignmentApprox
+ + separatorPostfixMTAApprox
+ + separatorLengthApprox
+ res
+ }
+
protected lazy val elementSpecifiedLengthApprox: LengthApprox = {
this match {
case eb: ElementBase => {
@@ -321,7 +493,22 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
case LengthKind.Delimited => encodingLengthApprox
case LengthKind.Pattern => encodingLengthApprox
case LengthKind.EndOfParent => LengthMultipleOf(1) // NYI
- case LengthKind.Prefixed => LengthMultipleOf(1) // NYI
+ case LengthKind.Prefixed => {
+ val prefixElem = eb.prefixedLengthElementDecl
+ if (prefixElem.lengthKind == Explicit || prefixElem.lengthKind == Implicit) {
+ LengthExact(
+ prefixElem.elementLengthInBitsEv.optConstant.get.get
+ )
+ } else if (prefixElem.representation == Representation.Text) {
+ getEncodingLengthApprox(prefixElem)
+ } else {
+ eb.lengthUnits match {
+ case LengthUnits.Bits => LengthMultipleOf(1)
+ case LengthUnits.Bytes => LengthMultipleOf(8)
+ case LengthUnits.Characters => encodingLengthApprox
+ }
+ }
+ }
}
}
case _: ModelGroup => Assert.usageError("Only for elements")
@@ -329,15 +516,7 @@ trait AlignedMixin extends GrammarMixin { self: Term =>
}
private lazy val encodingLengthApprox: LengthApprox = {
- if (isKnownEncoding) {
- val dfdlCharset = charsetEv.optConstant.get
- val fixedWidth =
- if (dfdlCharset.maybeFixedWidth.isDefined) dfdlCharset.maybeFixedWidth.get else 8
- LengthMultipleOf(fixedWidth)
- } else {
- // runtime encoding, which must be a byte-length encoding
- LengthMultipleOf(8)
- }
+ getEncodingLengthApprox(this)
}
protected lazy val isKnownToBeByteAlignedAndByteLength: Boolean = {
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml
index 119c799854..3ddb07887c 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml
@@ -603,6 +603,419 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ IaaBbb
+
+
+
+
+
+ aa
+ bb
+
+
+
+
+
+
+
+ IAaabbee
+
+
+
+
+
+ aa
+ bb
+
+ ee
+
+
+
+
+
+
+ aaTBbb
+
+
+
+
+
+ aa
+ bb
+
+
+
+
+
+
+
+ aabbT2ee
+
+
+
+
+
+ aa
+ bb
+
+ ee
+
+
+
+
+
+
+
+
+
+
+ Schema Definition Error
+ 1 bits
+ must be a multiple
+ 8 bits
+ US-ASCII
+
+
+
+
+
+
+ cc
+ aa
+ bb
+
+
+
+
+
+
+
+ cc2aaBbb
+
+
+
+
+
+ cc
+ aa
+ bb
+
+
+
+
+
+
+
+ cc16aabb
+
+
+
+
+
+ cc
+ aa
+ bb
+
+
+
+
+
+
+
+ cc16aabb
+
+
+
+
+
+ cc
+ aa
+ bb
+
+
+
+
+
+
+ Nested
+ prefixed
+ not supported
+
+
+
+
+
+ 1110
+ 1111
+ FFFFFFFF
+ 0111
+ 0000
+
+
+
+
+
+
+ 1110
+
+ 0111
+
+
+
+
+
+
+
+ eEaBb
+
+
+
+
+ e
+
+ a
+ b
+
+
+
+
+
+
+
+
+ 0011
+ 1111
+ 0000
+ 0100
+
+
+
+
+
+
+ 3
+ 4
+
+
+
+
+
+
+
+
+ 0011
+ 0000
+ -
+ 00000101
+ 00000100
+
+
+
+
+
+
+ 3
+ 5
+ 4
+
+
+
+
+
+
+
+ eE-ob
+
+
+
+
+ e
+
+ o
+ b
+
+
+
+
+
+
+
+ eEb
+
+
+
+
+ e
+
+ b
+
+
+
+
+
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml
index 0ab4ea916e..61d08de6b9 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml
@@ -143,14 +143,6 @@
separatorPosition="infix"
emptyElementParsePolicy="treatAsEmpty"/>
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ aa/Bbb
+
+
+
+
+ aa
+ bb
+
+
+
+
+
+
+
+ /Aaa/Bbb
+
+
+
+
+ aa
+ bb
+
+
+
+
+
+
+
+ aa/Bbb/
+
+
+
+
+
+ aa
+ bb
+
+
+
+
+
+
+
+ cc-Aaa/Bbb/-dd-
+
+
+
+
+
+ cc
+ aa
+ bb
+ dd
+
+
+
+
+
+
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala
index 23967c96a8..e22069d77f 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala
@@ -186,6 +186,28 @@ class TestAlignedData extends TdmlTests {
@Test def alignmentFillByteDefined = test
@Test def separatorMTA_01 = test
+
+ // DAFFODIL-3056
+ @Test def test_init_alignment_1 = test
+ @Test def test_init_alignment_2 = test
+ @Test def test_prefix_alignment_1 = test
+ @Test def test_prefix_alignment_2 = test
+ @Test def test_prefix_alignment_3 = test
+ @Test def test_valueMTA_alignment_1 = test
+
+ // DAFFODIL-3057
+ @Test def test_term_alignment_1 = test
+ @Test def test_term_alignment_2 = test
+
+ // DAFFODIL-3059
+ @Test def test_prefix_alignment_0 = test
+
+ // DAFFODIL-3060
+ @Test def prior_siblings_1 = test
+ @Test def prior_siblings_2 = test
+ @Test def prior_siblings_3 = test
+ @Test def prior_siblings_4 = test
+ @Test def prior_siblings_5 = test
}
class TestBinaryInput extends TdmlTests {
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala
index 3ffd036121..9c27ba7261 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala
@@ -55,4 +55,11 @@ class TestSepTests extends TdmlTests {
// DAFFODIL-2791
@Test def test_treatAsAbsent_occursIndex = test
+
+ // DAFFODIL-2295
+ @Test def test_sep_alignment_1 = test
+ @Test def test_sep_alignment_2 = test
+ @Test def test_sep_alignment_3 = test
+
+ @Test def test_sep_alignment_4 = test
}