m3.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

import xml.dom.minidom
from xml.dom.minidom import Node
import re
from sys import stderr
import struct
import copy
import sys


def increaseToValidSectionSize(size):
    blockSize = 16
    incompleteBlockBytes = (size % blockSize)
    if incompleteBlockBytes != 0:
        missingBytesToCompleteBlock = blockSize - incompleteBlockBytes
        return size + missingBytesToCompleteBlock
    else:
        return size


class Section:
    """Has fields indexEntry and structureDescription and sometimes also the fields rawBytes and content """

    def __init__(self):
        self.timesReferenced = 0

    def __str__(self):
        return 'Section %s timesReferenced=%d %sV%s' % (
            self.indexEntry,
            self.timesReferenced,
            self.structureDescription.structureName,
            self.structureDescription.structureVersion
        )

    def from_instances(self, content, tag=''):
        if tag:
            self.content = content
            self.structureDescription = structures[tag].structureDescription
        else:
            self.content = content
            self.structureDescription = content[0].structureDescription

    def determineContentField(self, checkExpectedValue):
        self.content = self.structureDescription.createInstances(buffer=self.rawBytes, count=self.indexEntry.repetitions, checkExpectedValue=checkExpectedValue)

    def determineFieldRawBytes(self):
        minRawBytes = self.determineRawBytesWithData()
        if len(minRawBytes) != self.bytesRequiredForContent():
            raise Exception("Section size calculation failed: Expected %s but was %s for %s; content: %s" % (self.bytesRequiredForContent(), len(minRawBytes), self.structureDescription.structureName, minRawBytes))
        sectionSize = increaseToValidSectionSize(len(minRawBytes))
        if len(minRawBytes) == sectionSize:
            self.rawBytes = minRawBytes
        else:
            rawBytes = bytearray(sectionSize)
            rawBytes[0:len(minRawBytes)] = minRawBytes
            for i in range(len(minRawBytes), sectionSize):
                rawBytes[i] = 0xaa
            self.rawBytes = rawBytes

    def determineRawBytesWithData(self):
        return self.structureDescription.instancesToBytes(self.content)

    def bytesRequiredForContent(self):
        return self.structureDescription.countBytesRequiredForInstances(self.content)

    def resolveReferences(self, sections):
        if not self.structureDescription.isPrimitive:
            for object in self.content:
                object.resolveReferences(sections)


primitiveFieldTypeSizes = {"uint32": 4, "int32": 4, "uint16": 2, "int16": 2, "uint8": 1, "int8": 1, "float": 4, "tag": 4, "fixed8": 1}
primitiveFieldTypeFormats = {"uint32": "I", "int32": "i", "uint16": "H", "int16": "h", "uint8": "B", "int8": "b", "float": "f", "tag": "4s", "fixed8": "B"}
intTypes = {"uint32", "int32", "uint16", "int16", "uint8", "int8"}

structureNamesOfPrimitiveTypes = set(["CHAR", "U8__", "REAL", "I16_", "U16_", "I32_", "U32_", "FLAG"])


class M3StructureHistory:
    "Describes the history of a structure with a specific name"

    def __init__(self, name, versionToSizeMap, allFields):
        self.name = name
        self.versionToSizeMap = versionToSizeMap
        self.allFields = allFields
        self.versionToStructureDescriptionMap = {}
        self.isPrimitive = self.name in structureNamesOfPrimitiveTypes
        # Create all to check sizes:
        for version in versionToSizeMap:
            self.getVersion(version)

    def createStructureDescription(self, version, usedFields, specifiedSize, fmagic):
        finalFields = []
        # dirty patching of all structures to make MD33 models work
        # - they use `SmallReference` instead of `Reference` across all structures
        if fmagic == 'MD33':
            for field in usedFields:
                if isinstance(field, ReferenceField):
                    field = copy.copy(field)
                    if field.referenceStructureDescription.structureName == 'Reference':
                        newStructureHistory = structures['SmallReference']
                        newDescription = newStructureHistory.getVersion(0, fmagic)
                        field.referenceStructureDescription = newDescription
                        field.size = newDescription.size
                elif self.name == 'MODL' and field.name == 'tightHitTest':
                    field = copy.copy(field)
                    newStructureHistory = structures[field.structureDescription.structureName]
                    newDescription = newStructureHistory.getVersion(1, fmagic)
                    field.structureDescription = newDescription
                    field.size = newDescription.size
                finalFields.append(field)
        else:
            finalFields = usedFields
        structure = M3StructureDescription(self.name, version, finalFields, specifiedSize, self, fmagic != 'MD33')
        return structure

    def getVersion(self, version, fmagic='MD34', force=False):
        structure = self.versionToStructureDescriptionMap.get(fmagic + '_' + str(version))
        if structure is None:
            usedFields = []
            for field in self.allFields:
                includeField = True
                if field.sinceVersion is not None and version < field.sinceVersion:
                    includeField = False
                if field.tillVersion is not None and version > field.tillVersion:
                    includeField = False
                if includeField:
                    usedFields.append(field)
            specifiedSize = self.versionToSizeMap.get(version)
            if not force and specifiedSize is None:
                return None
            structure = self.createStructureDescription(version, usedFields, specifiedSize, fmagic)
            self.versionToStructureDescriptionMap[fmagic + '_' + str(version)] = structure
        return structure

    def getNewestVersion(self):
        newestVersion = None
        for version in self.versionToSizeMap.keys():
            if newestVersion is None or version > newestVersion:
                newestVersion = version
        return self.getVersion(newestVersion)

    def createEmptyArray(self):
        if self.name == "CHAR":
            return None  # even no terminating character
        elif self.name == "U8__":
            return bytearray(0)
        else:
            return []

        def __str__():
            return self.name


class M3StructureDescription:

    def __init__(self, structureName, structureVersion, fields, specifiedSize, history, validateSize=True):
        self.structureName = structureName
        self.structureVersion = structureVersion
        self.fields = fields
        self.isPrimitive = self.structureName in structureNamesOfPrimitiveTypes
        self.history = history

        calculatedSize = 0
        for field in fields:
            calculatedSize += field.size
        self.size = calculatedSize

        # Validate the specified size:
        if validateSize and calculatedSize != specifiedSize:
            self.dumpOffsets()
            raise Exception("Size mismatch: %s in version %d has been specified to have size %d, but the calculated size was %d" % (structureName, structureVersion, specifiedSize, calculatedSize))

        nameToFieldMap = {}
        for field in fields:
            if field.name in nameToFieldMap:
                raise Exception("%s contains in version %s multiple fields with the name %s" % (structureName, structureVersion, field.name))
            nameToFieldMap[field.name] = field
        self.nameToFieldMap = nameToFieldMap

    def createInstance(self, buffer=None, offset=0, checkExpectedValue=True):
        return M3Structure(self, buffer, offset, checkExpectedValue)

    def createInstances(self, buffer, count, checkExpectedValue=True):
        if self.isPrimitive:
            if self.structureName == "CHAR":
                return buffer[:count - 1].decode("ASCII", "replace")
            elif self.structureName == "U8__":
                return bytearray(buffer[:count])
            else:
                structFormat = self.fields[0].structFormat
                list = []
                for offset in range(0, count * self.size, self.size):
                    bytesOfOneEntry = buffer[offset:(offset + self.size)]
                    intValue = structFormat.unpack(bytesOfOneEntry)[0]
                    list.append(intValue)
                return list
        else:
            list = []
            instanceOffset = 0
            for i in range(count):
                list.append(self.createInstance(buffer=buffer, offset=instanceOffset, checkExpectedValue=checkExpectedValue))
                instanceOffset += self.size
            return list

    def dumpOffsets(self):
        offset = 0
        stderr.write("Offsets of %s in version %d:\n" % (self.structureName, self.structureVersion))
        for field in self.fields:
            stderr.write("%s: %s\n" % (offset, field.name))
            offset += field.size

    def countInstances(self, instances):
        if self.structureName == "CHAR":
            if instances is None:
                return 0
            return len(instances) + 1  # +1 terminating null character
        elif hasattr(instances, "__len__"):  # either a list or an array of bytes
            return len(instances)
        else:
            raise Exception("Can't measure the length of %s which is a %s" % (instances, self.structureName))

    def validateInstance(self, instance, instanceName):
        for field in self.fields:
            try:
                fieldContent = getattr(instance, field.name)
            except AttributeError:
                raise Exception("%s does not have a field called %s" % (instanceName, field.name))
                raise
            field.validateContent(fieldContent, instanceName + "." + field.name)

    def hasField(self, fieldName):
        return fieldName in self.nameToFieldMap

    def instancesToBytes(self, instances):
        if self.structureName == "CHAR":
            if type(instances) != str:
                raise Exception("Expected a string but it was a %s" % type(instances))
            return instances.encode("ASCII") + b'\x00'
        elif self.structureName == "U8__":
            if type(instances) != bytes and type(instances) != bytearray:
                raise Exception("Expected a byte array but it was a %s" % type(instances))
            return instances
        else:
            rawBytes = bytearray(self.size * len(instances))
            offset = 0

            if self.isPrimitive:
                structFormat = self.fields[0].structFormat
                for value in instances:
                    structFormat.pack_into(rawBytes, offset, value)
                    offset += self.size
            else:
                for value in instances:
                    value.writeToBuffer(rawBytes, offset)
                    offset += self.size
            return rawBytes

    def countBytesRequiredForInstances(self, instances):
        if self.structureName == "CHAR":
            return len(instances) + 1  # +1 for terminating character
        return self.size * self.countInstances(instances)


class M3Structure:

    def __init__(self, structureDescription: M3StructureDescription, buffer=None, offset=0, checkExpectedValue=True):
        self.structureDescription = structureDescription

        if buffer is not None:
            self.readFromBuffer(buffer, offset, checkExpectedValue)
        else:
            for field in self.structureDescription.fields:
                field.setToDefault(self)

    def introduceIndexReferences(self, indexMaker):
        for field in self.structureDescription.fields:
            field.introduceIndexReferences(self, indexMaker)

    def resolveReferences(self, sections):
        for field in self.structureDescription.fields:
            field.resolveIndexReferences(self, sections)

    def readFromBuffer(self, buffer, offset, checkExpectedValue):
        fieldOffset = offset
        for field in self.structureDescription.fields:
            try:
                field.readFromBuffer(self, buffer, fieldOffset, checkExpectedValue)
            except struct.error as e:
                raise Exception('failed to unpack %sV%s %s' % (self.structureDescription.structureName, self.structureDescription.structureVersion, field.name), e)
            fieldOffset += field.size
        assert fieldOffset - offset == self.structureDescription.size

    def writeToBuffer(self, buffer, offset):
        fieldOffset = offset
        for field in self.structureDescription.fields:
            field.writeToBuffer(self, buffer, fieldOffset)
            fieldOffset += field.size
        assert fieldOffset - offset == self.structureDescription.size

    def __str__(self):
        fieldValueMap = {}
        for field in self.structureDescription.fields:
            fieldValueMap[field.name] = str(getattr(self, field.name))
        return "%sV%s: {%s}" % (self.structureDescription.structureName, self.structureDescription.structureVersion, fieldValueMap)

    def getNamedBit(self, fieldName, bitName):
        field = self.structureDescription.nameToFieldMap[fieldName]
        return field.getNamedBit(self, bitName)

    def setNamedBit(self, fieldName, bitName, value):
        field = self.structureDescription.nameToFieldMap[fieldName]
        return field.setNamedBit(self, bitName, value)

    def getBitNameMaskPairs(self, fieldName):
        field = self.structureDescription.nameToFieldMap[fieldName]
        return field.getBitNameMaskPairs()


class Field:
    def __init__(self, name, sinceVersion, tillVersion):
        self.name = name
        self.sinceVersion = sinceVersion
        self.tillVersion = tillVersion

    def introduceIndexReferences(self, owner, indexMaker):
        pass

    def resolveIndexReferences(self, owner, sections):
        pass


class TagField(Field):

    def __init__(self, name, sinceVersion, tillVersion):
        Field.__init__(self, name, sinceVersion, tillVersion)
        self.structFormat = struct.Struct("<4B")
        self.size = 4

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):
        b = self.structFormat.unpack_from(buffer, offset)
        if b[3] == 0:
            s = chr(b[2]) + chr(b[1]) + chr(b[0])
        else:
            s = chr(b[3]) + chr(b[2]) + chr(b[1]) + chr(b[0])

        setattr(owner, self.name, s)

    def writeToBuffer(self, owner, buffer, offset):
        s = getattr(owner, self.name)
        if len(s) == 4:
            b = (s[3] + s[2] + s[1] + s[0]).encode("ascii")
        else:
            b = (s[2] + s[1] + s[0]).encode("ascii") + b"\x00"
        return self.structFormat.pack_into(buffer, offset, b[0], b[1], b[2], b[3])

    def setToDefault(self, owner):
        pass

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != str) or (len(fieldContent) != 4):
            raise Exception("%s is not a string with 4 characters" % (fieldPath))


class ReferenceField(Field):
    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        Field.__init__(self, name, sinceVersion, tillVersion)
        self.referenceStructureDescription = referenceStructureDescription
        self.historyOfReferencedStructures = historyOfReferencedStructures
        self.size = referenceStructureDescription.size

    def introduceIndexReferences(self, owner, indexMaker):
        referencedObjects = getattr(owner, self.name)
        structureDescription = self.getListContentStructureDefinition(referencedObjects, "while adding index ref")

        indexReference = indexMaker.getIndexReferenceTo(referencedObjects, self.referenceStructureDescription, structureDescription)
        isPrimitive = self.historyOfReferencedStructures is not None and self.historyOfReferencedStructures.isPrimitive
        if not isPrimitive:
            for referencedObject in referencedObjects:
                referencedObject.introduceIndexReferences(indexMaker)
        setattr(owner, self.name, indexReference)

    def resolveIndexReferences(self, owner, sections):
        ref = getattr(owner, self.name)
        ownerName = owner.structureDescription.structureName
        variable = "%(ownerName)s.%(fieldName)s" % {"ownerName": ownerName, "fieldName": self.name}
        if ref.entries == 0:
            if self.historyOfReferencedStructures is None:
                referencedObjects = []
            else:
                referencedObjects = self.historyOfReferencedStructures.createEmptyArray()
        else:
            referencedSection = sections[ref.index]
            referencedSection.timesReferenced += 1
            indexEntry = referencedSection.indexEntry

            if indexEntry.repetitions < ref.entries:
                raise Exception("%s tries to reference %s elements in a %s section that contains just %s element(s)" % (variable, ref.entries, indexEntry.tag, indexEntry.repetitions))

            referencedObjects = referencedSection.content
            if self.historyOfReferencedStructures is not None:
                expectedTagName = self.historyOfReferencedStructures.name
                actualTagName = indexEntry.tag
                if actualTagName != expectedTagName:
                    raise Exception("Expected ref %s point to %s, but it points to %s" % (variable, expectedTagName, actualTagName))
            else:
                raise Exception("Field %s can be marked as a reference pointing to %s" % (variable, indexEntry.tag))

        setattr(owner, self.name, referencedObjects)

    def getListContentStructureDefinition(self, li, contextString):

        if self.historyOfReferencedStructures is None:
            if len(li) == 0:
                return None
            else:
                variable = "%(fieldName)s" % {"fieldName": self.name}
                raise Exception("%s: %s must be an empty list but wasn't" % (contextString, variable))
        if self.historyOfReferencedStructures.isPrimitive:
            return self.historyOfReferencedStructures.getVersion(0)

        if type(li) != list:
            raise Exception("%s: Expected a list, but was a %s" % (contextString, type(li)))
        if len(li) == 0:
            return None

        firstElement = li[0]
        contentClass = type(firstElement)
        if contentClass != M3Structure:
            raise Exception("%s: Expected a list to contain an M3Structure object and not a %s" % (contextString, contentClass))
        # Optional: Enable check:
        # if not contentClass.tagName == tagName:
        #     raise Exception("Expected a list to contain a object of a class with tagName %s, but it contained a object of class %s with tagName %s" % (tagName, contentClass, contentClass.tagName))
        return firstElement.structureDescription

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):
        referenceObject = self.referenceStructureDescription.createInstance(buffer, offset, checkExpectedValue)
        setattr(owner, self.name, referenceObject)

    def writeToBuffer(self, owner, buffer, offset):
        referenceObject = getattr(owner, self.name)
        referenceObject.writeToBuffer(buffer, offset)

    def setToDefault(self, owner):

        if self.historyOfReferencedStructures is not None:
            defaultValue = self.historyOfReferencedStructures.createEmptyArray()
        else:
            defaultValue = []
        setattr(owner, self.name, defaultValue)

    # The method validateContent is defined in subclasses


class CharReferenceField(ReferenceField):

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)

    def validateContent(self, fieldContent, fieldPath):
        if (fieldContent is not None) and (type(fieldContent) != str):
            raise Exception("%s is not a string but a %s" % (fieldPath, type(fieldContent)))


class ByteReferenceField(ReferenceField):

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != bytearray):
            raise Exception("%s is not a bytearray but a %s" % (fieldPath, type(fieldContent)))


class RealReferenceField(ReferenceField):

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != list):
            raise Exception("%s is not a list of float" % (fieldPath))
        for itemIndex, item in enumerate(fieldContent):
            if type(item) != float:
                itemPath = "%s[%d]" % (fieldPath, itemIndex)
                raise Exception("%s is not an float" % (itemPath))


class IntReferenceField(ReferenceField):
    intRefToMinValue = {"I16_": (-(1 << 15)), "U16_": 0, "I32_": (-(1 << 31)), "U32_": 0, "FLAG": 0}
    intRefToMaxValue = {"I16_": ((1 << 15) - 1), "U16_": ((1 << 16) - 1), "I32_": ((1 << 31) - 1), "U32_": ((1 << 32) - 1), "FLAG": ((1 << 32) - 1)}

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
        self.minValue = IntReferenceField.intRefToMinValue[historyOfReferencedStructures.name]
        self.maxValue = IntReferenceField.intRefToMaxValue[historyOfReferencedStructures.name]

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != list):
            raise Exception("%s is not a list of integers" % (fieldPath))
        for itemIndex, item in enumerate(fieldContent):
            itemPath = "%s[%d]" % (fieldPath, itemIndex)
            if type(item) != int:
                raise Exception("%s is not an integer" % (itemPath))
            if (item < self.minValue) or (item > self.maxValue):
                raise Exception("%s has value %d which is not in range [%s, %s]" % (itemPath, item, self.minValue, self.maxValue))


class StructureReferenceField(ReferenceField):

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != list):
            raise Exception("%s is not a list, but a %s" % (fieldPath, type(fieldContent)))
        if len(fieldContent) > 0:
            structureDescription = self.getListContentStructureDefinition(fieldContent, fieldPath)
            if structureDescription.history != self.historyOfReferencedStructures:
                raise Exception("Expected that %s is a list of %s and not %s" % (fieldPath, self.historyOfReferencedStructures.name, structureDescription.history.name))
            for itemIndex, item in enumerate(fieldContent):
                structureDescription.validateInstance(item, "%s[%d]" % (fieldPath, itemIndex))


class UnknownReferenceField(ReferenceField):

    def __init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion):
        ReferenceField.__init__(self, name, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != list) or (len(fieldContent) != 0):
            raise Exception("%s is not an empty list" % (fieldPath))


class EmbeddedStructureField(Field):

    def __init__(self, name, structureDescription, sinceVersion, tillVersion):
        Field.__init__(self, name, sinceVersion, tillVersion)
        self.structureDescription = structureDescription
        self.size = structureDescription.size

    def introduceIndexReferences(self, owner, indexMaker):
        emeddedStructure = getattr(owner, self.name)
        emeddedStructure.introduceIndexReferences(indexMaker)

    def resolveIndexReferences(self, owner, sections):
        emeddedStructure = getattr(owner, self.name)
        emeddedStructure.resolveReferences(sections)

    def toBytes(self, owner):
        emeddedStructure = getattr(owner, self.name)
        return emeddedStructure.toBytes()

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):

        referenceObject = self.structureDescription.createInstance(buffer, offset, checkExpectedValue)
        setattr(owner, self.name, referenceObject)

    def writeToBuffer(self, owner, buffer, offset):
        emeddedStructure = getattr(owner, self.name)
        emeddedStructure.writeToBuffer(buffer, offset)

    def setToDefault(self, owner):
        v = self.structureDescription.createInstance()
        setattr(owner, self.name, v)

    def validateContent(self, fieldContent, fieldPath):
        self.structureDescription.validateInstance(fieldContent, fieldPath)


class PrimitiveField(Field):
    """ Base class for IntField and FloatField """

    def __init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue):
        Field.__init__(self, name, sinceVersion, tillVersion)
        self.size = primitiveFieldTypeSizes[typeString]
        self.structFormat = struct.Struct("<" + primitiveFieldTypeFormats[typeString])
        self.typeString = typeString
        self.defaultValue = defaultValue
        self.expectedValue = expectedValue

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):
        value = self.structFormat.unpack_from(buffer, offset)[0]
        if self.expectedValue is not None and value != self.expectedValue:
            structureName = owner.structureDescription.structureName
            structureVersion = owner.structureDescription.structureVersion
            raise Exception("Expected that field %s of %s (V. %d) has always the value %s, but it was %s" % (self.name, structureName, structureVersion, self.expectedValue, value))
        setattr(owner, self.name, value)

    def writeToBuffer(self, owner, buffer, offset):
        value = getattr(owner, self.name)
        return self.structFormat.pack_into(buffer, offset, value)

    def setToDefault(self, owner):
        setattr(owner, self.name, self.defaultValue)


class IntField(PrimitiveField):
    intTypeToMinValue = {"int16": (-(1 << 15)), "uint16": 0, "int32": (-(1 << 31)), "uint32": 0, "int8": -(1 << 7), "uint8": 0}
    intTypeToMaxValue = {"int16": ((1 << 15) - 1), "uint16": ((1 << 16) - 1), "int32": ((1 << 31) - 1), "uint32": ((1 << 32) - 1), "int8": ((1 << 7) - 1), "uint8": ((1 << 8) - 1)}

    def __init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue, bitMaskMap):
        PrimitiveField.__init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue)
        self.minValue = IntField.intTypeToMinValue[typeString]
        self.maxValue = IntField.intTypeToMaxValue[typeString]
        self.bitMaskMap = bitMaskMap

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != int):
            raise Exception("%s is not an int but a %s!" % (fieldPath, type(fieldContent)))
        if (fieldContent < self.minValue) or (fieldContent > self.maxValue):
            raise Exception("%s has value %d which is not in range [%d, %d]" % (fieldPath, fieldContent, self.minValue, self.maxValue))

    def getNamedBit(self, owner, bitName):
        mask = self.bitMaskMap[bitName]
        intValue = getattr(owner, self.name)
        return ((intValue & mask) != 0)

    def setNamedBit(self, owner, bitName, value):
        mask = self.bitMaskMap[bitName]
        intValue = getattr(owner, self.name)
        if value:
            setattr(owner, self.name, intValue | mask)
        else:
            if (intValue & mask) != 0:
                setattr(owner, self.name, intValue ^ mask)

    def getBitNameMaskPairs(self):
        return self.bitMaskMap.items()


class FloatField(PrimitiveField):

    def __init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue):
        PrimitiveField.__init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != float):
            raise Exception("%s is not a float but a %s!" % (fieldPath, type(fieldContent)))


class Fixed8Field(PrimitiveField):

    def __init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue):
        PrimitiveField.__init__(self, name, typeString, sinceVersion, tillVersion, defaultValue, expectedValue)

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):
        intValue = self.structFormat.unpack_from(buffer, offset)[0]
        floatValue = ((intValue / 255.0 * 2.0) - 1)

        if checkExpectedValue and self.expectedValue is not None and floatValue != self.expectedValue:
            structureName = owner.structureDescription.structureName
            structureVersion = owner.structureDescription.structureVersion
            raise Exception("Expected that field %s of %s (V. %d) has always the value %s, but it was %s" % (self.name, structureName, structureVersion, self.expectedValue, intValue))
        setattr(owner, self.name, floatValue)

    def writeToBuffer(self, owner, buffer, offset):
        floatValue = getattr(owner, self.name)
        intValue = round((floatValue + 1) / 2.0 * 255.0)
        return self.structFormat.pack_into(buffer, offset, intValue)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != float):
            raise Exception("%s is not a float but a %s!" % (fieldPath, type(fieldContent)))


class UnknownBytesField(Field):

    def __init__(self, name, size, sinceVersion, tillVersion, defaultValue, expectedValue):
        Field.__init__(self, name, sinceVersion, tillVersion)
        self.size = size
        self.structFormat = struct.Struct("<%ss" % size)
        self.defaultValue = defaultValue
        self.expectedValue = expectedValue
        assert self.structFormat.size == self.size

    def readFromBuffer(self, owner, buffer, offset, checkExpectedValue):
        value = self.structFormat.unpack_from(buffer, offset)[0]
        if checkExpectedValue and self.expectedValue is not None and value != self.expectedValue:
            raise Exception("Expected that %sV%s.%s has always the value %s, but it was %s" % (owner.structureDescription.structureName, owner.structureDescription.structureVersion, self.name, self.expectedValue, value))

        setattr(owner, self.name, value)

    def writeToBuffer(self, owner, buffer, offset):
        value = getattr(owner, self.name)
        return self.structFormat.pack_into(buffer, offset, value)

    def setToDefault(self, owner):
        setattr(owner, self.name, self.defaultValue)

    def validateContent(self, fieldContent, fieldPath):
        if (type(fieldContent) != bytes) or (len(fieldContent) != self.size):
            raise Exception("%s is not an bytes object of size %s" % (fieldPath, self.size))


class Visitor:
    def visitStart(self, generalDataMap):
        pass

    def visitClassStart(self, generalDataMap, classDataMap):
        pass

    def visitVersion(self, generalDataMap, classDataMap, versionDataMap):
        pass

    def visitFieldStart(self, generalDataMap, classDataMap, fieldDataMap):
        pass

    def visitFieldBit(self, generalDataMap, classDataMap, fieldDataMap, bitDataMap):
        pass

    def visitFieldEnd(self, generalDataMap, classDataMap, fieldDataMap):
        pass

    def visitClassEnd(self, generalDataMap, classDataMap):
        pass

    def visitEnd(self, generalDataMap):
        pass


class StructureAttributesReader(Visitor):
    def visitClassStart(self, generalDataMap, classDataMap):
        xmlNode = classDataMap["xmlNode"]
        if xmlNode.hasAttribute("name"):
            classDataMap["structureName"] = xmlNode.getAttribute("name")
        else:
            raise Exception("There is a structure without a name attribute")
        classDataMap["versionToSizeMap"] = {}

    def visitVersion(self, generalDataMap, classDataMap, versionDataMap):
        xmlNode = versionDataMap["xmlNode"]
        if xmlNode.hasAttribute("number"):
            version = int(xmlNode.getAttribute("number"))
        else:
            raise Exception("The structure %s has a version element without a number attribute" % classDataMap["tagName"])
        if xmlNode.hasAttribute("size"):
            sizeString = xmlNode.getAttribute("size")
            try:
                size = int(sizeString)
            except ValueError:
                structureName = classDataMap["structureName"]
                raise Exception("The size specified for version %d of structure %s is not an int" % (version, structureName))
        else:
            raise Exception("The structure %s has a version element without a size attribute" % classDataMap["tagName"])
        versionToSizeMap = classDataMap["versionToSizeMap"]
        if version in versionToSizeMap:
            raise Exception(("The structure %s has two times the same version" % classDataMap["tagName"]))
        versionToSizeMap[version] = size


class StructureDescriptionReader(Visitor):
    def visitClassStart(self, generalDataMap, classDataMap):
        xmlNode = classDataMap["xmlNode"]
        tagDescriptionNodes = xmlNode.getElementsByTagName("description")
        if len(tagDescriptionNodes) != 1:
            fullName = classDataMap["fullName"]
            raise Exception("Tag %s has not exactly one description node", fullName)
        tagDescriptionNode = tagDescriptionNodes[0]
        tagDescription = ""
        for descriptionChild in tagDescriptionNode.childNodes:
            if descriptionChild.nodeType == Node.TEXT_NODE:
                tagDescription += descriptionChild.data
        classDataMap["description"] = tagDescription


class FieldAttributesReader(Visitor):
    def visitFieldStart(self, generalDataMap, classDataMap, fieldDataMap):
        xmlNode = fieldDataMap["xmlNode"]
        if xmlNode.hasAttribute("name"):
            fieldDataMap["fieldName"] = xmlNode.getAttribute("name")
        else:
            fullName = classDataMap["fullName"]
            raise Exception("There is a field in %s without a name attribute" % fullName)

        if xmlNode.hasAttribute("type"):
            fieldDataMap["typeString"] = xmlNode.getAttribute("type")
        else:
            fieldDataMap["typeString"] = None

        if xmlNode.hasAttribute("refTo"):
            fieldDataMap["refTo"] = xmlNode.getAttribute("refTo")
        else:
            fieldDataMap["refTo"] = None

        if xmlNode.hasAttribute("size"):
            fieldDataMap["specifiedFieldSize"] = int(xmlNode.getAttribute("size"))
        else:
            fieldDataMap["specifiedFieldSize"] = None

        if xmlNode.hasAttribute("expected-value"):
            fieldDataMap["expectedValueString"] = xmlNode.getAttribute("expected-value")
        else:
            fieldDataMap["expectedValueString"] = None

        if xmlNode.hasAttribute("default-value"):
            fieldDataMap["defaultValueString"] = xmlNode.getAttribute("default-value")
        else:
            fieldDataMap["defaultValueString"] = None

        if xmlNode.hasAttribute("till-version"):
            fieldDataMap["tillVersion"] = int(xmlNode.getAttribute("till-version"))
        else:
            fieldDataMap["tillVersion"] = None

        if xmlNode.hasAttribute("since-version"):
            fieldDataMap["sinceVersion"] = int(xmlNode.getAttribute("since-version"))
        else:
            fieldDataMap["sinceVersion"] = None


class BitAttributesReader(Visitor):

    def visitFieldBit(self, generalDataMap, classDataMap, fieldDataMap, bitDataMap):
        xmlNode = bitDataMap["xmlNode"]
        if xmlNode.hasAttribute("name"):
            bitDataMap["name"] = xmlNode.getAttribute("name")
        else:
            structureName = classDataMap["structureName"]
            fieldName = fieldDataMap["fieldName"]
            raise Exception("There is bit xml node in field %(fieldName)s of structure %(structureName)s without a name attribute" % {"fieldName": fieldName, "structureName": structureName})

        if xmlNode.hasAttribute("mask"):
            maskString = xmlNode.getAttribute("mask")
            if not re.match("0x[0-9]+", maskString):
                structureName = classDataMap["structureName"]
                fieldName = fieldDataMap["fieldName"]
                bitName = bitDataMap["name"]
                raise Exception("The bit %(bitName)s of %(structureName)s.%(fieldName)s has an invalid mask attribute" % {"fieldName": fieldName, "structureName": structureName, "bitName": bitName})

            bitDataMap["mask"] = int(maskString, 0)
        else:
            structureName = classDataMap["structureName"]
            fieldName = fieldDataMap["fieldName"]
            raise Exception("There is bit xml node in field %(fieldName)s of structure %(structureName)s without a mask attribute" % {"fieldName": fieldName, "structureName": structureName})


class ExpectedAndDefaultConstantsDeterminer(Visitor):
    def parseHex(self, hexString):
        hexString = hexString[2:]
        return bytes([int(hexString[x:x + 2], 16) for x in range(0, len(hexString), 2)])

    def visitFieldStart(self, generalDataMap, classDataMap, fieldDataMap):
        structureName = classDataMap["structureName"]
        fieldName = fieldDataMap["fieldName"]
        fieldType = fieldDataMap["typeString"]
        expectedValueString = fieldDataMap["expectedValueString"]
        defaultValueString = fieldDataMap["defaultValueString"]
        variableName = "%s.%s" % (structureName, fieldName)
        expectedValue = None
        defaultValue = None
        if fieldType in ("int32", "int16", "int8", "uint8", "uint16", "uint32"):
            if expectedValueString is not None:
                try:
                    expectedValue = int(expectedValueString, 0)
                except ValueError:
                    raise Exception("The specified expected value for %s is not an integer" % variableName)

            if defaultValueString is not None:
                try:
                    defaultValue = int(defaultValueString, 0)
                except ValueError:
                    raise Exception("The specified default value for %s is not an integer" % variableName)
            else:
                if expectedValue is not None:
                    defaultValue = expectedValue
                else:
                    defaultValue = 0
        elif fieldType == "float" or fieldType == "fixed8":
            if expectedValueString is not None:
                try:
                    expectedValue = float(expectedValueString)
                except ValueError:
                    raise Exception("The specified expected value for %s is not a float" % variableName)

            if defaultValueString is not None:
                try:
                    defaultValue = float(defaultValueString)
                except ValueError:
                    raise Exception("The specified default value for %s is not a a float" % variableName)
            else:
                if expectedValue is not None:
                    defaultValue = expectedValue
                else:
                    defaultValue = 0.0
        elif fieldType is None:
            specifiedFieldSize = fieldDataMap["specifiedFieldSize"]
            defaultValue = None
            if expectedValueString is not None:
                if not expectedValueString.startswith("0x"):
                    hexString = None  # * ???
                    raise Exception('The expected-value "%s" of field %s does not start with 0x' % (hexString, variableName))
                expectedValue = self.parseHex(expectedValueString)
                defaultValue = expectedValue

            if defaultValueString is not None:
                if not defaultValueString.startswith("0x"):
                    raise Exception('The expected-value "%s" of field %s does not start with 0x' % (hexString, variableName))
                defaultValue = self.parseHex(defaultValueString)

            if defaultValue is None:
                defaultValue = bytes(specifiedFieldSize)

        fieldDataMap["expectedValue"] = expectedValue
        fieldDataMap["defaultValue"] = defaultValue


class BitMaskMapDeterminer(Visitor):
    def visitFieldStart(self, generalDataMap, classDataMap, fieldDataMap):
        fieldDataMap["bitMaskMap"] = {}

    def visitFieldBit(self, generalDataMap, classDataMap, fieldDataMap, bitDataMap):
        bitName = bitDataMap["name"]
        bitMask = bitDataMap["mask"]
        bitMaskMap = fieldDataMap["bitMaskMap"]
        bitMaskMap[bitName] = bitMask


class FieldListCreator(Visitor):
    def visitClassStart(self, generalDataMap, classDataMap):
        classDataMap["fields"] = []

    def visitFieldEnd(self, generalDataMap, classDataMap, fieldDataMap):
        fields = classDataMap["fields"]
        structureName = classDataMap["structureName"]
        fieldName = fieldDataMap["fieldName"]
        typeString = fieldDataMap["typeString"]
        sinceVersion = fieldDataMap["sinceVersion"]
        tillVersion = fieldDataMap["tillVersion"]
        defaultValue = fieldDataMap["defaultValue"]
        expectedValue = fieldDataMap["expectedValue"]
        specifiedFieldSize = fieldDataMap["specifiedFieldSize"]
        structures = generalDataMap["structures"]
        bitMaskMap = fieldDataMap["bitMaskMap"]

        # TODO validate field size
        if typeString == "tag":
            field = TagField(fieldName, sinceVersion, tillVersion)
        elif typeString in intTypes:
            field = IntField(fieldName, typeString, sinceVersion, tillVersion, defaultValue, expectedValue, bitMaskMap)
        elif typeString == "float":
            field = FloatField(fieldName, typeString, sinceVersion, tillVersion, defaultValue, expectedValue)
        elif typeString == "fixed8":
            field = Fixed8Field(fieldName, typeString, sinceVersion, tillVersion, defaultValue, expectedValue)
        elif typeString is None:
            field = UnknownBytesField(fieldName, specifiedFieldSize, sinceVersion, tillVersion, defaultValue, expectedValue)
        else:
            vPos = typeString.rfind("V")
            if vPos != -1:
                fieldStructureName = typeString[:vPos]
                fieldStructureVersion = int(typeString[vPos + 1:])

            else:
                fieldStructureName = typeString
                fieldStructureVersion = 0
            if fieldStructureName == "Reference" or fieldStructureName == "SmallReference":
                refTo = fieldDataMap["refTo"]
                if (refTo is not None) and (not (refTo in structures)):
                    raise Exception("The structure with name %s referenced by %s.%s is not defined" % (refTo, structureName, fieldName))
                if refTo is not None:
                    historyOfReferencedStructures = structures[refTo]
                else:
                    historyOfReferencedStructures = None
                referenceStructureDescription = structures[fieldStructureName].getVersion(fieldStructureVersion)

                if refTo is None:
                    field = UnknownReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
                elif refTo == "CHAR":
                    field = CharReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
                elif refTo == "U8__":
                    field = ByteReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
                elif refTo == "REAL":
                    field = RealReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
                elif refTo in ["I16_", "U16_", "I32_", "U32_", "FLAG"]:
                    field = IntReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
                else:
                    field = StructureReferenceField(fieldName, referenceStructureDescription, historyOfReferencedStructures, sinceVersion, tillVersion)
            else:
                fieldStructureHistory = structures.get(fieldStructureName)
                if fieldStructureHistory is None:
                    raise Exception("The structure %s has not been defined before structure %s" % (fieldStructureName, structureName))
                fieldStructureDescription = fieldStructureHistory.getVersion(fieldStructureVersion)
                field = EmbeddedStructureField(fieldName, fieldStructureDescription, sinceVersion, tillVersion)
        fields.append(field)


class StructureHistoryListCreator(Visitor):
    def visitStart(self, generalDataMap):
        generalDataMap["structures"] = {}
        pass

    def visitClassEnd(self, generalDataMap, classDataMap):
        fields = classDataMap["fields"]
        structureName = classDataMap["structureName"]
        versionToSizeMap = classDataMap["versionToSizeMap"]
        structures = generalDataMap["structures"]

        structureHistory = M3StructureHistory(structureName, versionToSizeMap, fields)
        structures[structureName] = structureHistory

    def visitEnd(self, generalDataMap):
        pass


def foreachChildWithName(parentNode, childName):
    for childNode in parentNode.childNodes:
        if childNode.nodeName == childName:
            yield childNode


def firstNodeWithName(parentNode, childName):
    for childNode in parentNode.childNodes:
        if childNode.nodeName == childName:
            return childNode
    return None


def visitStructresDomWith(structuresDom, visitors, generalDataMap):
    for visitor in visitors:
        visitor.visitStart(generalDataMap)

    structuresNode = structuresDom.documentElement
    for structureNode in foreachChildWithName(structuresNode, "structure"):
        classDataMap = {}
        classDataMap["xmlNode"] = structureNode

        for visitor in visitors:
            visitor.visitClassStart(generalDataMap, classDataMap)

        versionsNode = firstNodeWithName(structureNode, "versions")
        for versionNode in foreachChildWithName(versionsNode, "version"):
            versionDataMap = {}
            versionDataMap["xmlNode"] = versionNode
            for visitor in visitors:

                visitor.visitVersion(generalDataMap, classDataMap, versionDataMap)

        fieldsNode = firstNodeWithName(structureNode, "fields")
        for fieldNode in foreachChildWithName(fieldsNode, "field"):
            fieldDataMap = {}
            fieldDataMap["xmlNode"] = fieldNode
            for visitor in visitors:
                visitor.visitFieldStart(generalDataMap, classDataMap, fieldDataMap)

            bitsNode = firstNodeWithName(fieldNode, "bits")
            if bitsNode is not None:
                for bitNode in foreachChildWithName(bitsNode, "bit"):
                    bitDataMap = {}
                    bitDataMap["xmlNode"] = bitNode

                    for visitor in visitors:
                        visitor.visitFieldBit(generalDataMap, classDataMap, fieldDataMap, bitDataMap)

            for visitor in visitors:
                visitor.visitFieldEnd(generalDataMap, classDataMap, fieldDataMap)

        for visitor in visitors:
            visitor. visitClassEnd(generalDataMap, classDataMap)

    for visitor in visitors:
        visitor.visitEnd(generalDataMap)


def readStructureDefinitions(structuresXmlFile):
    doc = xml.dom.minidom.parse(structuresXmlFile)
    generalDataMap = {}

    # first run is only for determing the complete list of known structures
    # firstRunVisitors = [
    #     StructureAttributesReader(),
    #     KnownStructuresListDeterminer(),
    #     KnownTagsListDeterminer()
    #     ]
    # visitStructresDomWith(doc, firstRunVisitors, generalDataMap)

    secondRunVisitors = [
        StructureAttributesReader(),
        StructureDescriptionReader(),
        FieldAttributesReader(),
        ExpectedAndDefaultConstantsDeterminer(),
        BitAttributesReader(),
        BitMaskMapDeterminer(),
        FieldListCreator(),
        StructureHistoryListCreator()
    ]

    visitStructresDomWith(doc, secondRunVisitors, generalDataMap)

    return generalDataMap["structures"]


def resolveAllReferences(list, sections):
    ListType = type([])
    for sublist in list:
        if type(sublist) == ListType:
            for entry in sublist:
                entry.resolveReferences(sections)


def loadSections(filename, checkExpectedValue=True):
    source = open(filename, "rb")
    try:
        fmagic = source.read(4)[::-1].decode('ascii')
        source.seek(0)

        m3Header = structures[fmagic].getVersion(11)
        headerBytes = source.read(m3Header.size)
        header = m3Header.createInstance(headerBytes, checkExpectedValue=checkExpectedValue)

        source.seek(header.indexOffset)
        MD34IndexEntryV0 = structures["MD34IndexEntry"].getVersion(0)
        sections = []
        for i in range(header.indexSize):
            section = Section()
            indexEntryBytes = source.read(MD34IndexEntryV0.size)
            section.indexEntry = MD34IndexEntryV0.createInstance(indexEntryBytes, checkExpectedValue=checkExpectedValue)
            sections.append(section)

        offsets = []
        for section in sections:
            indexEntry = section.indexEntry
            offsets.append(indexEntry.offset)
        offsets.append(header.indexOffset)
        offsets.sort()
        previousOffset = offsets[0]
        offsetToSizeMap = {}
        for offset in offsets[1:]:
            offsetToSizeMap[previousOffset] = offset - previousOffset
            previousOffset = offset

        unknownSections = set()
        for section in sections:
            indexEntry = section.indexEntry
            source.seek(indexEntry.offset)
            numberOfBytes = offsetToSizeMap[indexEntry.offset]
            section.rawBytes = source.read(numberOfBytes)

            structureHistory = structures.get(indexEntry.tag)
            if structureHistory is not None:
                structureDescription = structureHistory.getVersion(indexEntry.version, fmagic)
            else:
                structureDescription = None

            if structureDescription is not None:
                section.structureDescription = structureDescription
                section.determineContentField(checkExpectedValue)
            else:
                guessedUnusedSectionBytes = 0
                for i in range(1, 16):
                    if section.rawBytes[len(section.rawBytes) - i] == 0xaa:
                        guessedUnusedSectionBytes += 1
                    else:
                        break
                guessedBytesPerEntry = float(len(section.rawBytes) - guessedUnusedSectionBytes) / indexEntry.repetitions
                message = "ERROR: Unknown section at offset %s with tag=%s version=%s repetitions=%s sectionLengthInBytes=%s guessedUnusedSectionBytes=%s guessedBytesPerEntry=%s\n" % (indexEntry.offset, indexEntry.tag, indexEntry.version, indexEntry.repetitions, len(section.rawBytes), guessedUnusedSectionBytes, guessedBytesPerEntry)
                stderr.write(message)
                if sys.stderr.isatty():
                    for entryNum in range(indexEntry.repetitions):
                        stderr.write('Entry %d\n' % entryNum)
                        offset = 0
                        while offset < int(guessedBytesPerEntry):
                            val_u32 = struct.unpack_from('<I', section.rawBytes, int(guessedBytesPerEntry) * entryNum + offset)[0]
                            val_float = struct.unpack_from('<f', section.rawBytes, int(guessedBytesPerEntry) * entryNum + offset)[0]
                            stderr.write('%sV%d @%02d -> %04X:%04d = 0x%08X  %15d  %20.5f\n' % (
                                indexEntry.tag,
                                indexEntry.version,
                                entryNum,
                                offset,
                                offset,
                                val_u32,
                                val_u32,
                                val_float,
                            ))
                            offset += 4
                unknownSections.add("%sV%s" % (indexEntry.tag, indexEntry.version))
        if len(unknownSections) != 0:
            raise Exception("There were %s unknown sections: %s (see console log for more details)" % (len(unknownSections), unknownSections))
    finally:
        source.close()
    return sections


def resolveReferencesOfSections(sections):
    for section in sections:
        section.resolveReferences(sections)


def checkThatAllSectionsGotReferenced(sections):
    numberOfUnreferencedSections = 0
    referenceStructureDescription = reference = structures["SmallReference"].getVersion(0)
    for sectionIndex, section in enumerate(sections):

        if (section.timesReferenced == 0) and (sectionIndex != 0):
            numberOfUnreferencedSections += 1
            stderr.write("WARNING: %sV%s (%d repetitions) got %d times referenced\n" % (section.indexEntry.tag, section.indexEntry.version, section.indexEntry.repetitions, section.timesReferenced))
            reference = referenceStructureDescription.createInstance()
            reference.entries = section.indexEntry.repetitions
            reference.index = sectionIndex
            bytesToSearch = referenceStructureDescription.instancesToBytes([reference])
            for sectionToCheck in sections:
                positionInSection = sectionToCheck.rawBytes.find(bytesToSearch)
                if positionInSection != -1:
                    flagBytes = sectionToCheck.rawBytes[positionInSection + 8:positionInSection + 12]
                    flagsAsHex = ''.join(["%02x" % x for x in flagBytes])
                    stderr.write("  -> Found possible reference at offset %d in a section of type %sV%s with flag %s\n" % (
                        positionInSection % sectionToCheck.structureDescription.size,
                        sectionToCheck.indexEntry.tag,
                        sectionToCheck.indexEntry.version,
                        flagsAsHex
                    ))
                    sectionToCheck.structureDescription.dumpOffsets()

    if numberOfUnreferencedSections > 0:
        raise Exception("Unable to load all data: There were %d unreferenced sections. View log for details" % numberOfUnreferencedSections)


def loadModel(filename, checkExpectedValue=True):
    sections = loadSections(filename, checkExpectedValue)
    resolveReferencesOfSections(sections)
    checkThatAllSectionsGotReferenced(sections)
    header = sections[0].content[0]
    model = header.model[0]
    modelDescription = model.structureDescription
    modelDescription.validateInstance(model, "model")
    return model


class IndexReferenceSourceAndSectionListMaker:
    """ Creates a list of sections which are needed to store the objects for which index references are requested"""
    def __init__(self):
        self.objectsIdToIndexReferenceMap = {}
        self.offset = 0
        self.nextFreeIndexPosition = 0
        self.sections = []
        self.MD34IndexEntry = structures["MD34IndexEntry"].getVersion(0)

    def getIndexReferenceTo(self, objectsToSave, referenceStructureDescription, structureDescription):
        if id(objectsToSave) in self.objectsIdToIndexReferenceMap.keys():
            return self.objectsIdToIndexReferenceMap[id(objectsToSave)]

        if structureDescription is None:
            repetitions = 0
        else:
            repetitions = structureDescription.countInstances(objectsToSave)

        indexReference = referenceStructureDescription.createInstance()
        if repetitions > 0:
            indexReference.entries = repetitions
            indexReference.index = self.nextFreeIndexPosition

        if (repetitions > 0):
            indexEntry = self.MD34IndexEntry.createInstance()
            indexEntry.tag = structureDescription.structureName
            indexEntry.offset = self.offset
            indexEntry.repetitions = repetitions
            indexEntry.version = structureDescription.structureVersion

            section = Section()
            section.indexEntry = indexEntry
            section.content = objectsToSave
            section.structureDescription = structureDescription
            self.sections.append(section)
            self.objectsIdToIndexReferenceMap[id(objectsToSave)] = indexReference
            totalBytes = section.bytesRequiredForContent()
            totalBytes = increaseToValidSectionSize(totalBytes)
            self.offset += totalBytes
            self.nextFreeIndexPosition += 1
        return indexReference


def modelToSections(model):
    MD34V11 = structures["MD34"].getVersion(11)
    header = MD34V11.createInstance()
    header.tag = "MD34"
    header.model = [model]
    ReferenceV0 = structures["Reference"].getVersion(0)
    indexMaker = IndexReferenceSourceAndSectionListMaker()
    indexMaker.getIndexReferenceTo([header], ReferenceV0, MD34V11)
    header.introduceIndexReferences(indexMaker)
    sections = indexMaker.sections
    header.indexOffset = indexMaker.offset
    header.indexSize = len(sections)

    for section in sections:
        section.determineFieldRawBytes()
    return sections


def saveSections(sections, filename):
    fileObject = open(filename, "w+b")
    try:
        previousSection = None
        for section in sections:
            if section.indexEntry.offset != fileObject.tell():
                raise Exception("Section length problem: Section with index entry %(previousIndexEntry)s has length %(previousLength)s and gets followed by section with index entry %(currentIndexEntry)s" % {"previousIndexEntry": previousSection.indexEntry, "previousLength": len(previousSection.rawBytes), "currentIndexEntry": section.indexEntry})
            fileObject.write(section.rawBytes)
            previousSection = section
        header = sections[0].content[0]
        if fileObject.tell() != header.indexOffset:
            raise Exception("Not at expected write position %s after writing sections, but %s" % (header.indexOffset, fileObject.tell()))
        for section in sections:
            indexEntryBytesBuffer = bytearray(section.indexEntry.structureDescription.size)
            section.indexEntry.writeToBuffer(indexEntryBytesBuffer, 0)
            fileObject.write(indexEntryBytesBuffer)
    finally:
        fileObject.close()


def saveAndInvalidateModel(model, filename):
    '''Do not use the model object after calling this method since it gets modified'''
    model.structureDescription.validateInstance(model, "model")
    sections = modelToSections(model)
    saveSections(sections, filename)


def readStructures():
    from os import path
    directory = path.dirname(__file__)
    structuresXmlPath = path.join(directory, "structures.xml")
    return readStructureDefinitions(structuresXmlPath)


structures = readStructures()