examples.elementtree.optimized_al — SQLAlchemy 1.2 Documentation

Release: 1.2.0b2 | Release Date: July 24, 2017

# SQLAlchemy 1.2 Documentation

# Source code for examples.elementtree.optimized_al

    """Uses the same strategy as
      ``adjacency_list.py``, but associates each   DOM row with its owning
      document row, so that a full document of DOM nodes can be loaded
      using O(1) queries - the construction of the "hierarchy" is performed
      after the load in a non-recursive fashion and is more
      efficient.

    """

    ##################### PART I - Imports/Configuration #########################
    from sqlalchemy import (MetaData, Table, Column, Integer, String, ForeignKey,
        Unicode, and_, create_engine)
    from sqlalchemy.orm import mapper, relationship, Session, lazyload

    import sys, os, io, re

    from xml.etree import ElementTree

    e = create_engine('sqlite://', echo=True)
    meta = MetaData()

    ####################### PART II - Table Metadata #############################

    # stores a top level record of an XML document.
    documents = Table('documents', meta,
        Column('document_id', Integer, primary_key=True),
        Column('filename', String(30), unique=True),
    )

    # stores XML nodes in an adjacency list model.  This corresponds to
    # Element and SubElement objects.
    elements = Table('elements', meta,
        Column('element_id', Integer, primary_key=True),
        Column('parent_id', Integer, ForeignKey('elements.element_id')),
        Column('document_id', Integer, ForeignKey('documents.document_id')),
        Column('tag', Unicode(30), nullable=False),
        Column('text', Unicode),
        Column('tail', Unicode)
        )

    # stores attributes.  This corresponds to the dictionary of attributes
    # stored by an Element or SubElement.
    attributes = Table('attributes', meta,
        Column('element_id', Integer, ForeignKey('elements.element_id'), primary_key=True),
        Column('name', Unicode(100), nullable=False, primary_key=True),
        Column('value', Unicode(255)))

    meta.create_all(e)

    ########################### PART III - Model #################################

    # our document class.  contains a string name,
    # and the ElementTree root element.
    class Document(object):
        def __init__(self, name, element):
            self.filename = name
            self.element = element

        def __str__(self):
            buf = io.StringIO()
            self.element.write(buf)
            return buf.getvalue()

    ########################## PART IV - Persistence Mapping #####################

    # Node class.  a non-public class which will represent
    # the DB-persisted Element/SubElement object.  We cannot create mappers for
    # ElementTree elements directly because they are at the very least not new-style
    # classes, and also may be backed by native implementations.
    # so here we construct an adapter.
    class _Node(object):
        pass

    # Attribute class.  also internal, this will represent the key/value attributes stored for
    # a particular Node.
    class _Attribute(object):
        def __init__(self, name, value):
            self.name = name
            self.value = value

    # setup mappers.  Document will eagerly load a list of _Node objects.
    # they will be ordered in primary key/insert order, so that we can reconstruct
    # an ElementTree structure from the list.
    mapper(Document, documents, properties={
        '_nodes':relationship(_Node, lazy='joined', cascade="all, delete-orphan")
    })

    # the _Node objects change the way they load so that a list of _Nodes will organize
    # themselves hierarchically using the ElementTreeMarshal.  this depends on the ordering of
    # nodes being hierarchical as well; relationship() always applies at least ROWID/primary key
    # ordering to rows which will suffice.
    mapper(_Node, elements, properties={
        'children':relationship(_Node, lazy=None),  # doesnt load; used only for the save relationship
        'attributes':relationship(_Attribute, lazy='joined', cascade="all, delete-orphan"), # eagerly load attributes
    })

    mapper(_Attribute, attributes)

    # define marshalling functions that convert from _Node/_Attribute to/from ElementTree objects.
    # this will set the ElementTree element as "document._element", and append the root _Node
    # object to the "_nodes" mapped collection.
    class ElementTreeMarshal(object):
        def __get__(self, document, owner):
            if document is None:
                return self

            if hasattr(document, '_element'):
                return document._element

            nodes = {}
            root = None
            for node in document._nodes:
                if node.parent_id is not None:
                    parent = nodes[node.parent_id]
                    elem = ElementTree.SubElement(parent, node.tag)
                    nodes[node.element_id] = elem
                else:
                    parent = None
                    elem = root = ElementTree.Element(node.tag)
                    nodes[node.element_id] = root
                for attr in node.attributes:
                    elem.attrib[attr.name] = attr.value
                elem.text = node.text
                elem.tail = node.tail

            document._element = ElementTree.ElementTree(root)
            return document._element

        def __set__(self, document, element):
            def traverse(node):
                n = _Node()
                n.tag = str(node.tag)
                n.text = str(node.text)
                n.tail = str(node.tail)
                document._nodes.append(n)
                n.children = [traverse(n2) for n2 in node]
                n.attributes = [_Attribute(str(k), str(v)) for k, v in node.attrib.items()]
                return n

            traverse(element.getroot())
            document._element = element

        def __delete__(self, document):
            del document._element
            document._nodes = []

    # override Document's "element" attribute with the marshaller.
    Document.element = ElementTreeMarshal()

    ###################### PART V - Basic Persistence Example ####################

    line = "\n--------------------------------------------------------"

    # save to DB
    session = Session(e)

    # get ElementTree documents
    for file in ('test.xml', 'test2.xml', 'test3.xml'):
        filename = os.path.join(os.path.dirname(__file__), file)
        doc = ElementTree.parse(filename)
        session.add(Document(file, doc))

    print("\nSaving three documents...", line)
    session.commit()
    print("Done.")

    print("\nFull text of document 'text.xml':", line)
    document = session.query(Document).filter_by(filename="test.xml").first()

    print(document)

    ######################## PART VI - Searching for Paths #######################

    # manually search for a document which contains "/somefile/header/field1:hi"
    print("\nManual search for /somefile/header/field1=='hi':", line)
    d = session.query(Document).join('_nodes', aliased=True).\
                    filter(and_(_Node.parent_id==None, _Node.tag=='somefile')).\
                    join('children', aliased=True, from_joinpoint=True).\
                    filter(_Node.tag=='header').\
                    join('children', aliased=True, from_joinpoint=True).\
                    filter(and_(_Node.tag=='field1', _Node.text=='hi')).\
                    one()
    print(d)

    # generalize the above approach into an extremely impoverished xpath function:
    def find_document(path, compareto):
        j = documents
        prev_elements = None
        query = session.query(Document)
        first = True
        for i, match in enumerate(re.finditer(r'/([\w_]+)(?:\[@([\w_]+)(?:=(.*))?\])?', path)):
            (token, attrname, attrvalue) = match.group(1, 2, 3)
            if first:
                query = query.join('_nodes', aliased=True).filter(_Node.parent_id==None)
                first = False
            else:
                query = query.join('children', aliased=True, from_joinpoint=True)
            query = query.filter(_Node.tag==token)
            if attrname:
                query = query.join('attributes', aliased=True, from_joinpoint=True)
                if attrvalue:
                    query = query.filter(and_(_Attribute.name==attrname, _Attribute.value==attrvalue))
                else:
                    query = query.filter(_Attribute.name==attrname)
        return query.options(lazyload('_nodes')).filter(_Node.text==compareto).all()

    for path, compareto in (
            ('/somefile/header/field1', 'hi'),
            ('/somefile/field1', 'hi'),
            ('/somefile/header/field2', 'there'),
            ('/somefile/header/field2[@attr=foo]', 'there')
        ):
        print("\nDocuments containing '%s=%s':" % (path, compareto), line)
        print([d.filename for d in find_document(path, compareto)])