pugixml.cpp

Go to the documentation of this file.

 
#ifndef SOURCE_PUGIXML_CPP
#define SOURCE_PUGIXML_CPP
 
#ifndef NO_ETISS
#include "etiss/fault/xml/pugixml.hpp"
#else
#include "fault/xml/pugixml.hpp"
#endif
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
 
#ifdef PUGIXML_WCHAR_MODE
#include <wchar.h>
#endif
 
#ifndef PUGIXML_NO_XPATH
#include <math.h>
#include <float.h>
#ifdef PUGIXML_NO_EXCEPTIONS
#include <setjmp.h>
#endif
#endif
 
#ifndef PUGIXML_NO_STL
#include <istream>
#include <ostream>
#include <string>
#endif
 
// For placement new
#include <new>
 
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4127) // conditional expression is constant
#pragma warning(disable : 4324) // structure was padded due to __declspec(align())
#pragma warning(disable : 4611) // interaction between '_setjmp' and C++ object destruction is non-portable
#pragma warning(disable : 4702) // unreachable code
#pragma warning(disable : 4996) // this function or variable may be unsafe
#pragma warning(disable : 4793) // function compiled as native: presence of '_setjmp' makes a function unmanaged
#endif
 
#ifdef __INTEL_COMPILER
#pragma warning(disable : 177)       // function was declared but never referenced
#pragma warning(disable : 279)       // controlling expression is constant
#pragma warning(disable : 1478 1786) // function was declared "deprecated"
#pragma warning(disable : 1684)      // conversion from pointer to same-sized integral type
#endif
 
#if defined(__BORLANDC__) && defined(PUGIXML_HEADER_ONLY)
#pragma warn - 8080 // symbol is declared but never used; disabling this inside push/pop bracket does not make the
                    // warning go away
#endif
 
#ifdef __BORLANDC__
#pragma option push
#pragma warn - 8008 // condition is always false
#pragma warn - 8066 // unreachable code
#endif
 
#ifdef __SNC__
// Using diag_push/diag_pop does not disable the warnings inside templates due to a compiler bug
#pragma diag_suppress = 178 // function was declared but never referenced
#pragma diag_suppress = 237 // controlling expression is constant
#endif
 
// Inlining controls
#if defined(_MSC_VER) && _MSC_VER >= 1300
#define PUGI__NO_INLINE __declspec(noinline)
#elif defined(__GNUC__)
#define PUGI__NO_INLINE __attribute__((noinline))
#else
#define PUGI__NO_INLINE
#endif
 
// Simple static assertion
#define PUGI__STATIC_ASSERT(cond)                                    \
    {                                                                \
        static const char condition_failed[(cond) ? 1 : -1] = { 0 }; \
        (void)condition_failed[0];                                   \
    }
 
// Digital Mars C++ bug workaround for passing char loaded from memory via stack
#ifdef __DMC__
#define PUGI__DMC_VOLATILE volatile
#else
#define PUGI__DMC_VOLATILE
#endif
 
// Borland C++ bug workaround for not defining ::memcpy depending on header include order (can't always use std::memcpy
// because some compilers don't have it at all)
#if defined(__BORLANDC__) && !defined(__MEM_H_USING_LIST)
using std::memcpy;
using std::memmove;
#endif
 
// In some environments MSVC is a compiler but the CRT lacks certain
// MSVC-specific features
#if defined(_MSC_VER) && !defined(__S3E__)
#define PUGI__MSVC_CRT_VERSION _MSC_VER
#endif
 
#ifdef PUGIXML_HEADER_ONLY
#define PUGI__NS_BEGIN \
    namespace pugi     \
    {                  \
    namespace impl     \
    {
#define PUGI__NS_END \
    }                \
    }
#define PUGI__FN inline
#define PUGI__FN_NO_INLINE inline
#else
#if defined(_MSC_VER) && \
    _MSC_VER < 1300 // MSVC6 seems to have an amusing bug with anonymous namespaces inside namespaces
#define PUGI__NS_BEGIN \
    namespace pugi     \
    {                  \
    namespace impl     \
    {
#define PUGI__NS_END \
    }                \
    }
#else
#define PUGI__NS_BEGIN \
    namespace pugi     \
    {                  \
    namespace impl     \
    {                  \
    namespace          \
    {
#define PUGI__NS_END \
    }                \
    }                \
    }
#endif
#define PUGI__FN
#define PUGI__FN_NO_INLINE PUGI__NO_INLINE
#endif
 
// uintptr_t
#if !defined(_MSC_VER) || _MSC_VER >= 1600
#include <stdint.h>
#else
#ifndef _UINTPTR_T_DEFINED
// No native uintptr_t in MSVC6 and in some WinCE versions
typedef size_t uintptr_t;
#define _UINTPTR_T_DEFINED
#endif
PUGI__NS_BEGIN
typedef unsigned __int8 uint8_t;
typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t;
PUGI__NS_END
#endif
 
// Memory allocation
PUGI__NS_BEGIN
PUGI__FN void *default_allocate(size_t size)
{
    return malloc(size);
}
 
PUGI__FN void default_deallocate(void *ptr)
{
    free(ptr);
}
 
template <typename T>
struct xml_memory_management_function_storage
{
    static allocation_function allocate;
    static deallocation_function deallocate;
};
 
template <typename T>
allocation_function xml_memory_management_function_storage<T>::allocate = default_allocate;
template <typename T>
deallocation_function xml_memory_management_function_storage<T>::deallocate = default_deallocate;
 
typedef xml_memory_management_function_storage<int> xml_memory;
PUGI__NS_END
 
// String utilities
PUGI__NS_BEGIN
// Get string length
PUGI__FN size_t strlength(const char_t *s)
{
    assert(s);
 
#ifdef PUGIXML_WCHAR_MODE
    return wcslen(s);
#else
    return strlen(s);
#endif
}
 
// Compare two strings
PUGI__FN bool strequal(const char_t *src, const char_t *dst)
{
    assert(src && dst);
 
#ifdef PUGIXML_WCHAR_MODE
    return wcscmp(src, dst) == 0;
#else
    return strcmp(src, dst) == 0;
#endif
}
 
// Compare lhs with [rhs_begin, rhs_end)
PUGI__FN bool strequalrange(const char_t *lhs, const char_t *rhs, size_t count)
{
    for (size_t i = 0; i < count; ++i)
        if (lhs[i] != rhs[i])
            return false;
 
    return lhs[count] == 0;
}
 
// Get length of wide string, even if CRT lacks wide character support
PUGI__FN size_t strlength_wide(const wchar_t *s)
{
    assert(s);
 
#ifdef PUGIXML_WCHAR_MODE
    return wcslen(s);
#else
    const wchar_t *end = s;
    while (*end)
        end++;
    return static_cast<size_t>(end - s);
#endif
}
 
#ifdef PUGIXML_WCHAR_MODE
// Convert string to wide string, assuming all symbols are ASCII
PUGI__FN void widen_ascii(wchar_t *dest, const char *source)
{
    for (const char *i = source; *i; ++i)
        *dest++ = *i;
    *dest = 0;
}
#endif
PUGI__NS_END
 
#if !defined(PUGIXML_NO_STL) || !defined(PUGIXML_NO_XPATH)
// auto_ptr-like buffer holder for exception recovery
PUGI__NS_BEGIN
struct buffer_holder
{
    void *data;
    void (*deleter)(void *);
 
    buffer_holder(void *data_, void (*deleter_)(void *)) : data(data_), deleter(deleter_) {}
 
    ~buffer_holder()
    {
        if (data)
            deleter(data);
    }
 
    void *release()
    {
        void *result = data;
        data = 0;
        return result;
    }
};
PUGI__NS_END
#endif
 
PUGI__NS_BEGIN
static const size_t xml_memory_page_size =
#ifdef PUGIXML_MEMORY_PAGE_SIZE
    PUGIXML_MEMORY_PAGE_SIZE
#else
    32768
#endif
    ;
 
static const uintptr_t xml_memory_page_alignment = 32;
static const uintptr_t xml_memory_page_pointer_mask = ~(xml_memory_page_alignment - 1);
static const uintptr_t xml_memory_page_name_allocated_mask = 16;
static const uintptr_t xml_memory_page_value_allocated_mask = 8;
static const uintptr_t xml_memory_page_type_mask = 7;
 
struct xml_allocator;
 
struct xml_memory_page
{
    static xml_memory_page *construct(void *memory)
    {
        if (!memory)
            return 0; //$ redundant, left for performance
 
        xml_memory_page *result = static_cast<xml_memory_page *>(memory);
 
        result->allocator = 0;
        result->memory = 0;
        result->prev = 0;
        result->next = 0;
        result->busy_size = 0;
        result->freed_size = 0;
 
        return result;
    }
 
    xml_allocator *allocator;
 
    void *memory;
 
    xml_memory_page *prev;
    xml_memory_page *next;
 
    size_t busy_size;
    size_t freed_size;
 
    char data[1];
};
 
struct xml_memory_string_header
{
    uint16_t page_offset; // offset from page->data
    uint16_t full_size;   // 0 if string occupies whole page
};
 
struct xml_allocator
{
    xml_allocator(xml_memory_page *root) : _root(root), _busy_size(root->busy_size) {}
 
    xml_memory_page *allocate_page(size_t data_size)
    {
        size_t size = offsetof(xml_memory_page, data) + data_size;
 
        // allocate block with some alignment, leaving memory for worst-case padding
        void *memory = xml_memory::allocate(size + xml_memory_page_alignment);
        if (!memory)
            return 0;
 
        // align upwards to page boundary
        void *page_memory = reinterpret_cast<void *>(
            (reinterpret_cast<uintptr_t>(memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));
 
        // prepare page structure
        xml_memory_page *page = xml_memory_page::construct(page_memory);
        assert(page);
 
        page->memory = memory;
        page->allocator = _root->allocator;
 
        return page;
    }
 
    static void deallocate_page(xml_memory_page *page) { xml_memory::deallocate(page->memory); }
 
    void *allocate_memory_oob(size_t size, xml_memory_page *&out_page);
 
    void *allocate_memory(size_t size, xml_memory_page *&out_page)
    {
        if (_busy_size + size > xml_memory_page_size)
            return allocate_memory_oob(size, out_page);
 
        void *buf = _root->data + _busy_size;
 
        _busy_size += size;
 
        out_page = _root;
 
        return buf;
    }
 
    void deallocate_memory(void *ptr, size_t size, xml_memory_page *page)
    {
        if (page == _root)
            page->busy_size = _busy_size;
 
        assert(ptr >= page->data && ptr < page->data + page->busy_size);
        (void)!ptr;
 
        page->freed_size += size;
        assert(page->freed_size <= page->busy_size);
 
        if (page->freed_size == page->busy_size)
        {
            if (page->next == 0)
            {
                assert(_root == page);
 
                // top page freed, just reset sizes
                page->busy_size = page->freed_size = 0;
                _busy_size = 0;
            }
            else
            {
                assert(_root != page);
                assert(page->prev);
 
                // remove from the list
                page->prev->next = page->next;
                page->next->prev = page->prev;
 
                // deallocate
                deallocate_page(page);
            }
        }
    }
 
    char_t *allocate_string(size_t length)
    {
        // allocate memory for string and header block
        size_t size = sizeof(xml_memory_string_header) + length * sizeof(char_t);
 
        // round size up to pointer alignment boundary
        size_t full_size = (size + (sizeof(void *) - 1)) & ~(sizeof(void *) - 1);
 
        xml_memory_page *page;
        xml_memory_string_header *header = static_cast<xml_memory_string_header *>(allocate_memory(full_size, page));
 
        if (!header)
            return 0;
 
        // setup header
        ptrdiff_t page_offset = reinterpret_cast<char *>(header) - page->data;
 
        assert(page_offset >= 0 && page_offset < (1 << 16));
        header->page_offset = static_cast<uint16_t>(page_offset);
 
        // full_size == 0 for large strings that occupy the whole page
        assert(full_size < (1 << 16) || (page->busy_size == full_size && page_offset == 0));
        header->full_size = static_cast<uint16_t>(full_size < (1 << 16) ? full_size : 0);
 
        // round-trip through void* to avoid 'cast increases required alignment of target type' warning
        // header is guaranteed a pointer-sized alignment, which should be enough for char_t
        return static_cast<char_t *>(static_cast<void *>(header + 1));
    }
 
    void deallocate_string(char_t *string)
    {
        // this function casts pointers through void* to avoid 'cast increases required alignment of target type'
        // warnings we're guaranteed the proper (pointer-sized) alignment on the input string if it was allocated via
        // allocate_string
 
        // get header
        xml_memory_string_header *header = static_cast<xml_memory_string_header *>(static_cast<void *>(string)) - 1;
 
        // deallocate
        size_t page_offset = offsetof(xml_memory_page, data) + header->page_offset;
        xml_memory_page *page =
            reinterpret_cast<xml_memory_page *>(static_cast<void *>(reinterpret_cast<char *>(header) - page_offset));
 
        // if full_size == 0 then this string occupies the whole page
        size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size;
 
        deallocate_memory(header, full_size, page);
    }
 
    xml_memory_page *_root;
    size_t _busy_size;
};
 
PUGI__FN_NO_INLINE void *xml_allocator::allocate_memory_oob(size_t size, xml_memory_page *&out_page)
{
    const size_t large_allocation_threshold = xml_memory_page_size / 4;
 
    xml_memory_page *page = allocate_page(size <= large_allocation_threshold ? xml_memory_page_size : size);
    out_page = page;
 
    if (!page)
        return 0;
 
    if (size <= large_allocation_threshold)
    {
        _root->busy_size = _busy_size;
 
        // insert page at the end of linked list
        page->prev = _root;
        _root->next = page;
        _root = page;
 
        _busy_size = size;
    }
    else
    {
        // insert page before the end of linked list, so that it is deleted as soon as possible
        // the last page is not deleted even if it's empty (see deallocate_memory)
        assert(_root->prev);
 
        page->prev = _root->prev;
        page->next = _root;
 
        _root->prev->next = page;
        _root->prev = page;
    }
 
    // allocate inside page
    page->busy_size = size;
 
    return page->data;
}
PUGI__NS_END
 
namespace pugi
{
struct xml_attribute_struct
{
    xml_attribute_struct(impl::xml_memory_page *page)
        : header(reinterpret_cast<uintptr_t>(page)), name(0), value(0), prev_attribute_c(0), next_attribute(0)
    {
    }
 
    uintptr_t header;
 
    char_t *name;  
    char_t *value; 
 
    xml_attribute_struct *prev_attribute_c; 
    xml_attribute_struct *next_attribute;   
};
 
struct xml_node_struct
{
    xml_node_struct(impl::xml_memory_page *page, xml_node_type type)
        : header(reinterpret_cast<uintptr_t>(page) | (type - 1))
        , parent(0)
        , name(0)
        , value(0)
        , first_child(0)
        , prev_sibling_c(0)
        , next_sibling(0)
        , first_attribute(0)
    {
    }
 
    uintptr_t header;
 
    xml_node_struct *parent; 
 
    char_t *name;  
    char_t *value; 
 
    xml_node_struct *first_child; 
 
    xml_node_struct *prev_sibling_c; 
    xml_node_struct *next_sibling;   
 
    xml_attribute_struct *first_attribute; 
};
} // namespace pugi
 
PUGI__NS_BEGIN
struct xml_extra_buffer
{
    char_t *buffer;
    xml_extra_buffer *next;
};
 
struct xml_document_struct : public xml_node_struct, public xml_allocator
{
    xml_document_struct(xml_memory_page *page)
        : xml_node_struct(page, node_document), xml_allocator(page), buffer(0), extra_buffers(0)
    {
    }
 
    const char_t *buffer;
 
    xml_extra_buffer *extra_buffers;
};
 
inline xml_allocator &get_allocator(const xml_node_struct *node)
{
    assert(node);
 
    return *reinterpret_cast<xml_memory_page *>(node->header & xml_memory_page_pointer_mask)->allocator;
}
PUGI__NS_END
 
// Low-level DOM operations
PUGI__NS_BEGIN
inline xml_attribute_struct *allocate_attribute(xml_allocator &alloc)
{
    xml_memory_page *page;
    void *memory = alloc.allocate_memory(sizeof(xml_attribute_struct), page);
 
    return new (memory) xml_attribute_struct(page);
}
 
inline xml_node_struct *allocate_node(xml_allocator &alloc, xml_node_type type)
{
    xml_memory_page *page;
    void *memory = alloc.allocate_memory(sizeof(xml_node_struct), page);
 
    return new (memory) xml_node_struct(page, type);
}
 
inline void destroy_attribute(xml_attribute_struct *a, xml_allocator &alloc)
{
    uintptr_t header = a->header;
 
    if (header & impl::xml_memory_page_name_allocated_mask)
        alloc.deallocate_string(a->name);
    if (header & impl::xml_memory_page_value_allocated_mask)
        alloc.deallocate_string(a->value);
 
    alloc.deallocate_memory(a, sizeof(xml_attribute_struct),
                            reinterpret_cast<xml_memory_page *>(header & xml_memory_page_pointer_mask));
}
 
inline void destroy_node(xml_node_struct *n, xml_allocator &alloc)
{
    uintptr_t header = n->header;
 
    if (header & impl::xml_memory_page_name_allocated_mask)
        alloc.deallocate_string(n->name);
    if (header & impl::xml_memory_page_value_allocated_mask)
        alloc.deallocate_string(n->value);
 
    for (xml_attribute_struct *attr = n->first_attribute; attr;)
    {
        xml_attribute_struct *next = attr->next_attribute;
 
        destroy_attribute(attr, alloc);
 
        attr = next;
    }
 
    for (xml_node_struct *child = n->first_child; child;)
    {
        xml_node_struct *next = child->next_sibling;
 
        destroy_node(child, alloc);
 
        child = next;
    }
 
    alloc.deallocate_memory(n, sizeof(xml_node_struct),
                            reinterpret_cast<xml_memory_page *>(header & xml_memory_page_pointer_mask));
}
 
PUGI__FN_NO_INLINE xml_node_struct *append_node(xml_node_struct *node, xml_allocator &alloc,
                                                xml_node_type type = node_element)
{
    xml_node_struct *child = allocate_node(alloc, type);
    if (!child)
        return 0;
 
    child->parent = node;
 
    xml_node_struct *first_child = node->first_child;
 
    if (first_child)
    {
        xml_node_struct *last_child = first_child->prev_sibling_c;
 
        last_child->next_sibling = child;
        child->prev_sibling_c = last_child;
        first_child->prev_sibling_c = child;
    }
    else
    {
        node->first_child = child;
        child->prev_sibling_c = child;
    }
 
    return child;
}
 
PUGI__FN_NO_INLINE xml_attribute_struct *append_attribute_ll(xml_node_struct *node, xml_allocator &alloc)
{
    xml_attribute_struct *a = allocate_attribute(alloc);
    if (!a)
        return 0;
 
    xml_attribute_struct *first_attribute = node->first_attribute;
 
    if (first_attribute)
    {
        xml_attribute_struct *last_attribute = first_attribute->prev_attribute_c;
 
        last_attribute->next_attribute = a;
        a->prev_attribute_c = last_attribute;
        first_attribute->prev_attribute_c = a;
    }
    else
    {
        node->first_attribute = a;
        a->prev_attribute_c = a;
    }
 
    return a;
}
PUGI__NS_END
 
// Helper classes for code generation
PUGI__NS_BEGIN
struct opt_false
{
    enum
    {
        value = 0
    };
};
 
struct opt_true
{
    enum
    {
        value = 1
    };
};
PUGI__NS_END
 
// Unicode utilities
PUGI__NS_BEGIN
inline uint16_t endian_swap(uint16_t value)
{
    return static_cast<uint16_t>(((value & 0xff) << 8) | (value >> 8));
}
 
inline uint32_t endian_swap(uint32_t value)
{
    return ((value & 0xff) << 24) | ((value & 0xff00) << 8) | ((value & 0xff0000) >> 8) | (value >> 24);
}
 
struct utf8_counter
{
    typedef size_t value_type;
 
    static value_type low(value_type result, uint32_t ch)
    {
        // U+0000..U+007F
        if (ch < 0x80)
            return result + 1;
        // U+0080..U+07FF
        else if (ch < 0x800)
            return result + 2;
        // U+0800..U+FFFF
        else
            return result + 3;
    }
 
    static value_type high(value_type result, uint32_t)
    {
        // U+10000..U+10FFFF
        return result + 4;
    }
};
 
struct utf8_writer
{
    typedef uint8_t *value_type;
 
    static value_type low(value_type result, uint32_t ch)
    {
        // U+0000..U+007F
        if (ch < 0x80)
        {
            *result = static_cast<uint8_t>(ch);
            return result + 1;
        }
        // U+0080..U+07FF
        else if (ch < 0x800)
        {
            result[0] = static_cast<uint8_t>(0xC0 | (ch >> 6));
            result[1] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
            return result + 2;
        }
        // U+0800..U+FFFF
        else
        {
            result[0] = static_cast<uint8_t>(0xE0 | (ch >> 12));
            result[1] = static_cast<uint8_t>(0x80 | ((ch >> 6) & 0x3F));
            result[2] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
            return result + 3;
        }
    }
 
    static value_type high(value_type result, uint32_t ch)
    {
        // U+10000..U+10FFFF
        result[0] = static_cast<uint8_t>(0xF0 | (ch >> 18));
        result[1] = static_cast<uint8_t>(0x80 | ((ch >> 12) & 0x3F));
        result[2] = static_cast<uint8_t>(0x80 | ((ch >> 6) & 0x3F));
        result[3] = static_cast<uint8_t>(0x80 | (ch & 0x3F));
        return result + 4;
    }
 
    static value_type any(value_type result, uint32_t ch)
    {
        return (ch < 0x10000) ? low(result, ch) : high(result, ch);
    }
};
 
struct utf16_counter
{
    typedef size_t value_type;
 
    static value_type low(value_type result, uint32_t) { return result + 1; }
 
    static value_type high(value_type result, uint32_t) { return result + 2; }
};
 
struct utf16_writer
{
    typedef uint16_t *value_type;
 
    static value_type low(value_type result, uint32_t ch)
    {
        *result = static_cast<uint16_t>(ch);
 
        return result + 1;
    }
 
    static value_type high(value_type result, uint32_t ch)
    {
        uint32_t msh = static_cast<uint32_t>(ch - 0x10000) >> 10;
        uint32_t lsh = static_cast<uint32_t>(ch - 0x10000) & 0x3ff;
 
        result[0] = static_cast<uint16_t>(0xD800 + msh);
        result[1] = static_cast<uint16_t>(0xDC00 + lsh);
 
        return result + 2;
    }
 
    static value_type any(value_type result, uint32_t ch)
    {
        return (ch < 0x10000) ? low(result, ch) : high(result, ch);
    }
};
 
struct utf32_counter
{
    typedef size_t value_type;
 
    static value_type low(value_type result, uint32_t) { return result + 1; }
 
    static value_type high(value_type result, uint32_t) { return result + 1; }
};
 
struct utf32_writer
{
    typedef uint32_t *value_type;
 
    static value_type low(value_type result, uint32_t ch)
    {
        *result = ch;
 
        return result + 1;
    }
 
    static value_type high(value_type result, uint32_t ch)
    {
        *result = ch;
 
        return result + 1;
    }
 
    static value_type any(value_type result, uint32_t ch)
    {
        *result = ch;
 
        return result + 1;
    }
};
 
struct latin1_writer
{
    typedef uint8_t *value_type;
 
    static value_type low(value_type result, uint32_t ch)
    {
        *result = static_cast<uint8_t>(ch > 255 ? '?' : ch);
 
        return result + 1;
    }
 
    static value_type high(value_type result, uint32_t ch)
    {
        (void)ch;
 
        *result = '?';
 
        return result + 1;
    }
};
 
template <size_t size>
struct wchar_selector;
 
template <>
struct wchar_selector<2>
{
    typedef uint16_t type;
    typedef utf16_counter counter;
    typedef utf16_writer writer;
};
 
template <>
struct wchar_selector<4>
{
    typedef uint32_t type;
    typedef utf32_counter counter;
    typedef utf32_writer writer;
};
 
typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;
 
template <typename Traits, typename opt_swap = opt_false>
struct utf_decoder
{
    static inline typename Traits::value_type decode_utf8_block(const uint8_t *data, size_t size,
                                                                typename Traits::value_type result)
    {
        const uint8_t utf8_byte_mask = 0x3f;
 
        while (size)
        {
            uint8_t lead = *data;
 
            // 0xxxxxxx -> U+0000..U+007F
            if (lead < 0x80)
            {
                result = Traits::low(result, lead);
                data += 1;
                size -= 1;
 
                // process aligned single-byte (ascii) blocks
                if ((reinterpret_cast<uintptr_t>(data) & 3) == 0)
                {
                    // round-trip through void* to silence 'cast increases required alignment of target type' warnings
                    while (size >= 4 &&
                           (*static_cast<const uint32_t *>(static_cast<const void *>(data)) & 0x80808080) == 0)
                    {
                        result = Traits::low(result, data[0]);
                        result = Traits::low(result, data[1]);
                        result = Traits::low(result, data[2]);
                        result = Traits::low(result, data[3]);
                        data += 4;
                        size -= 4;
                    }
                }
            }
            // 110xxxxx -> U+0080..U+07FF
            else if (static_cast<unsigned int>(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
            {
                result = Traits::low(result, ((lead & ~0xC0) << 6) | (data[1] & utf8_byte_mask));
                data += 2;
                size -= 2;
            }
            // 1110xxxx -> U+0800-U+FFFF
            else if (static_cast<unsigned int>(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 &&
                     (data[2] & 0xc0) == 0x80)
            {
                result = Traits::low(result, ((lead & ~0xE0) << 12) | ((data[1] & utf8_byte_mask) << 6) |
                                                 (data[2] & utf8_byte_mask));
                data += 3;
                size -= 3;
            }
            // 11110xxx -> U+10000..U+10FFFF
            else if (static_cast<unsigned int>(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 &&
                     (data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
            {
                result = Traits::high(result, ((lead & ~0xF0) << 18) | ((data[1] & utf8_byte_mask) << 12) |
                                                  ((data[2] & utf8_byte_mask) << 6) | (data[3] & utf8_byte_mask));
                data += 4;
                size -= 4;
            }
            // 10xxxxxx or 11111xxx -> invalid
            else
            {
                data += 1;
                size -= 1;
            }
        }
 
        return result;
    }
 
    static inline typename Traits::value_type decode_utf16_block(const uint16_t *data, size_t size,
                                                                 typename Traits::value_type result)
    {
        const uint16_t *end = data + size;
 
        while (data < end)
        {
            unsigned int lead = opt_swap::value ? endian_swap(*data) : *data;
 
            // U+0000..U+D7FF
            if (lead < 0xD800)
            {
                result = Traits::low(result, lead);
                data += 1;
            }
            // U+E000..U+FFFF
            else if (static_cast<unsigned int>(lead - 0xE000) < 0x2000)
            {
                result = Traits::low(result, lead);
                data += 1;
            }
            // surrogate pair lead
            else if (static_cast<unsigned int>(lead - 0xD800) < 0x400 && data + 1 < end)
            {
                uint16_t next = opt_swap::value ? endian_swap(data[1]) : data[1];
 
                if (static_cast<unsigned int>(next - 0xDC00) < 0x400)
                {
                    result = Traits::high(result, 0x10000 + ((lead & 0x3ff) << 10) + (next & 0x3ff));
                    data += 2;
                }
                else
                {
                    data += 1;
                }
            }
            else
            {
                data += 1;
            }
        }
 
        return result;
    }
 
    static inline typename Traits::value_type decode_utf32_block(const uint32_t *data, size_t size,
                                                                 typename Traits::value_type result)
    {
        const uint32_t *end = data + size;
 
        while (data < end)
        {
            uint32_t lead = opt_swap::value ? endian_swap(*data) : *data;
 
            // U+0000..U+FFFF
            if (lead < 0x10000)
            {
                result = Traits::low(result, lead);
                data += 1;
            }
            // U+10000..U+10FFFF
            else
            {
                result = Traits::high(result, lead);
                data += 1;
            }
        }
 
        return result;
    }
 
    static inline typename Traits::value_type decode_latin1_block(const uint8_t *data, size_t size,
                                                                  typename Traits::value_type result)
    {
        for (size_t i = 0; i < size; ++i)
        {
            result = Traits::low(result, data[i]);
        }
 
        return result;
    }
 
    static inline typename Traits::value_type decode_wchar_block_impl(const uint16_t *data, size_t size,
                                                                      typename Traits::value_type result)
    {
        return decode_utf16_block(data, size, result);
    }
 
    static inline typename Traits::value_type decode_wchar_block_impl(const uint32_t *data, size_t size,
                                                                      typename Traits::value_type result)
    {
        return decode_utf32_block(data, size, result);
    }
 
    static inline typename Traits::value_type decode_wchar_block(const wchar_t *data, size_t size,
                                                                 typename Traits::value_type result)
    {
        return decode_wchar_block_impl(reinterpret_cast<const wchar_selector<sizeof(wchar_t)>::type *>(data), size,
                                       result);
    }
};
 
template <typename T>
PUGI__FN void convert_utf_endian_swap(T *result, const T *data, size_t length)
{
    for (size_t i = 0; i < length; ++i)
        result[i] = endian_swap(data[i]);
}
 
#ifdef PUGIXML_WCHAR_MODE
PUGI__FN void convert_wchar_endian_swap(wchar_t *result, const wchar_t *data, size_t length)
{
    for (size_t i = 0; i < length; ++i)
        result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
}
#endif
PUGI__NS_END
 
PUGI__NS_BEGIN
enum chartype_t
{
    ct_parse_pcdata = 1,   // \0, &, \r, <
    ct_parse_attr = 2,     // \0, &, \r, ', "
    ct_parse_attr_ws = 4,  // \0, &, \r, ', ", \n, tab
    ct_space = 8,          // \r, \n, space, tab
    ct_parse_cdata = 16,   // \0, ], >, \r
    ct_parse_comment = 32, // \0, -, >, \r
    ct_symbol = 64,        // Any symbol > 127, a-z, A-Z, 0-9, _, :, -, .
    ct_start_symbol = 128  // Any symbol > 127, a-z, A-Z, _, :
};
 
static const unsigned char chartype_table[256] = {
    55,  0,   0,   0,   0,   0,   0,   0,   0,   12,  12,  0,   0,   63,  0,   0,   // 0-15
    0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   // 16-31
    8,   0,   6,   0,   0,   0,   7,   6,   0,   0,   0,   0,   0,   96,  64,  0,   // 32-47
    64,  64,  64,  64,  64,  64,  64,  64,  64,  64,  192, 0,   1,   0,   48,  0,   // 48-63
    0,   192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 64-79
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 0,   0,   16,  0,   192, // 80-95
    0,   192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 96-111
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 0,   0,   0,   0,   0,   // 112-127
 
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, // 128+
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192,
    192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, 192
};
 
enum chartypex_t
{
    ctx_special_pcdata = 1, // Any symbol >= 0 and < 32 (except \t, \r, \n), &, <, >
    ctx_special_attr = 2,   // Any symbol >= 0 and < 32 (except \t), &, <, >, "
    ctx_start_symbol = 4,   // Any symbol > 127, a-z, A-Z, _
    ctx_digit = 8,          // 0-9
    ctx_symbol = 16         // Any symbol > 127, a-z, A-Z, 0-9, _, -, .
};
 
static const unsigned char chartypex_table[256] = {
    3,  3,  3,  3,  3,  3,  3,  3,  3,  0,  2,  3,  3,  2,  3,  3, // 0-15
    3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, // 16-31
    0,  0,  2,  0,  0,  0,  3,  0,  0,  0,  0,  0,  0,  16, 16, 0, // 32-47
    24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0,  0,  3,  0,  3,  0, // 48-63
 
    0,  20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 64-79
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 0,  0,  0,  0,  20, // 80-95
    0,  20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 96-111
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 0,  0,  0,  0,  0,  // 112-127
 
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, // 128+
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
    20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20
};
 
#ifdef PUGIXML_WCHAR_MODE
#define PUGI__IS_CHARTYPE_IMPL(c, ct, table) \
    ((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
#else
#define PUGI__IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
#endif
 
#define PUGI__IS_CHARTYPE(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartype_table)
#define PUGI__IS_CHARTYPEX(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartypex_table)
 
PUGI__FN bool is_little_endian()
{
    unsigned int ui = 1;
 
    return *reinterpret_cast<unsigned char *>(&ui) == 1;
}
 
PUGI__FN xml_encoding get_wchar_encoding()
{
    PUGI__STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
 
    if (sizeof(wchar_t) == 2)
        return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
    else
        return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
}
 
PUGI__FN xml_encoding guess_buffer_encoding(uint8_t d0, uint8_t d1, uint8_t d2, uint8_t d3)
{
    // look for BOM in first few bytes
    if (d0 == 0 && d1 == 0 && d2 == 0xfe && d3 == 0xff)
        return encoding_utf32_be;
    if (d0 == 0xff && d1 == 0xfe && d2 == 0 && d3 == 0)
        return encoding_utf32_le;
    if (d0 == 0xfe && d1 == 0xff)
        return encoding_utf16_be;
    if (d0 == 0xff && d1 == 0xfe)
        return encoding_utf16_le;
    if (d0 == 0xef && d1 == 0xbb && d2 == 0xbf)
        return encoding_utf8;
 
    // look for <, <? or <?xm in various encodings
    if (d0 == 0 && d1 == 0 && d2 == 0 && d3 == 0x3c)
        return encoding_utf32_be;
    if (d0 == 0x3c && d1 == 0 && d2 == 0 && d3 == 0)
        return encoding_utf32_le;
    if (d0 == 0 && d1 == 0x3c && d2 == 0 && d3 == 0x3f)
        return encoding_utf16_be;
    if (d0 == 0x3c && d1 == 0 && d2 == 0x3f && d3 == 0)
        return encoding_utf16_le;
    if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d)
        return encoding_utf8;
 
    // look for utf16 < followed by node name (this may fail, but is better than utf8 since it's zero terminated so
    // early)
    if (d0 == 0 && d1 == 0x3c)
        return encoding_utf16_be;
    if (d0 == 0x3c && d1 == 0)
        return encoding_utf16_le;
 
    // no known BOM detected, assume utf8
    return encoding_utf8;
}
 
PUGI__FN xml_encoding get_buffer_encoding(xml_encoding encoding, const void *contents, size_t size)
{
    // replace wchar encoding with utf implementation
    if (encoding == encoding_wchar)
        return get_wchar_encoding();
 
    // replace utf16 encoding with utf16 with specific endianness
    if (encoding == encoding_utf16)
        return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
    // replace utf32 encoding with utf32 with specific endianness
    if (encoding == encoding_utf32)
        return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
    // only do autodetection if no explicit encoding is requested
    if (encoding != encoding_auto)
        return encoding;
 
    // skip encoding autodetection if input buffer is too small
    if (size < 4)
        return encoding_utf8;
 
    // try to guess encoding (based on XML specification, Appendix F.1)
    const uint8_t *data = static_cast<const uint8_t *>(contents);
 
    PUGI__DMC_VOLATILE uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];
 
    return guess_buffer_encoding(d0, d1, d2, d3);
}
 
PUGI__FN bool get_mutable_buffer(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size,
                                 bool is_mutable)
{
    size_t length = size / sizeof(char_t);
 
    if (is_mutable)
    {
        out_buffer = static_cast<char_t *>(const_cast<void *>(contents));
        out_length = length;
    }
    else
    {
        char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
        if (!buffer)
            return false;
 
        memcpy(buffer, contents, length * sizeof(char_t));
        buffer[length] = 0;
 
        out_buffer = buffer;
        out_length = length + 1;
    }
 
    return true;
}
 
#ifdef PUGIXML_WCHAR_MODE
PUGI__FN bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
{
    return (le == encoding_utf16_be && re == encoding_utf16_le) ||
           (le == encoding_utf16_le && re == encoding_utf16_be) ||
           (le == encoding_utf32_be && re == encoding_utf32_le) || (le == encoding_utf32_le && re == encoding_utf32_be);
}
 
PUGI__FN bool convert_buffer_endian_swap(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size,
                                         bool is_mutable)
{
    const char_t *data = static_cast<const char_t *>(contents);
    size_t length = size / sizeof(char_t);
 
    if (is_mutable)
    {
        char_t *buffer = const_cast<char_t *>(data);
 
        convert_wchar_endian_swap(buffer, data, length);
 
        out_buffer = buffer;
        out_length = length;
    }
    else
    {
        char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
        if (!buffer)
            return false;
 
        convert_wchar_endian_swap(buffer, data, length);
        buffer[length] = 0;
 
        out_buffer = buffer;
        out_length = length + 1;
    }
 
    return true;
}
 
PUGI__FN bool convert_buffer_utf8(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size)
{
    const uint8_t *data = static_cast<const uint8_t *>(contents);
    size_t data_length = size;
 
    // first pass: get length in wchar_t units
    size_t length = utf_decoder<wchar_counter>::decode_utf8_block(data, data_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert utf8 input to wchar_t
    wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
    wchar_writer::value_type oend = utf_decoder<wchar_writer>::decode_utf8_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
template <typename opt_swap>
PUGI__FN bool convert_buffer_utf16(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size, opt_swap)
{
    const uint16_t *data = static_cast<const uint16_t *>(contents);
    size_t data_length = size / sizeof(uint16_t);
 
    // first pass: get length in wchar_t units
    size_t length = utf_decoder<wchar_counter, opt_swap>::decode_utf16_block(data, data_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert utf16 input to wchar_t
    wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
    wchar_writer::value_type oend = utf_decoder<wchar_writer, opt_swap>::decode_utf16_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
template <typename opt_swap>
PUGI__FN bool convert_buffer_utf32(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size, opt_swap)
{
    const uint32_t *data = static_cast<const uint32_t *>(contents);
    size_t data_length = size / sizeof(uint32_t);
 
    // first pass: get length in wchar_t units
    size_t length = utf_decoder<wchar_counter, opt_swap>::decode_utf32_block(data, data_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert utf32 input to wchar_t
    wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
    wchar_writer::value_type oend = utf_decoder<wchar_writer, opt_swap>::decode_utf32_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
PUGI__FN bool convert_buffer_latin1(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size)
{
    const uint8_t *data = static_cast<const uint8_t *>(contents);
    size_t data_length = size;
 
    // get length in wchar_t units
    size_t length = data_length;
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // convert latin1 input to wchar_t
    wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
    wchar_writer::value_type oend = utf_decoder<wchar_writer>::decode_latin1_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
PUGI__FN bool convert_buffer(char_t *&out_buffer, size_t &out_length, xml_encoding encoding, const void *contents,
                             size_t size, bool is_mutable)
{
    // get native encoding
    xml_encoding wchar_encoding = get_wchar_encoding();
 
    // fast path: no conversion required
    if (encoding == wchar_encoding)
        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
    // only endian-swapping is required
    if (need_endian_swap_utf(encoding, wchar_encoding))
        return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);
 
    // source encoding is utf8
    if (encoding == encoding_utf8)
        return convert_buffer_utf8(out_buffer, out_length, contents, size);
 
    // source encoding is utf16
    if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
    {
        xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
        return (native_encoding == encoding) ? convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false())
                                             : convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
    }
 
    // source encoding is utf32
    if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
    {
        xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
        return (native_encoding == encoding) ? convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false())
                                             : convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
    }
 
    // source encoding is latin1
    if (encoding == encoding_latin1)
        return convert_buffer_latin1(out_buffer, out_length, contents, size);
 
    assert(!"Invalid encoding");
    return false;
}
#else
template <typename opt_swap>
PUGI__FN bool convert_buffer_utf16(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size, opt_swap)
{
    const uint16_t *data = static_cast<const uint16_t *>(contents);
    size_t data_length = size / sizeof(uint16_t);
 
    // first pass: get length in utf8 units
    size_t length = utf_decoder<utf8_counter, opt_swap>::decode_utf16_block(data, data_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert utf16 input to utf8
    uint8_t *obegin = reinterpret_cast<uint8_t *>(buffer);
    uint8_t *oend = utf_decoder<utf8_writer, opt_swap>::decode_utf16_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
template <typename opt_swap>
PUGI__FN bool convert_buffer_utf32(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size, opt_swap)
{
    const uint32_t *data = static_cast<const uint32_t *>(contents);
    size_t data_length = size / sizeof(uint32_t);
 
    // first pass: get length in utf8 units
    size_t length = utf_decoder<utf8_counter, opt_swap>::decode_utf32_block(data, data_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert utf32 input to utf8
    uint8_t *obegin = reinterpret_cast<uint8_t *>(buffer);
    uint8_t *oend = utf_decoder<utf8_writer, opt_swap>::decode_utf32_block(data, data_length, obegin);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
PUGI__FN size_t get_latin1_7bit_prefix_length(const uint8_t *data, size_t size)
{
    for (size_t i = 0; i < size; ++i)
        if (data[i] > 127)
            return i;
 
    return size;
}
 
PUGI__FN bool convert_buffer_latin1(char_t *&out_buffer, size_t &out_length, const void *contents, size_t size,
                                    bool is_mutable)
{
    const uint8_t *data = static_cast<const uint8_t *>(contents);
    size_t data_length = size;
 
    // get size of prefix that does not need utf8 conversion
    size_t prefix_length = get_latin1_7bit_prefix_length(data, data_length);
    assert(prefix_length <= data_length);
 
    const uint8_t *postfix = data + prefix_length;
    size_t postfix_length = data_length - prefix_length;
 
    // if no conversion is needed, just return the original buffer
    if (postfix_length == 0)
        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
    // first pass: get length in utf8 units
    size_t length = prefix_length + utf_decoder<utf8_counter>::decode_latin1_block(postfix, postfix_length, 0);
 
    // allocate buffer of suitable length
    char_t *buffer = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
    if (!buffer)
        return false;
 
    // second pass: convert latin1 input to utf8
    memcpy(buffer, data, prefix_length);
 
    uint8_t *obegin = reinterpret_cast<uint8_t *>(buffer);
    uint8_t *oend = utf_decoder<utf8_writer>::decode_latin1_block(postfix, postfix_length, obegin + prefix_length);
 
    assert(oend == obegin + length);
    *oend = 0;
 
    out_buffer = buffer;
    out_length = length + 1;
 
    return true;
}
 
PUGI__FN bool convert_buffer(char_t *&out_buffer, size_t &out_length, xml_encoding encoding, const void *contents,
                             size_t size, bool is_mutable)
{
    // fast path: no conversion required
    if (encoding == encoding_utf8)
        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
    // source encoding is utf16
    if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
    {
        xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
        return (native_encoding == encoding) ? convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false())
                                             : convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
    }
 
    // source encoding is utf32
    if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
    {
        xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
        return (native_encoding == encoding) ? convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false())
                                             : convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
    }
 
    // source encoding is latin1
    if (encoding == encoding_latin1)
        return convert_buffer_latin1(out_buffer, out_length, contents, size, is_mutable);
 
    assert(!"Invalid encoding");
    return false;
}
#endif
 
PUGI__FN size_t as_utf8_begin(const wchar_t *str, size_t length)
{
    // get length in utf8 characters
    return utf_decoder<utf8_counter>::decode_wchar_block(str, length, 0);
}
 
PUGI__FN void as_utf8_end(char *buffer, size_t size, const wchar_t *str, size_t length)
{
    // convert to utf8
    uint8_t *begin = reinterpret_cast<uint8_t *>(buffer);
    uint8_t *end = utf_decoder<utf8_writer>::decode_wchar_block(str, length, begin);
 
    assert(begin + size == end);
    (void)!end;
 
    // zero-terminate
    buffer[size] = 0;
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN std::string as_utf8_impl(const wchar_t *str, size_t length)
{
    // first pass: get length in utf8 characters
    size_t size = as_utf8_begin(str, length);
 
    // allocate resulting string
    std::string result;
    result.resize(size);
 
    // second pass: convert to utf8
    if (size > 0)
        as_utf8_end(&result[0], size, str, length);
 
    return result;
}
 
PUGI__FN std::basic_string<wchar_t> as_wide_impl(const char *str, size_t size)
{
    const uint8_t *data = reinterpret_cast<const uint8_t *>(str);
 
    // first pass: get length in wchar_t units
    size_t length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);
 
    // allocate resulting string
    std::basic_string<wchar_t> result;
    result.resize(length);
 
    // second pass: convert to wchar_t
    if (length > 0)
    {
        wchar_writer::value_type begin = reinterpret_cast<wchar_writer::value_type>(&result[0]);
        wchar_writer::value_type end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, begin);
 
        assert(begin + length == end);
        (void)!end;
    }
 
    return result;
}
#endif
 
inline bool strcpy_insitu_allow(size_t length, uintptr_t allocated, char_t *target)
{
    assert(target);
    size_t target_length = strlength(target);
 
    // always reuse document buffer memory if possible
    if (!allocated)
        return target_length >= length;
 
    // reuse heap memory if waste is not too great
    const size_t reuse_threshold = 32;
 
    return target_length >= length && (target_length < reuse_threshold || target_length - length < target_length / 2);
}
 
PUGI__FN bool strcpy_insitu(char_t *&dest, uintptr_t &header, uintptr_t header_mask, const char_t *source)
{
    assert(header);
 
    size_t source_length = strlength(source);
 
    if (source_length == 0)
    {
        // empty string and null pointer are equivalent, so just deallocate old memory
        xml_allocator *alloc = reinterpret_cast<xml_memory_page *>(header & xml_memory_page_pointer_mask)->allocator;
 
        if (header & header_mask)
            alloc->deallocate_string(dest);
 
        // mark the string as not allocated
        dest = 0;
        header &= ~header_mask;
 
        return true;
    }
    else if (dest && strcpy_insitu_allow(source_length, header & header_mask, dest))
    {
        // we can reuse old buffer, so just copy the new data (including zero terminator)
        memcpy(dest, source, (source_length + 1) * sizeof(char_t));
 
        return true;
    }
    else
    {
        xml_allocator *alloc = reinterpret_cast<xml_memory_page *>(header & xml_memory_page_pointer_mask)->allocator;
 
        // allocate new buffer
        char_t *buf = alloc->allocate_string(source_length + 1);
        if (!buf)
            return false;
 
        // copy the string (including zero terminator)
        memcpy(buf, source, (source_length + 1) * sizeof(char_t));
 
        // deallocate old buffer (*after* the above to protect against overlapping memory and/or allocation failures)
        if (header & header_mask)
            alloc->deallocate_string(dest);
 
        // the string is now allocated, so set the flag
        dest = buf;
        header |= header_mask;
 
        return true;
    }
}
 
struct gap
{
    char_t *end;
    size_t size;
 
    gap() : end(0), size(0) {}
 
    // Push new gap, move s count bytes further (skipping the gap).
    // Collapse previous gap.
    void push(char_t *&s, size_t count)
    {
        if (end) // there was a gap already; collapse it
        {
            // Move [old_gap_end, new_gap_start) to [old_gap_start, ...)
            assert(s >= end);
            memmove(end - size, end, reinterpret_cast<char *>(s) - reinterpret_cast<char *>(end));
        }
 
        s += count; // end of current gap
 
        // "merge" two gaps
        end = s;
        size += count;
    }
 
    // Collapse all gaps, return past-the-end pointer
    char_t *flush(char_t *s)
    {
        if (end)
        {
            // Move [old_gap_end, current_pos) to [old_gap_start, ...)
            assert(s >= end);
            memmove(end - size, end, reinterpret_cast<char *>(s) - reinterpret_cast<char *>(end));
 
            return s - size;
        }
        else
            return s;
    }
};
 
PUGI__FN char_t *strconv_escape(char_t *s, gap &g)
{
    char_t *stre = s + 1;
 
    switch (*stre)
    {
    case '#': // &#...
    {
        unsigned int ucsc = 0;
 
        if (stre[1] == 'x') // &#x... (hex code)
        {
            stre += 2;
 
            char_t ch = *stre;
 
            if (ch == ';')
                return stre;
 
            for (;;)
            {
                if (static_cast<unsigned int>(ch - '0') <= 9)
                    ucsc = 16 * ucsc + (ch - '0');
                else if (static_cast<unsigned int>((ch | ' ') - 'a') <= 5)
                    ucsc = 16 * ucsc + ((ch | ' ') - 'a' + 10);
                else if (ch == ';')
                    break;
                else // cancel
                    return stre;
 
                ch = *++stre;
            }
 
            ++stre;
        }
        else // &#... (dec code)
        {
            char_t ch = *++stre;
 
            if (ch == ';')
                return stre;
 
            for (;;)
            {
                if (static_cast<unsigned int>(static_cast<unsigned int>(ch) - '0') <= 9)
                    ucsc = 10 * ucsc + (ch - '0');
                else if (ch == ';')
                    break;
                else // cancel
                    return stre;
 
                ch = *++stre;
            }
 
            ++stre;
        }
 
#ifdef PUGIXML_WCHAR_MODE
        s = reinterpret_cast<char_t *>(wchar_writer::any(reinterpret_cast<wchar_writer::value_type>(s), ucsc));
#else
        s = reinterpret_cast<char_t *>(utf8_writer::any(reinterpret_cast<uint8_t *>(s), ucsc));
#endif
 
        g.push(s, stre - s);
        return stre;
    }
 
    case 'a': // &a
    {
        ++stre;
 
        if (*stre == 'm') // &am
        {
            if (*++stre == 'p' && *++stre == ';') // &amp;
            {
                *s++ = '&';
                ++stre;
 
                g.push(s, stre - s);
                return stre;
            }
        }
        else if (*stre == 'p') // &ap
        {
            if (*++stre == 'o' && *++stre == 's' && *++stre == ';') // &apos;
            {
                *s++ = '\'';
                ++stre;
 
                g.push(s, stre - s);
                return stre;
            }
        }
        break;
    }
 
    case 'g': // &g
    {
        if (*++stre == 't' && *++stre == ';') // &gt;
        {
            *s++ = '>';
            ++stre;
 
            g.push(s, stre - s);
            return stre;
        }
        break;
    }
 
    case 'l': // &l
    {
        if (*++stre == 't' && *++stre == ';') // &lt;
        {
            *s++ = '<';
            ++stre;
 
            g.push(s, stre - s);
            return stre;
        }
        break;
    }
 
    case 'q': // &q
    {
        if (*++stre == 'u' && *++stre == 'o' && *++stre == 't' && *++stre == ';') // &quot;
        {
            *s++ = '"';
            ++stre;
 
            g.push(s, stre - s);
            return stre;
        }
        break;
    }
 
    default:
        break;
    }
 
    return stre;
}
 
// Utility macro for last character handling
#define ENDSWITH(c, e) ((c) == (e) || ((c) == 0 && endch == (e)))
 
PUGI__FN char_t *strconv_comment(char_t *s, char_t endch)
{
    gap g;
 
    while (true)
    {
        while (!PUGI__IS_CHARTYPE(*s, ct_parse_comment))
            ++s;
 
        if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
        {
            *s++ = '\n'; // replace first one with 0x0a
 
            if (*s == '\n')
                g.push(s, 1);
        }
        else if (s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>')) // comment ends here
        {
            *g.flush(s) = 0;
 
            return s + (s[2] == '>' ? 3 : 2);
        }
        else if (*s == 0)
        {
            return 0;
        }
        else
            ++s;
    }
}
 
PUGI__FN char_t *strconv_cdata(char_t *s, char_t endch)
{
    gap g;
 
    while (true)
    {
        while (!PUGI__IS_CHARTYPE(*s, ct_parse_cdata))
            ++s;
 
        if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
        {
            *s++ = '\n'; // replace first one with 0x0a
 
            if (*s == '\n')
                g.push(s, 1);
        }
        else if (s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>')) // CDATA ends here
        {
            *g.flush(s) = 0;
 
            return s + 1;
        }
        else if (*s == 0)
        {
            return 0;
        }
        else
            ++s;
    }
}
 
typedef char_t *(*strconv_pcdata_t)(char_t *);
 
template <typename opt_trim, typename opt_eol, typename opt_escape>
struct strconv_pcdata_impl
{
    static char_t *parse(char_t *s)
    {
        gap g;
 
        char_t *begin = s;
 
        while (true)
        {
            while (!PUGI__IS_CHARTYPE(*s, ct_parse_pcdata))
                ++s;
 
            if (*s == '<') // PCDATA ends here
            {
                char_t *end = g.flush(s);
 
                if (opt_trim::value)
                    while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
                        --end;
 
                *end = 0;
 
                return s + 1;
            }
            else if (opt_eol::value && *s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
            {
                *s++ = '\n'; // replace first one with 0x0a
 
                if (*s == '\n')
                    g.push(s, 1);
            }
            else if (opt_escape::value && *s == '&')
            {
                s = strconv_escape(s, g);
            }
            else if (*s == 0)
            {
                char_t *end = g.flush(s);
 
                if (opt_trim::value)
                    while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
                        --end;
 
                *end = 0;
 
                return s;
            }
            else
                ++s;
        }
    }
};
 
PUGI__FN strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
{
    PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_trim_pcdata == 0x0800);
 
    switch (((optmask >> 4) & 3) | ((optmask >> 9) & 4)) // get bitmask for flags (eol escapes trim)
    {
    case 0:
        return strconv_pcdata_impl<opt_false, opt_false, opt_false>::parse;
    case 1:
        return strconv_pcdata_impl<opt_false, opt_false, opt_true>::parse;
    case 2:
        return strconv_pcdata_impl<opt_false, opt_true, opt_false>::parse;
    case 3:
        return strconv_pcdata_impl<opt_false, opt_true, opt_true>::parse;
    case 4:
        return strconv_pcdata_impl<opt_true, opt_false, opt_false>::parse;
    case 5:
        return strconv_pcdata_impl<opt_true, opt_false, opt_true>::parse;
    case 6:
        return strconv_pcdata_impl<opt_true, opt_true, opt_false>::parse;
    case 7:
        return strconv_pcdata_impl<opt_true, opt_true, opt_true>::parse;
    default:
        assert(false);
        return 0; // should not get here
    }
}
 
typedef char_t *(*strconv_attribute_t)(char_t *, char_t);
 
template <typename opt_escape>
struct strconv_attribute_impl
{
    static char_t *parse_wnorm(char_t *s, char_t end_quote)
    {
        gap g;
 
        // trim leading whitespaces
        if (PUGI__IS_CHARTYPE(*s, ct_space))
        {
            char_t *str = s;
 
            do
                ++str;
            while (PUGI__IS_CHARTYPE(*str, ct_space));
 
            g.push(s, str - s);
        }
 
        while (true)
        {
            while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr_ws | ct_space))
                ++s;
 
            if (*s == end_quote)
            {
                char_t *str = g.flush(s);
 
                do
                    *str-- = 0;
                while (PUGI__IS_CHARTYPE(*str, ct_space));
 
                return s + 1;
            }
            else if (PUGI__IS_CHARTYPE(*s, ct_space))
            {
                *s++ = ' ';
 
                if (PUGI__IS_CHARTYPE(*s, ct_space))
                {
                    char_t *str = s + 1;
                    while (PUGI__IS_CHARTYPE(*str, ct_space))
                        ++str;
 
                    g.push(s, str - s);
                }
            }
            else if (opt_escape::value && *s == '&')
            {
                s = strconv_escape(s, g);
            }
            else if (!*s)
            {
                return 0;
            }
            else
                ++s;
        }
    }
 
    static char_t *parse_wconv(char_t *s, char_t end_quote)
    {
        gap g;
 
        while (true)
        {
            while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr_ws))
                ++s;
 
            if (*s == end_quote)
            {
                *g.flush(s) = 0;
 
                return s + 1;
            }
            else if (PUGI__IS_CHARTYPE(*s, ct_space))
            {
                if (*s == '\r')
                {
                    *s++ = ' ';
 
                    if (*s == '\n')
                        g.push(s, 1);
                }
                else
                    *s++ = ' ';
            }
            else if (opt_escape::value && *s == '&')
            {
                s = strconv_escape(s, g);
            }
            else if (!*s)
            {
                return 0;
            }
            else
                ++s;
        }
    }
 
    static char_t *parse_eol(char_t *s, char_t end_quote)
    {
        gap g;
 
        while (true)
        {
            while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr))
                ++s;
 
            if (*s == end_quote)
            {
                *g.flush(s) = 0;
 
                return s + 1;
            }
            else if (*s == '\r')
            {
                *s++ = '\n';
 
                if (*s == '\n')
                    g.push(s, 1);
            }
            else if (opt_escape::value && *s == '&')
            {
                s = strconv_escape(s, g);
            }
            else if (!*s)
            {
                return 0;
            }
            else
                ++s;
        }
    }
 
    static char_t *parse_simple(char_t *s, char_t end_quote)
    {
        gap g;
 
        while (true)
        {
            while (!PUGI__IS_CHARTYPE(*s, ct_parse_attr))
                ++s;
 
            if (*s == end_quote)
            {
                *g.flush(s) = 0;
 
                return s + 1;
            }
            else if (opt_escape::value && *s == '&')
            {
                s = strconv_escape(s, g);
            }
            else if (!*s)
            {
                return 0;
            }
            else
                ++s;
        }
    }
};
 
PUGI__FN strconv_attribute_t get_strconv_attribute(unsigned int optmask)
{
    PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 &&
                        parse_wnorm_attribute == 0x80);
 
    switch ((optmask >> 4) & 15) // get bitmask for flags (wconv wnorm eol escapes)
    {
    case 0:
        return strconv_attribute_impl<opt_false>::parse_simple;
    case 1:
        return strconv_attribute_impl<opt_true>::parse_simple;
    case 2:
        return strconv_attribute_impl<opt_false>::parse_eol;
    case 3:
        return strconv_attribute_impl<opt_true>::parse_eol;
    case 4:
        return strconv_attribute_impl<opt_false>::parse_wconv;
    case 5:
        return strconv_attribute_impl<opt_true>::parse_wconv;
    case 6:
        return strconv_attribute_impl<opt_false>::parse_wconv;
    case 7:
        return strconv_attribute_impl<opt_true>::parse_wconv;
    case 8:
        return strconv_attribute_impl<opt_false>::parse_wnorm;
    case 9:
        return strconv_attribute_impl<opt_true>::parse_wnorm;
    case 10:
        return strconv_attribute_impl<opt_false>::parse_wnorm;
    case 11:
        return strconv_attribute_impl<opt_true>::parse_wnorm;
    case 12:
        return strconv_attribute_impl<opt_false>::parse_wnorm;
    case 13:
        return strconv_attribute_impl<opt_true>::parse_wnorm;
    case 14:
        return strconv_attribute_impl<opt_false>::parse_wnorm;
    case 15:
        return strconv_attribute_impl<opt_true>::parse_wnorm;
    default:
        assert(false);
        return 0; // should not get here
    }
}
 
inline xml_parse_result make_parse_result(xml_parse_status status, ptrdiff_t offset = 0)
{
    xml_parse_result result;
    result.status = status;
    result.offset = offset;
 
    return result;
}
 
struct xml_parser
{
    xml_allocator alloc;
    char_t *error_offset;
    xml_parse_status error_status;
 
// Parser utilities.
#define PUGI__SKIPWS()                          \
    {                                           \
        while (PUGI__IS_CHARTYPE(*s, ct_space)) \
            ++s;                                \
    }
#define PUGI__OPTSET(OPT) (optmsk & (OPT))
#define PUGI__PUSHNODE(TYPE)                            \
    {                                                   \
        cursor = append_node(cursor, alloc, TYPE);      \
        if (!cursor)                                    \
            PUGI__THROW_ERROR(status_out_of_memory, s); \
    }
#define PUGI__POPNODE()          \
    {                            \
        cursor = cursor->parent; \
    }
#define PUGI__SCANFOR(X)        \
    {                           \
        while (*s != 0 && !(X)) \
            ++s;                \
    }
#define PUGI__SCANWHILE(X) \
    {                      \
        while ((X))        \
            ++s;           \
    }
#define PUGI__ENDSEG() \
    {                  \
        ch = *s;       \
        *s = 0;        \
        ++s;           \
    }
#define PUGI__THROW_ERROR(err, m) return error_offset = m, error_status = err, static_cast<char_t *>(0)
#define PUGI__CHECK_ERROR(err, m)      \
    {                                  \
        if (*s == 0)                   \
            PUGI__THROW_ERROR(err, m); \
    }
 
    xml_parser(const xml_allocator &alloc_) : alloc(alloc_), error_offset(0), error_status(status_ok) {}
 
    // DOCTYPE consists of nested sections of the following possible types:
    // <!-- ... -->, <? ... ?>, "...", '...'
    // <![...]]>
    // <!...>
    // First group can not contain nested groups
    // Second group can contain nested groups of the same type
    // Third group can contain all other groups
    char_t *parse_doctype_primitive(char_t *s)
    {
        if (*s == '"' || *s == '\'')
        {
            // quoted string
            char_t ch = *s++;
            PUGI__SCANFOR(*s == ch);
            if (!*s)
                PUGI__THROW_ERROR(status_bad_doctype, s);
 
            s++;
        }
        else if (s[0] == '<' && s[1] == '?')
        {
            // <? ... ?>
            s += 2;
            PUGI__SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
            if (!*s)
                PUGI__THROW_ERROR(status_bad_doctype, s);
 
            s += 2;
        }
        else if (s[0] == '<' && s[1] == '!' && s[2] == '-' && s[3] == '-')
        {
            s += 4;
            PUGI__SCANFOR(s[0] == '-' && s[1] == '-' &&
                          s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
            if (!*s)
                PUGI__THROW_ERROR(status_bad_doctype, s);
 
            s += 4;
        }
        else
            PUGI__THROW_ERROR(status_bad_doctype, s);
 
        return s;
    }
 
    char_t *parse_doctype_ignore(char_t *s)
    {
        assert(s[0] == '<' && s[1] == '!' && s[2] == '[');
        s++;
 
        while (*s)
        {
            if (s[0] == '<' && s[1] == '!' && s[2] == '[')
            {
                // nested ignore section
                s = parse_doctype_ignore(s);
                if (!s)
                    return s;
            }
            else if (s[0] == ']' && s[1] == ']' && s[2] == '>')
            {
                // ignore section end
                s += 3;
 
                return s;
            }
            else
                s++;
        }
 
        PUGI__THROW_ERROR(status_bad_doctype, s);
    }
 
    char_t *parse_doctype_group(char_t *s, char_t endch, bool toplevel)
    {
        assert((s[0] == '<' || s[0] == 0) && s[1] == '!');
        s++;
 
        while (*s)
        {
            if (s[0] == '<' && s[1] == '!' && s[2] != '-')
            {
                if (s[2] == '[')
                {
                    // ignore
                    s = parse_doctype_ignore(s);
                    if (!s)
                        return s;
                }
                else
                {
                    // some control group
                    s = parse_doctype_group(s, endch, false);
                    if (!s)
                        return s;
 
                    // skip >
                    assert(*s == '>');
                    s++;
                }
            }
            else if (s[0] == '<' || s[0] == '"' || s[0] == '\'')
            {
                // unknown tag (forbidden), or some primitive group
                s = parse_doctype_primitive(s);
                if (!s)
                    return s;
            }
            else if (*s == '>')
            {
                return s;
            }
            else
                s++;
        }
 
        if (!toplevel || endch != '>')
            PUGI__THROW_ERROR(status_bad_doctype, s);
 
        return s;
    }
 
    char_t *parse_exclamation(char_t *s, xml_node_struct *cursor, unsigned int optmsk, char_t endch)
    {
        // parse node contents, starting with exclamation mark
        ++s;
 
        if (*s == '-') // '<!-...'
        {
            ++s;
 
            if (*s == '-') // '<!--...'
            {
                ++s;
 
                if (PUGI__OPTSET(parse_comments))
                {
                    PUGI__PUSHNODE(node_comment); // Append a new node on the tree.
                    cursor->value = s;            // Save the offset.
                }
 
                if (PUGI__OPTSET(parse_eol) && PUGI__OPTSET(parse_comments))
                {
                    s = strconv_comment(s, endch);
 
                    if (!s)
                        PUGI__THROW_ERROR(status_bad_comment, cursor->value);
                }
                else
                {
                    // Scan for terminating '-->'.
                    PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>'));
                    PUGI__CHECK_ERROR(status_bad_comment, s);
 
                    if (PUGI__OPTSET(parse_comments))
                        *s = 0; // Zero-terminate this segment at the first terminating '-'.
 
                    s += (s[2] == '>' ? 3 : 2); // Step over the '\0->'.
                }
            }
            else
                PUGI__THROW_ERROR(status_bad_comment, s);
        }
        else if (*s == '[')
        {
            // '<![CDATA[...'
            if (*++s == 'C' && *++s == 'D' && *++s == 'A' && *++s == 'T' && *++s == 'A' && *++s == '[')
            {
                ++s;
 
                if (PUGI__OPTSET(parse_cdata))
                {
                    PUGI__PUSHNODE(node_cdata); // Append a new node on the tree.
                    cursor->value = s;          // Save the offset.
 
                    if (PUGI__OPTSET(parse_eol))
                    {
                        s = strconv_cdata(s, endch);
 
                        if (!s)
                            PUGI__THROW_ERROR(status_bad_cdata, cursor->value);
                    }
                    else
                    {
                        // Scan for terminating ']]>'.
                        PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
                        PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                        *s++ = 0; // Zero-terminate this segment.
                    }
                }
                else // Flagged for discard, but we still have to scan for the terminator.
                {
                    // Scan for terminating ']]>'.
                    PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
                    PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                    ++s;
                }
 
                s += (s[1] == '>' ? 2 : 1); // Step over the last ']>'.
            }
            else
                PUGI__THROW_ERROR(status_bad_cdata, s);
        }
        else if (s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' &&
                 ENDSWITH(s[6], 'E'))
        {
            s -= 2;
 
            if (cursor->parent)
                PUGI__THROW_ERROR(status_bad_doctype, s);
 
            char_t *mark = s + 9;
 
            s = parse_doctype_group(s, endch, true);
            if (!s)
                return s;
 
            assert((*s == 0 && endch == '>') || *s == '>');
            if (*s)
                *s++ = 0;
 
            if (PUGI__OPTSET(parse_doctype))
            {
                while (PUGI__IS_CHARTYPE(*mark, ct_space))
                    ++mark;
 
                PUGI__PUSHNODE(node_doctype);
 
                cursor->value = mark;
 
                PUGI__POPNODE();
            }
        }
        else if (*s == 0 && endch == '-')
            PUGI__THROW_ERROR(status_bad_comment, s);
        else if (*s == 0 && endch == '[')
            PUGI__THROW_ERROR(status_bad_cdata, s);
        else
            PUGI__THROW_ERROR(status_unrecognized_tag, s);
 
        return s;
    }
 
    char_t *parse_question(char_t *s, xml_node_struct *&ref_cursor, unsigned int optmsk, char_t endch)
    {
        // load into registers
        xml_node_struct *cursor = ref_cursor;
        char_t ch = 0;
 
        // parse node contents, starting with question mark
        ++s;
 
        // read PI target
        char_t *target = s;
 
        if (!PUGI__IS_CHARTYPE(*s, ct_start_symbol))
            PUGI__THROW_ERROR(status_bad_pi, s);
 
        PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol));
        PUGI__CHECK_ERROR(status_bad_pi, s);
 
        // determine node type; stricmp / strcasecmp is not portable
        bool declaration =
            (target[0] | ' ') == 'x' && (target[1] | ' ') == 'm' && (target[2] | ' ') == 'l' && target + 3 == s;
 
        if (declaration ? PUGI__OPTSET(parse_declaration) : PUGI__OPTSET(parse_pi))
        {
            if (declaration)
            {
                // disallow non top-level declarations
                if (cursor->parent)
                    PUGI__THROW_ERROR(status_bad_pi, s);
 
                PUGI__PUSHNODE(node_declaration);
            }
            else
            {
                PUGI__PUSHNODE(node_pi);
            }
 
            cursor->name = target;
 
            PUGI__ENDSEG();
 
            // parse value/attributes
            if (ch == '?')
            {
                // empty node
                if (!ENDSWITH(*s, '>'))
                    PUGI__THROW_ERROR(status_bad_pi, s);
                s += (*s == '>');
 
                PUGI__POPNODE();
            }
            else if (PUGI__IS_CHARTYPE(ch, ct_space))
            {
                PUGI__SKIPWS();
 
                // scan for tag end
                char_t *value = s;
 
                PUGI__SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
                PUGI__CHECK_ERROR(status_bad_pi, s);
 
                if (declaration)
                {
                    // replace ending ? with / so that 'element' terminates properly
                    *s = '/';
 
                    // we exit from this function with cursor at node_declaration, which is a signal to parse() to go to
                    // LOC_ATTRIBUTES
                    s = value;
                }
                else
                {
                    // store value and step over >
                    cursor->value = value;
                    PUGI__POPNODE();
 
                    PUGI__ENDSEG();
 
                    s += (*s == '>');
                }
            }
            else
                PUGI__THROW_ERROR(status_bad_pi, s);
        }
        else
        {
            // scan for tag end
            PUGI__SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
            PUGI__CHECK_ERROR(status_bad_pi, s);
 
            s += (s[1] == '>' ? 2 : 1);
        }
 
        // store from registers
        ref_cursor = cursor;
 
        return s;
    }
 
    char_t *parse_tree(char_t *s, xml_node_struct *root, unsigned int optmsk, char_t endch)
    {
        strconv_attribute_t strconv_attribute = get_strconv_attribute(optmsk);
        strconv_pcdata_t strconv_pcdata = get_strconv_pcdata(optmsk);
 
        char_t ch = 0;
        xml_node_struct *cursor = root;
        char_t *mark = s;
 
        while (*s != 0)
        {
            if (*s == '<')
            {
                ++s;
 
            LOC_TAG:
                if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
                {
                    PUGI__PUSHNODE(node_element); // Append a new node to the tree.
 
                    cursor->name = s;
 
                    PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
                    PUGI__ENDSEG();                                    // Save char in 'ch', terminate & step over.
 
                    if (ch == '>')
                    {
                        // end of tag
                    }
                    else if (PUGI__IS_CHARTYPE(ch, ct_space))
                    {
                    LOC_ATTRIBUTES:
                        while (true)
                        {
                            PUGI__SKIPWS(); // Eat any whitespace.
 
                            if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
                            {
                                xml_attribute_struct *a =
                                    append_attribute_ll(cursor, alloc); // Make space for this attribute.
                                if (!a)
                                    PUGI__THROW_ERROR(status_out_of_memory, s);
 
                                a->name = s; // Save the offset.
 
                                PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
                                PUGI__CHECK_ERROR(status_bad_attribute, s);        //$ redundant, left for performance
 
                                PUGI__ENDSEG();                             // Save char in 'ch', terminate & step over.
                                PUGI__CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
 
                                if (PUGI__IS_CHARTYPE(ch, ct_space))
                                {
                                    PUGI__SKIPWS();                             // Eat any whitespace.
                                    PUGI__CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
 
                                    ch = *s;
                                    ++s;
                                }
 
                                if (ch == '=') // '<... #=...'
                                {
                                    PUGI__SKIPWS(); // Eat any whitespace.
 
                                    if (*s == '"' || *s == '\'') // '<... #="...'
                                    {
                                        ch = *s;      // Save quote char to avoid breaking on "''" -or- '""'.
                                        ++s;          // Step over the quote.
                                        a->value = s; // Save the offset.
 
                                        s = strconv_attribute(s, ch);
 
                                        if (!s)
                                            PUGI__THROW_ERROR(status_bad_attribute, a->value);
 
                                        // After this line the loop continues from the start;
                                        // Whitespaces, / and > are ok, symbols and EOF are wrong,
                                        // everything else will be detected
                                        if (PUGI__IS_CHARTYPE(*s, ct_start_symbol))
                                            PUGI__THROW_ERROR(status_bad_attribute, s);
                                    }
                                    else
                                        PUGI__THROW_ERROR(status_bad_attribute, s);
                                }
                                else
                                    PUGI__THROW_ERROR(status_bad_attribute, s);
                            }
                            else if (*s == '/')
                            {
                                ++s;
 
                                if (*s == '>')
                                {
                                    PUGI__POPNODE();
                                    s++;
                                    break;
                                }
                                else if (*s == 0 && endch == '>')
                                {
                                    PUGI__POPNODE();
                                    break;
                                }
                                else
                                    PUGI__THROW_ERROR(status_bad_start_element, s);
                            }
                            else if (*s == '>')
                            {
                                ++s;
 
                                break;
                            }
                            else if (*s == 0 && endch == '>')
                            {
                                break;
                            }
                            else
                                PUGI__THROW_ERROR(status_bad_start_element, s);
                        }
 
                        // !!!
                    }
                    else if (ch == '/') // '<#.../'
                    {
                        if (!ENDSWITH(*s, '>'))
                            PUGI__THROW_ERROR(status_bad_start_element, s);
 
                        PUGI__POPNODE(); // Pop.
 
                        s += (*s == '>');
                    }
                    else if (ch == 0)
                    {
                        // we stepped over null terminator, backtrack & handle closing tag
                        --s;
 
                        if (endch != '>')
                            PUGI__THROW_ERROR(status_bad_start_element, s);
                    }
                    else
                        PUGI__THROW_ERROR(status_bad_start_element, s);
                }
                else if (*s == '/')
                {
                    ++s;
 
                    char_t *name = cursor->name;
                    if (!name)
                        PUGI__THROW_ERROR(status_end_element_mismatch, s);
 
                    while (PUGI__IS_CHARTYPE(*s, ct_symbol))
                    {
                        if (*s++ != *name++)
                            PUGI__THROW_ERROR(status_end_element_mismatch, s);
                    }
 
                    if (*name)
                    {
                        if (*s == 0 && name[0] == endch && name[1] == 0)
                            PUGI__THROW_ERROR(status_bad_end_element, s);
                        else
                            PUGI__THROW_ERROR(status_end_element_mismatch, s);
                    }
 
                    PUGI__POPNODE(); // Pop.
 
                    PUGI__SKIPWS();
 
                    if (*s == 0)
                    {
                        if (endch != '>')
                            PUGI__THROW_ERROR(status_bad_end_element, s);
                    }
                    else
                    {
                        if (*s != '>')
                            PUGI__THROW_ERROR(status_bad_end_element, s);
                        ++s;
                    }
                }
                else if (*s == '?') // '<?...'
                {
                    s = parse_question(s, cursor, optmsk, endch);
                    if (!s)
                        return s;
 
                    assert(cursor);
                    if ((cursor->header & xml_memory_page_type_mask) + 1 == node_declaration)
                        goto LOC_ATTRIBUTES;
                }
                else if (*s == '!') // '<!...'
                {
                    s = parse_exclamation(s, cursor, optmsk, endch);
                    if (!s)
                        return s;
                }
                else if (*s == 0 && endch == '?')
                    PUGI__THROW_ERROR(status_bad_pi, s);
                else
                    PUGI__THROW_ERROR(status_unrecognized_tag, s);
            }
            else
            {
                mark = s; // Save this offset while searching for a terminator.
 
                PUGI__SKIPWS(); // Eat whitespace if no genuine PCDATA here.
 
                if (*s == '<' || !*s)
                {
                    // We skipped some whitespace characters because otherwise we would take the tag branch instead of
                    // PCDATA one
                    assert(mark != s);
 
                    if (!PUGI__OPTSET(parse_ws_pcdata | parse_ws_pcdata_single) || PUGI__OPTSET(parse_trim_pcdata))
                    {
                        continue;
                    }
                    else if (PUGI__OPTSET(parse_ws_pcdata_single))
                    {
                        if (s[0] != '<' || s[1] != '/' || cursor->first_child)
                            continue;
                    }
                }
 
                if (!PUGI__OPTSET(parse_trim_pcdata))
                    s = mark;
 
                if (cursor->parent || PUGI__OPTSET(parse_fragment))
                {
                    PUGI__PUSHNODE(node_pcdata); // Append a new node on the tree.
                    cursor->value = s;           // Save the offset.
 
                    s = strconv_pcdata(s);
 
                    PUGI__POPNODE(); // Pop since this is a standalone.
 
                    if (!*s)
                        break;
                }
                else
                {
                    PUGI__SCANFOR(*s == '<'); // '...<'
                    if (!*s)
                        break;
 
                    ++s;
                }
 
                // We're after '<'
                goto LOC_TAG;
            }
        }
 
        // check that last tag is closed
        if (cursor != root)
            PUGI__THROW_ERROR(status_end_element_mismatch, s);
 
        return s;
    }
 
#ifdef PUGIXML_WCHAR_MODE
    static char_t *parse_skip_bom(char_t *s)
    {
        unsigned int bom = 0xfeff;
        return (s[0] == static_cast<wchar_t>(bom)) ? s + 1 : s;
    }
#else
    static char_t *parse_skip_bom(char_t *s)
    {
        return (s[0] == '\xef' && s[1] == '\xbb' && s[2] == '\xbf') ? s + 3 : s;
    }
#endif
 
    static bool has_element_node_siblings(xml_node_struct *node)
    {
        while (node)
        {
            xml_node_type type = static_cast<xml_node_type>((node->header & impl::xml_memory_page_type_mask) + 1);
            if (type == node_element)
                return true;
 
            node = node->next_sibling;
        }
 
        return false;
    }
 
    static xml_parse_result parse(char_t *buffer, size_t length, xml_document_struct *xmldoc, xml_node_struct *root,
                                  unsigned int optmsk)
    {
        // allocator object is a part of document object
        xml_allocator &alloc = *static_cast<xml_allocator *>(xmldoc);
 
        // early-out for empty documents
        if (length == 0)
            return make_parse_result(PUGI__OPTSET(parse_fragment) ? status_ok : status_no_document_element);
 
        // get last child of the root before parsing
        xml_node_struct *last_root_child = root->first_child ? root->first_child->prev_sibling_c : 0;
 
        // create parser on stack
        xml_parser parser(alloc);
 
        // save last character and make buffer zero-terminated (speeds up parsing)
        char_t endch = buffer[length - 1];
        buffer[length - 1] = 0;
 
        // skip BOM to make sure it does not end up as part of parse output
        char_t *buffer_data = parse_skip_bom(buffer);
 
        // perform actual parsing
        parser.parse_tree(buffer_data, root, optmsk, endch);
 
        // update allocator state
        alloc = parser.alloc;
 
        xml_parse_result result =
            make_parse_result(parser.error_status, parser.error_offset ? parser.error_offset - buffer : 0);
        assert(result.offset >= 0 && static_cast<size_t>(result.offset) <= length);
 
        if (result)
        {
            // since we removed last character, we have to handle the only possible false positive (stray <)
            if (endch == '<')
                return make_parse_result(status_unrecognized_tag, length - 1);
 
            // check if there are any element nodes parsed
            xml_node_struct *first_root_child_parsed =
                last_root_child ? last_root_child->next_sibling : root->first_child;
 
            if (!PUGI__OPTSET(parse_fragment) && !has_element_node_siblings(first_root_child_parsed))
                return make_parse_result(status_no_document_element, length - 1);
        }
        else
        {
            // roll back offset if it occurs on a null terminator in the source buffer
            if (result.offset > 0 && static_cast<size_t>(result.offset) == length - 1 && endch == 0)
                result.offset--;
        }
 
        return result;
    }
};
 
// Output facilities
PUGI__FN xml_encoding get_write_native_encoding()
{
#ifdef PUGIXML_WCHAR_MODE
    return get_wchar_encoding();
#else
    return encoding_utf8;
#endif
}
 
PUGI__FN xml_encoding get_write_encoding(xml_encoding encoding)
{
    // replace wchar encoding with utf implementation
    if (encoding == encoding_wchar)
        return get_wchar_encoding();
 
    // replace utf16 encoding with utf16 with specific endianness
    if (encoding == encoding_utf16)
        return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
    // replace utf32 encoding with utf32 with specific endianness
    if (encoding == encoding_utf32)
        return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
    // only do autodetection if no explicit encoding is requested
    if (encoding != encoding_auto)
        return encoding;
 
    // assume utf8 encoding
    return encoding_utf8;
}
 
#ifdef PUGIXML_WCHAR_MODE
PUGI__FN size_t get_valid_length(const char_t *data, size_t length)
{
    assert(length > 0);
 
    // discard last character if it's the lead of a surrogate pair
    return (sizeof(wchar_t) == 2 && static_cast<unsigned int>(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400)
               ? length - 1
               : length;
}
 
PUGI__FN size_t convert_buffer_output(char_t *r_char, uint8_t *r_u8, uint16_t *r_u16, uint32_t *r_u32,
                                      const char_t *data, size_t length, xml_encoding encoding)
{
    // only endian-swapping is required
    if (need_endian_swap_utf(encoding, get_wchar_encoding()))
    {
        convert_wchar_endian_swap(r_char, data, length);
 
        return length * sizeof(char_t);
    }
 
    // convert to utf8
    if (encoding == encoding_utf8)
    {
        uint8_t *dest = r_u8;
        uint8_t *end = utf_decoder<utf8_writer>::decode_wchar_block(data, length, dest);
 
        return static_cast<size_t>(end - dest);
    }
 
    // convert to utf16
    if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
    {
        uint16_t *dest = r_u16;
 
        // convert to native utf16
        uint16_t *end = utf_decoder<utf16_writer>::decode_wchar_block(data, length, dest);
 
        // swap if necessary
        xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
        if (native_encoding != encoding)
            convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
 
        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
    }
 
    // convert to utf32
    if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
    {
        uint32_t *dest = r_u32;
 
        // convert to native utf32
        uint32_t *end = utf_decoder<utf32_writer>::decode_wchar_block(data, length, dest);
 
        // swap if necessary
        xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
        if (native_encoding != encoding)
            convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
 
        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
    }
 
    // convert to latin1
    if (encoding == encoding_latin1)
    {
        uint8_t *dest = r_u8;
        uint8_t *end = utf_decoder<latin1_writer>::decode_wchar_block(data, length, dest);
 
        return static_cast<size_t>(end - dest);
    }
 
    assert(!"Invalid encoding");
    return 0;
}
#else
PUGI__FN size_t get_valid_length(const char_t *data, size_t length)
{
    assert(length > 4);
 
    for (size_t i = 1; i <= 4; ++i)
    {
        uint8_t ch = static_cast<uint8_t>(data[length - i]);
 
        // either a standalone character or a leading one
        if ((ch & 0xc0) != 0x80)
            return length - i;
    }
 
    // there are four non-leading characters at the end, sequence tail is broken so might as well process the whole
    // chunk
    return length;
}
 
PUGI__FN size_t convert_buffer_output(char_t * /* r_char */, uint8_t *r_u8, uint16_t *r_u16, uint32_t *r_u32,
                                      const char_t *data, size_t length, xml_encoding encoding)
{
    if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
    {
        uint16_t *dest = r_u16;
 
        // convert to native utf16
        uint16_t *end =
            utf_decoder<utf16_writer>::decode_utf8_block(reinterpret_cast<const uint8_t *>(data), length, dest);
 
        // swap if necessary
        xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
        if (native_encoding != encoding)
            convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
 
        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
    }
 
    if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
    {
        uint32_t *dest = r_u32;
 
        // convert to native utf32
        uint32_t *end =
            utf_decoder<utf32_writer>::decode_utf8_block(reinterpret_cast<const uint8_t *>(data), length, dest);
 
        // swap if necessary
        xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
        if (native_encoding != encoding)
            convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
 
        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
    }
 
    if (encoding == encoding_latin1)
    {
        uint8_t *dest = r_u8;
        uint8_t *end =
            utf_decoder<latin1_writer>::decode_utf8_block(reinterpret_cast<const uint8_t *>(data), length, dest);
 
        return static_cast<size_t>(end - dest);
    }
 
    assert(!"Invalid encoding");
    return 0;
}
#endif
 
class xml_buffered_writer
{
    xml_buffered_writer(const xml_buffered_writer &);
    xml_buffered_writer &operator=(const xml_buffered_writer &);
 
  public:
    xml_buffered_writer(xml_writer &writer_, xml_encoding user_encoding)
        : writer(writer_), bufsize(0), encoding(get_write_encoding(user_encoding))
    {
        PUGI__STATIC_ASSERT(bufcapacity >= 8);
    }
 
    ~xml_buffered_writer() { flush(); }
 
    void flush()
    {
        flush(buffer, bufsize);
        bufsize = 0;
    }
 
    void flush(const char_t *data, size_t size)
    {
        if (size == 0)
            return;
 
        // fast path, just write data
        if (encoding == get_write_native_encoding())
            writer.write(data, size * sizeof(char_t));
        else
        {
            // convert chunk
            size_t result = convert_buffer_output(scratch.data_char, scratch.data_u8, scratch.data_u16,
                                                  scratch.data_u32, data, size, encoding);
            assert(result <= sizeof(scratch));
 
            // write data
            writer.write(scratch.data_u8, result);
        }
    }
 
    void write(const char_t *data, size_t length)
    {
        if (bufsize + length > bufcapacity)
        {
            // flush the remaining buffer contents
            flush();
 
            // handle large chunks
            if (length > bufcapacity)
            {
                if (encoding == get_write_native_encoding())
                {
                    // fast path, can just write data chunk
                    writer.write(data, length * sizeof(char_t));
                    return;
                }
 
                // need to convert in suitable chunks
                while (length > bufcapacity)
                {
                    // get chunk size by selecting such number of characters that are guaranteed to fit into scratch
                    // buffer and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
                    size_t chunk_size = get_valid_length(data, bufcapacity);
 
                    // convert chunk and write
                    flush(data, chunk_size);
 
                    // iterate
                    data += chunk_size;
                    length -= chunk_size;
                }
 
                // small tail is copied below
                bufsize = 0;
            }
        }
 
        memcpy(buffer + bufsize, data, length * sizeof(char_t));
        bufsize += length;
    }
 
    void write(const char_t *data) { write(data, strlength(data)); }
 
    void write(char_t d0)
    {
        if (bufsize + 1 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        bufsize += 1;
    }
 
    void write(char_t d0, char_t d1)
    {
        if (bufsize + 2 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        buffer[bufsize + 1] = d1;
        bufsize += 2;
    }
 
    void write(char_t d0, char_t d1, char_t d2)
    {
        if (bufsize + 3 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        buffer[bufsize + 1] = d1;
        buffer[bufsize + 2] = d2;
        bufsize += 3;
    }
 
    void write(char_t d0, char_t d1, char_t d2, char_t d3)
    {
        if (bufsize + 4 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        buffer[bufsize + 1] = d1;
        buffer[bufsize + 2] = d2;
        buffer[bufsize + 3] = d3;
        bufsize += 4;
    }
 
    void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4)
    {
        if (bufsize + 5 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        buffer[bufsize + 1] = d1;
        buffer[bufsize + 2] = d2;
        buffer[bufsize + 3] = d3;
        buffer[bufsize + 4] = d4;
        bufsize += 5;
    }
 
    void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4, char_t d5)
    {
        if (bufsize + 6 > bufcapacity)
            flush();
 
        buffer[bufsize + 0] = d0;
        buffer[bufsize + 1] = d1;
        buffer[bufsize + 2] = d2;
        buffer[bufsize + 3] = d3;
        buffer[bufsize + 4] = d4;
        buffer[bufsize + 5] = d5;
        bufsize += 6;
    }
 
    // utf8 maximum expansion: x4 (-> utf32)
    // utf16 maximum expansion: x2 (-> utf32)
    // utf32 maximum expansion: x1
    enum
    {
        bufcapacitybytes =
#ifdef PUGIXML_MEMORY_OUTPUT_STACK
            PUGIXML_MEMORY_OUTPUT_STACK
#else
            10240
#endif
        ,
        bufcapacity = bufcapacitybytes / (sizeof(char_t) + 4)
    };
 
    char_t buffer[bufcapacity];
 
    union {
        uint8_t data_u8[4 * bufcapacity];
        uint16_t data_u16[2 * bufcapacity];
        uint32_t data_u32[bufcapacity];
        char_t data_char[bufcapacity];
    } scratch;
 
    xml_writer &writer;
    size_t bufsize;
    xml_encoding encoding;
};
 
PUGI__FN void text_output_escaped(xml_buffered_writer &writer, const char_t *s, chartypex_t type)
{
    while (*s)
    {
        const char_t *prev = s;
 
        // While *s is a usual symbol
        while (!PUGI__IS_CHARTYPEX(*s, type))
            ++s;
 
        writer.write(prev, static_cast<size_t>(s - prev));
 
        switch (*s)
        {
        case 0:
            break;
        case '&':
            writer.write('&', 'a', 'm', 'p', ';');
            ++s;
            break;
        case '<':
            writer.write('&', 'l', 't', ';');
            ++s;
            break;
        case '>':
            writer.write('&', 'g', 't', ';');
            ++s;
            break;
        case '"':
            writer.write('&', 'q', 'u', 'o', 't', ';');
            ++s;
            break;
        default: // s is not a usual symbol
        {
            unsigned int ch = static_cast<unsigned int>(*s++);
            assert(ch < 32);
 
            writer.write('&', '#', static_cast<char_t>((ch / 10) + '0'), static_cast<char_t>((ch % 10) + '0'), ';');
        }
        }
    }
}
 
PUGI__FN void text_output(xml_buffered_writer &writer, const char_t *s, chartypex_t type, unsigned int flags)
{
    if (flags & format_no_escapes)
        writer.write(s);
    else
        text_output_escaped(writer, s, type);
}
 
PUGI__FN void text_output_cdata(xml_buffered_writer &writer, const char_t *s)
{
    do
    {
        writer.write('<', '!', '[', 'C', 'D');
        writer.write('A', 'T', 'A', '[');
 
        const char_t *prev = s;
 
        // look for ]]> sequence - we can't output it as is since it terminates CDATA
        while (*s && !(s[0] == ']' && s[1] == ']' && s[2] == '>'))
            ++s;
 
        // skip ]] if we stopped at ]]>, > will go to the next CDATA section
        if (*s)
            s += 2;
 
        writer.write(prev, static_cast<size_t>(s - prev));
 
        writer.write(']', ']', '>');
    } while (*s);
}
 
PUGI__FN void node_output_attributes(xml_buffered_writer &writer, const xml_node &node, unsigned int flags)
{
    const char_t *default_name = PUGIXML_TEXT(":anonymous");
 
    for (xml_attribute a = node.first_attribute(); a; a = a.next_attribute())
    {
        writer.write(' ');
        writer.write(a.name()[0] ? a.name() : default_name);
        writer.write('=', '"');
 
        text_output(writer, a.value(), ctx_special_attr, flags);
 
        writer.write('"');
    }
}
 
PUGI__FN void node_output(xml_buffered_writer &writer, const xml_node &node, const char_t *indent, unsigned int flags,
                          unsigned int depth)
{
    const char_t *default_name = PUGIXML_TEXT(":anonymous");
 
    if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
        for (unsigned int i = 0; i < depth; ++i)
            writer.write(indent);
 
    switch (node.type())
    {
    case node_document:
    {
        for (xml_node n = node.first_child(); n; n = n.next_sibling())
            node_output(writer, n, indent, flags, depth);
        break;
    }
 
    case node_element:
    {
        const char_t *name = node.name()[0] ? node.name() : default_name;
 
        writer.write('<');
        writer.write(name);
 
        node_output_attributes(writer, node, flags);
 
        if (flags & format_raw)
        {
            if (!node.first_child())
                writer.write(' ', '/', '>');
            else
            {
                writer.write('>');
 
                for (xml_node n = node.first_child(); n; n = n.next_sibling())
                    node_output(writer, n, indent, flags, depth + 1);
 
                writer.write('<', '/');
                writer.write(name);
                writer.write('>');
            }
        }
        else if (!node.first_child())
            writer.write(' ', '/', '>', '\n');
        else if (node.first_child() == node.last_child() &&
                 (node.first_child().type() == node_pcdata || node.first_child().type() == node_cdata))
        {
            writer.write('>');
 
            if (node.first_child().type() == node_pcdata)
                text_output(writer, node.first_child().value(), ctx_special_pcdata, flags);
            else
                text_output_cdata(writer, node.first_child().value());
 
            writer.write('<', '/');
            writer.write(name);
            writer.write('>', '\n');
        }
        else
        {
            writer.write('>', '\n');
 
            for (xml_node n = node.first_child(); n; n = n.next_sibling())
                node_output(writer, n, indent, flags, depth + 1);
 
            if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
                for (unsigned int i = 0; i < depth; ++i)
                    writer.write(indent);
 
            writer.write('<', '/');
            writer.write(name);
            writer.write('>', '\n');
        }
 
        break;
    }
 
    case node_pcdata:
        text_output(writer, node.value(), ctx_special_pcdata, flags);
        if ((flags & format_raw) == 0)
            writer.write('\n');
        break;
 
    case node_cdata:
        text_output_cdata(writer, node.value());
        if ((flags & format_raw) == 0)
            writer.write('\n');
        break;
 
    case node_comment:
        writer.write('<', '!', '-', '-');
        writer.write(node.value());
        writer.write('-', '-', '>');
        if ((flags & format_raw) == 0)
            writer.write('\n');
        break;
 
    case node_pi:
    case node_declaration:
        writer.write('<', '?');
        writer.write(node.name()[0] ? node.name() : default_name);
 
        if (node.type() == node_declaration)
        {
            node_output_attributes(writer, node, flags);
        }
        else if (node.value()[0])
        {
            writer.write(' ');
            writer.write(node.value());
        }
 
        writer.write('?', '>');
        if ((flags & format_raw) == 0)
            writer.write('\n');
        break;
 
    case node_doctype:
        writer.write('<', '!', 'D', 'O', 'C');
        writer.write('T', 'Y', 'P', 'E');
 
        if (node.value()[0])
        {
            writer.write(' ');
            writer.write(node.value());
        }
 
        writer.write('>');
        if ((flags & format_raw) == 0)
            writer.write('\n');
        break;
 
    default:
        assert(!"Invalid node type");
    }
}
 
inline bool has_declaration(const xml_node &node)
{
    for (xml_node child = node.first_child(); child; child = child.next_sibling())
    {
        xml_node_type type = child.type();
 
        if (type == node_declaration)
            return true;
        if (type == node_element)
            return false;
    }
 
    return false;
}
 
inline bool allow_insert_child(xml_node_type parent, xml_node_type child)
{
    if (parent != node_document && parent != node_element)
        return false;
    if (child == node_document || child == node_null)
        return false;
    if (parent != node_document && (child == node_declaration || child == node_doctype))
        return false;
 
    return true;
}
 
PUGI__FN void recursive_copy_skip(xml_node &dest, const xml_node &source, const xml_node &skip)
{
    assert(dest.type() == source.type());
 
    switch (source.type())
    {
    case node_element:
    {
        dest.set_name(source.name());
 
        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
            dest.append_attribute(a.name()).set_value(a.value());
 
        for (xml_node c = source.first_child(); c; c = c.next_sibling())
        {
            if (c == skip)
                continue;
 
            xml_node cc = dest.append_child(c.type());
            assert(cc);
 
            recursive_copy_skip(cc, c, skip);
        }
 
        break;
    }
 
    case node_pcdata:
    case node_cdata:
    case node_comment:
    case node_doctype:
        dest.set_value(source.value());
        break;
 
    case node_pi:
        dest.set_name(source.name());
        dest.set_value(source.value());
        break;
 
    case node_declaration:
    {
        dest.set_name(source.name());
 
        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
            dest.append_attribute(a.name()).set_value(a.value());
 
        break;
    }
 
    default:
        assert(!"Invalid node type");
    }
}
 
inline bool is_text_node(xml_node_struct *node)
{
    xml_node_type type = static_cast<xml_node_type>((node->header & impl::xml_memory_page_type_mask) + 1);
 
    return type == node_pcdata || type == node_cdata;
}
 
// get value with conversion functions
PUGI__FN int get_integer_base(const char_t *value)
{
    const char_t *s = value;
 
    while (PUGI__IS_CHARTYPE(*s, ct_space))
        s++;
 
    if (*s == '-')
        s++;
 
    return (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) ? 16 : 10;
}
 
PUGI__FN int get_value_int(const char_t *value, int def)
{
    if (!value)
        return def;
 
    int base = get_integer_base(value);
 
#ifdef PUGIXML_WCHAR_MODE
    return static_cast<int>(wcstol(value, 0, base));
#else
    return static_cast<int>(strtol(value, 0, base));
#endif
}
 
PUGI__FN unsigned int get_value_uint(const char_t *value, unsigned int def)
{
    if (!value)
        return def;
 
    int base = get_integer_base(value);
 
#ifdef PUGIXML_WCHAR_MODE
    return static_cast<unsigned int>(wcstoul(value, 0, base));
#else
    return static_cast<unsigned int>(strtoul(value, 0, base));
#endif
}
 
PUGI__FN double get_value_double(const char_t *value, double def)
{
    if (!value)
        return def;
 
#ifdef PUGIXML_WCHAR_MODE
    return wcstod(value, 0);
#else
    return strtod(value, 0);
#endif
}
 
PUGI__FN float get_value_float(const char_t *value, float def)
{
    if (!value)
        return def;
 
#ifdef PUGIXML_WCHAR_MODE
    return static_cast<float>(wcstod(value, 0));
#else
    return static_cast<float>(strtod(value, 0));
#endif
}
 
PUGI__FN bool get_value_bool(const char_t *value, bool def)
{
    if (!value)
        return def;
 
    // only look at first char
    char_t first = *value;
 
    // 1*, t* (true), T* (True), y* (yes), Y* (YES)
    return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN long long get_value_llong(const char_t *value, long long def)
{
    if (!value)
        return def;
 
    int base = get_integer_base(value);
 
#ifdef PUGIXML_WCHAR_MODE
#ifdef PUGI__MSVC_CRT_VERSION
    return _wcstoi64(value, 0, base);
#else
    return wcstoll(value, 0, base);
#endif
#else
#ifdef PUGI__MSVC_CRT_VERSION
    return _strtoi64(value, 0, base);
#else
    return strtoll(value, 0, base);
#endif
#endif
}
 
PUGI__FN unsigned long long get_value_ullong(const char_t *value, unsigned long long def)
{
    if (!value)
        return def;
 
    int base = get_integer_base(value);
 
#ifdef PUGIXML_WCHAR_MODE
#ifdef PUGI__MSVC_CRT_VERSION
    return _wcstoui64(value, 0, base);
#else
    return wcstoull(value, 0, base);
#endif
#else
#ifdef PUGI__MSVC_CRT_VERSION
    return _strtoui64(value, 0, base);
#else
    return strtoull(value, 0, base);
#endif
#endif
}
#endif
 
// set value with conversion functions
PUGI__FN bool set_value_buffer(char_t *&dest, uintptr_t &header, uintptr_t header_mask, char (&buf)[128])
{
#ifdef PUGIXML_WCHAR_MODE
    char_t wbuf[128];
    impl::widen_ascii(wbuf, buf);
 
    return strcpy_insitu(dest, header, header_mask, wbuf);
#else
    return strcpy_insitu(dest, header, header_mask, buf);
#endif
}
 
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, int value)
{
    char buf[128];
    sprintf(buf, "%d", value);
 
    return set_value_buffer(dest, header, header_mask, buf);
}
 
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, unsigned int value)
{
    char buf[128];
    sprintf(buf, "%u", value);
 
    return set_value_buffer(dest, header, header_mask, buf);
}
 
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, double value)
{
    char buf[128];
    sprintf(buf, "%g", value);
 
    return set_value_buffer(dest, header, header_mask, buf);
}
 
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, bool value)
{
    return strcpy_insitu(dest, header, header_mask, value ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, long long value)
{
    char buf[128];
    sprintf(buf, "%lld", value);
 
    return set_value_buffer(dest, header, header_mask, buf);
}
 
PUGI__FN bool set_value_convert(char_t *&dest, uintptr_t &header, uintptr_t header_mask, unsigned long long value)
{
    char buf[128];
    sprintf(buf, "%llu", value);
 
    return set_value_buffer(dest, header, header_mask, buf);
}
#endif
 
// we need to get length of entire file to load it in memory; the only (relatively) sane way to do it is via seek/tell
// trick
PUGI__FN xml_parse_status get_file_size(FILE *file, size_t &out_result)
{
#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
    // there are 64-bit versions of fseek/ftell, let's use them
    typedef __int64 length_type;
 
    _fseeki64(file, 0, SEEK_END);
    length_type length = _ftelli64(file);
    _fseeki64(file, 0, SEEK_SET);
#elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && !defined(__STRICT_ANSI__)
    // there are 64-bit versions of fseek/ftell, let's use them
    typedef off64_t length_type;
 
    fseeko64(file, 0, SEEK_END);
    length_type length = ftello64(file);
    fseeko64(file, 0, SEEK_SET);
#else
    // if this is a 32-bit OS, long is enough; if this is a unix system, long is 64-bit, which is enough; otherwise we
    // can't do anything anyway.
    typedef long length_type;
 
    fseek(file, 0, SEEK_END);
    length_type length = ftell(file);
    fseek(file, 0, SEEK_SET);
#endif
 
    // check for I/O errors
    if (length < 0)
        return status_io_error;
 
    // check for overflow
    size_t result = static_cast<size_t>(length);
 
    if (static_cast<length_type>(result) != length)
        return status_out_of_memory;
 
    // finalize
    out_result = result;
 
    return status_ok;
}
 
PUGI__FN size_t zero_terminate_buffer(void *buffer, size_t size, xml_encoding encoding)
{
    // We only need to zero-terminate if encoding conversion does not do it for us
#ifdef PUGIXML_WCHAR_MODE
    xml_encoding wchar_encoding = get_wchar_encoding();
 
    if (encoding == wchar_encoding || need_endian_swap_utf(encoding, wchar_encoding))
    {
        size_t length = size / sizeof(char_t);
 
        static_cast<char_t *>(buffer)[length] = 0;
        return (length + 1) * sizeof(char_t);
    }
#else
    if (encoding == encoding_utf8)
    {
        static_cast<char *>(buffer)[size] = 0;
        return size + 1;
    }
#endif
 
    return size;
}
 
PUGI__FN xml_parse_result load_file_impl(xml_document &doc, FILE *file, unsigned int options, xml_encoding encoding)
{
    if (!file)
        return make_parse_result(status_file_not_found);
 
    // get file size (can result in I/O errors)
    size_t size = 0;
    xml_parse_status size_status = get_file_size(file, size);
 
    if (size_status != status_ok)
    {
        fclose(file);
        return make_parse_result(size_status);
    }
 
    size_t max_suffix_size = sizeof(char_t);
 
    // allocate buffer for the whole file
    char *contents = static_cast<char *>(xml_memory::allocate(size + max_suffix_size));
 
    if (!contents)
    {
        fclose(file);
        return make_parse_result(status_out_of_memory);
    }
 
    // read file in memory
    size_t read_size = fread(contents, 1, size, file);
    fclose(file);
 
    if (read_size != size)
    {
        xml_memory::deallocate(contents);
        return make_parse_result(status_io_error);
    }
 
    xml_encoding real_encoding = get_buffer_encoding(encoding, contents, size);
 
    return doc.load_buffer_inplace_own(contents, zero_terminate_buffer(contents, size, real_encoding), options,
                                       real_encoding);
}
 
#ifndef PUGIXML_NO_STL
template <typename T>
struct xml_stream_chunk
{
    static xml_stream_chunk *create()
    {
        void *memory = xml_memory::allocate(sizeof(xml_stream_chunk));
 
        return new (memory) xml_stream_chunk();
    }
 
    static void destroy(void *ptr)
    {
        xml_stream_chunk *chunk = static_cast<xml_stream_chunk *>(ptr);
 
        // free chunk chain
        while (chunk)
        {
            xml_stream_chunk *next = chunk->next;
            xml_memory::deallocate(chunk);
            chunk = next;
        }
    }
 
    xml_stream_chunk() : next(0), size(0) {}
 
    xml_stream_chunk *next;
    size_t size;
 
    T data[xml_memory_page_size / sizeof(T)];
};
 
template <typename T>
PUGI__FN xml_parse_status load_stream_data_noseek(std::basic_istream<T> &stream, void **out_buffer, size_t *out_size)
{
    buffer_holder chunks(0, xml_stream_chunk<T>::destroy);
 
    // read file to a chunk list
    size_t total = 0;
    xml_stream_chunk<T> *last = 0;
 
    while (!stream.eof())
    {
        // allocate new chunk
        xml_stream_chunk<T> *chunk = xml_stream_chunk<T>::create();
        if (!chunk)
            return status_out_of_memory;
 
        // append chunk to list
        if (last)
            last = last->next = chunk;
        else
            chunks.data = last = chunk;
 
        // read data to chunk
        stream.read(chunk->data, static_cast<std::streamsize>(sizeof(chunk->data) / sizeof(T)));
        chunk->size = static_cast<size_t>(stream.gcount()) * sizeof(T);
 
        // read may set failbit | eofbit in case gcount() is less than read length, so check for other I/O errors
        if (stream.bad() || (!stream.eof() && stream.fail()))
            return status_io_error;
 
        // guard against huge files (chunk size is small enough to make this overflow check work)
        if (total + chunk->size < total)
            return status_out_of_memory;
        total += chunk->size;
    }
 
    size_t max_suffix_size = sizeof(char_t);
 
    // copy chunk list to a contiguous buffer
    char *buffer = static_cast<char *>(xml_memory::allocate(total + max_suffix_size));
    if (!buffer)
        return status_out_of_memory;
 
    char *write = buffer;
 
    for (xml_stream_chunk<T> *chunk = static_cast<xml_stream_chunk<T> *>(chunks.data); chunk; chunk = chunk->next)
    {
        assert(write + chunk->size <= buffer + total);
        memcpy(write, chunk->data, chunk->size);
        write += chunk->size;
    }
 
    assert(write == buffer + total);
 
    // return buffer
    *out_buffer = buffer;
    *out_size = total;
 
    return status_ok;
}
 
template <typename T>
PUGI__FN xml_parse_status load_stream_data_seek(std::basic_istream<T> &stream, void **out_buffer, size_t *out_size)
{
    // get length of remaining data in stream
    typename std::basic_istream<T>::pos_type pos = stream.tellg();
    stream.seekg(0, std::ios::end);
    std::streamoff length = stream.tellg() - pos;
    stream.seekg(pos);
 
    if (stream.fail() || pos < 0)
        return status_io_error;
 
    // guard against huge files
    size_t read_length = static_cast<size_t>(length);
 
    if (static_cast<std::streamsize>(read_length) != length || length < 0)
        return status_out_of_memory;
 
    size_t max_suffix_size = sizeof(char_t);
 
    // read stream data into memory (guard against stream exceptions with buffer holder)
    buffer_holder buffer(xml_memory::allocate(read_length * sizeof(T) + max_suffix_size), xml_memory::deallocate);
    if (!buffer.data)
        return status_out_of_memory;
 
    stream.read(static_cast<T *>(buffer.data), static_cast<std::streamsize>(read_length));
 
    // read may set failbit | eofbit in case gcount() is less than read_length (i.e. line ending conversion), so check
    // for other I/O errors
    if (stream.bad() || (!stream.eof() && stream.fail()))
        return status_io_error;
 
    // return buffer
    size_t actual_length = static_cast<size_t>(stream.gcount());
    assert(actual_length <= read_length);
 
    *out_buffer = buffer.release();
    *out_size = actual_length * sizeof(T);
 
    return status_ok;
}
 
template <typename T>
PUGI__FN xml_parse_result load_stream_impl(xml_document &doc, std::basic_istream<T> &stream, unsigned int options,
                                           xml_encoding encoding)
{
    void *buffer = 0;
    size_t size = 0;
    xml_parse_status status = status_ok;
 
    // if stream has an error bit set, bail out (otherwise tellg() can fail and we'll clear error bits)
    if (stream.fail())
        return make_parse_result(status_io_error);
 
    // load stream to memory (using seek-based implementation if possible, since it's faster and takes less memory)
    if (stream.tellg() < 0)
    {
        stream.clear(); // clear error flags that could be set by a failing tellg
        status = load_stream_data_noseek(stream, &buffer, &size);
    }
    else
        status = load_stream_data_seek(stream, &buffer, &size);
 
    if (status != status_ok)
        return make_parse_result(status);
 
    xml_encoding real_encoding = get_buffer_encoding(encoding, buffer, size);
 
    return doc.load_buffer_inplace_own(buffer, zero_terminate_buffer(buffer, size, real_encoding), options,
                                       real_encoding);
}
#endif
 
#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__) || (defined(__MINGW32__) && !defined(__STRICT_ANSI__))
PUGI__FN FILE *open_file_wide(const wchar_t *path, const wchar_t *mode)
{
    return _wfopen(path, mode);
}
#else
PUGI__FN char *convert_path_heap(const wchar_t *str)
{
    assert(str);
 
    // first pass: get length in utf8 characters
    size_t length = strlength_wide(str);
    size_t size = as_utf8_begin(str, length);
 
    // allocate resulting string
    char *result = static_cast<char *>(xml_memory::allocate(size + 1));
    if (!result)
        return 0;
 
    // second pass: convert to utf8
    as_utf8_end(result, size, str, length);
 
    return result;
}
 
PUGI__FN FILE *open_file_wide(const wchar_t *path, const wchar_t *mode)
{
    // there is no standard function to open wide paths, so our best bet is to try utf8 path
    char *path_utf8 = convert_path_heap(path);
    if (!path_utf8)
        return 0;
 
    // convert mode to ASCII (we mirror _wfopen interface)
    char mode_ascii[4] = { 0 };
    for (size_t i = 0; mode[i]; ++i)
        mode_ascii[i] = static_cast<char>(mode[i]);
 
    // try to open the utf8 path
    FILE *result = fopen(path_utf8, mode_ascii);
 
    // free dummy buffer
    xml_memory::deallocate(path_utf8);
 
    return result;
}
#endif
 
PUGI__FN bool save_file_impl(const xml_document &doc, FILE *file, const char_t *indent, unsigned int flags,
                             xml_encoding encoding)
{
    if (!file)
        return false;
 
    xml_writer_file writer(file);
    doc.save(writer, indent, flags, encoding);
 
    int result = ferror(file);
 
    fclose(file);
 
    return result == 0;
}
 
PUGI__FN xml_parse_result load_buffer_impl(xml_document_struct *doc, xml_node_struct *root, void *contents, size_t size,
                                           unsigned int options, xml_encoding encoding, bool is_mutable, bool own,
                                           char_t **out_buffer)
{
    // check input buffer
    assert(contents || size == 0);
 
    // get actual encoding
    xml_encoding buffer_encoding = impl::get_buffer_encoding(encoding, contents, size);
 
    // get private buffer
    char_t *buffer = 0;
    size_t length = 0;
 
    if (!impl::convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable))
        return impl::make_parse_result(status_out_of_memory);
 
    // delete original buffer if we performed a conversion
    if (own && buffer != contents && contents)
        impl::xml_memory::deallocate(contents);
 
    // store buffer for offset_debug
    doc->buffer = buffer;
 
    // parse
    xml_parse_result res = impl::xml_parser::parse(buffer, length, doc, root, options);
 
    // remember encoding
    res.encoding = buffer_encoding;
 
    // grab onto buffer if it's our buffer, user is responsible for deallocating contents himself
    if (own || buffer != contents)
        *out_buffer = buffer;
 
    return res;
}
PUGI__NS_END
 
namespace pugi
{
PUGI__FN xml_writer_file::xml_writer_file(void *file_) : file(file_) {}
 
PUGI__FN void xml_writer_file::write(const void *data, size_t size)
{
    size_t result = fwrite(data, 1, size, static_cast<FILE *>(file));
    (void)!result; // unfortunately we can't do proper error handling here
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char>> &stream)
    : narrow_stream(&stream), wide_stream(0)
{
}
 
PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t>> &stream)
    : narrow_stream(0), wide_stream(&stream)
{
}
 
PUGI__FN void xml_writer_stream::write(const void *data, size_t size)
{
    if (narrow_stream)
    {
        assert(!wide_stream);
        narrow_stream->write(reinterpret_cast<const char *>(data), static_cast<std::streamsize>(size));
    }
    else
    {
        assert(wide_stream);
        assert(size % sizeof(wchar_t) == 0);
 
        wide_stream->write(reinterpret_cast<const wchar_t *>(data),
                           static_cast<std::streamsize>(size / sizeof(wchar_t)));
    }
}
#endif
 
PUGI__FN xml_tree_walker::xml_tree_walker() : _depth(0) {}
 
PUGI__FN xml_tree_walker::~xml_tree_walker() {}
 
PUGI__FN int xml_tree_walker::depth() const
{
    return _depth;
}
 
PUGI__FN bool xml_tree_walker::begin(xml_node &)
{
    return true;
}
 
PUGI__FN bool xml_tree_walker::end(xml_node &)
{
    return true;
}
 
PUGI__FN xml_attribute::xml_attribute() : _attr(0) {}
 
PUGI__FN xml_attribute::xml_attribute(xml_attribute_struct *attr) : _attr(attr) {}
 
PUGI__FN static void unspecified_bool_xml_attribute(xml_attribute ***) {}
 
PUGI__FN xml_attribute::operator xml_attribute::unspecified_bool_type() const
{
    return _attr ? unspecified_bool_xml_attribute : 0;
}
 
PUGI__FN bool xml_attribute::operator!() const
{
    return !_attr;
}
 
PUGI__FN bool xml_attribute::operator==(const xml_attribute &r) const
{
    return (_attr == r._attr);
}
 
PUGI__FN bool xml_attribute::operator!=(const xml_attribute &r) const
{
    return (_attr != r._attr);
}
 
PUGI__FN bool xml_attribute::operator<(const xml_attribute &r) const
{
    return (_attr < r._attr);
}
 
PUGI__FN bool xml_attribute::operator>(const xml_attribute &r) const
{
    return (_attr > r._attr);
}
 
PUGI__FN bool xml_attribute::operator<=(const xml_attribute &r) const
{
    return (_attr <= r._attr);
}
 
PUGI__FN bool xml_attribute::operator>=(const xml_attribute &r) const
{
    return (_attr >= r._attr);
}
 
PUGI__FN xml_attribute xml_attribute::next_attribute() const
{
    return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
}
 
PUGI__FN xml_attribute xml_attribute::previous_attribute() const
{
    return _attr && _attr->prev_attribute_c->next_attribute ? xml_attribute(_attr->prev_attribute_c) : xml_attribute();
}
 
PUGI__FN const char_t *xml_attribute::as_string(const char_t *def) const
{
    return (_attr && _attr->value) ? _attr->value : def;
}
 
PUGI__FN int xml_attribute::as_int(int def) const
{
    return impl::get_value_int(_attr ? _attr->value : 0, def);
}
 
PUGI__FN unsigned int xml_attribute::as_uint(unsigned int def) const
{
    return impl::get_value_uint(_attr ? _attr->value : 0, def);
}
 
PUGI__FN double xml_attribute::as_double(double def) const
{
    return impl::get_value_double(_attr ? _attr->value : 0, def);
}
 
PUGI__FN float xml_attribute::as_float(float def) const
{
    return impl::get_value_float(_attr ? _attr->value : 0, def);
}
 
PUGI__FN bool xml_attribute::as_bool(bool def) const
{
    return impl::get_value_bool(_attr ? _attr->value : 0, def);
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN long long xml_attribute::as_llong(long long def) const
{
    return impl::get_value_llong(_attr ? _attr->value : 0, def);
}
 
PUGI__FN unsigned long long xml_attribute::as_ullong(unsigned long long def) const
{
    return impl::get_value_ullong(_attr ? _attr->value : 0, def);
}
#endif
 
PUGI__FN bool xml_attribute::empty() const
{
    return !_attr;
}
 
PUGI__FN const char_t *xml_attribute::name() const
{
    return (_attr && _attr->name) ? _attr->name : PUGIXML_TEXT("");
}
 
PUGI__FN const char_t *xml_attribute::value() const
{
    return (_attr && _attr->value) ? _attr->value : PUGIXML_TEXT("");
}
 
PUGI__FN size_t xml_attribute::hash_value() const
{
    return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
}
 
PUGI__FN xml_attribute_struct *xml_attribute::internal_object() const
{
    return _attr;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(const char_t *rhs)
{
    set_value(rhs);
    return *this;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(int rhs)
{
    set_value(rhs);
    return *this;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(unsigned int rhs)
{
    set_value(rhs);
    return *this;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(double rhs)
{
    set_value(rhs);
    return *this;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(bool rhs)
{
    set_value(rhs);
    return *this;
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN xml_attribute &xml_attribute::operator=(long long rhs)
{
    set_value(rhs);
    return *this;
}
 
PUGI__FN xml_attribute &xml_attribute::operator=(unsigned long long rhs)
{
    set_value(rhs);
    return *this;
}
#endif
 
PUGI__FN bool xml_attribute::set_name(const char_t *rhs)
{
    if (!_attr)
        return false;
 
    return impl::strcpy_insitu(_attr->name, _attr->header, impl::xml_memory_page_name_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(const char_t *rhs)
{
    if (!_attr)
        return false;
 
    return impl::strcpy_insitu(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(int rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(unsigned int rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(double rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(bool rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN bool xml_attribute::set_value(long long rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
 
PUGI__FN bool xml_attribute::set_value(unsigned long long rhs)
{
    if (!_attr)
        return false;
 
    return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
}
#endif
 
#ifdef __BORLANDC__
PUGI__FN bool operator&&(const xml_attribute &lhs, bool rhs)
{
    return (bool)lhs && rhs;
}
 
PUGI__FN bool operator||(const xml_attribute &lhs, bool rhs)
{
    return (bool)lhs || rhs;
}
#endif
 
PUGI__FN xml_node::xml_node() : _root(0) {}
 
PUGI__FN xml_node::xml_node(xml_node_struct *p) : _root(p) {}
 
PUGI__FN static void unspecified_bool_xml_node(xml_node ***) {}
 
PUGI__FN xml_node::operator xml_node::unspecified_bool_type() const
{
    return _root ? unspecified_bool_xml_node : 0;
}
 
PUGI__FN bool xml_node::operator!() const
{
    return !_root;
}
 
PUGI__FN xml_node::iterator xml_node::begin() const
{
    return iterator(_root ? _root->first_child : 0, _root);
}
 
PUGI__FN xml_node::iterator xml_node::end() const
{
    return iterator(0, _root);
}
 
PUGI__FN xml_node::attribute_iterator xml_node::attributes_begin() const
{
    return attribute_iterator(_root ? _root->first_attribute : 0, _root);
}
 
PUGI__FN xml_node::attribute_iterator xml_node::attributes_end() const
{
    return attribute_iterator(0, _root);
}
 
PUGI__FN xml_object_range<xml_node_iterator> xml_node::children() const
{
    return xml_object_range<xml_node_iterator>(begin(), end());
}
 
PUGI__FN xml_object_range<xml_named_node_iterator> xml_node::children(const char_t *name_) const
{
    return xml_object_range<xml_named_node_iterator>(xml_named_node_iterator(child(name_)._root, _root, name_),
                                                     xml_named_node_iterator(0, _root, name_));
}
 
PUGI__FN xml_object_range<xml_attribute_iterator> xml_node::attributes() const
{
    return xml_object_range<xml_attribute_iterator>(attributes_begin(), attributes_end());
}
 
PUGI__FN bool xml_node::operator==(const xml_node &r) const
{
    return (_root == r._root);
}
 
PUGI__FN bool xml_node::operator!=(const xml_node &r) const
{
    return (_root != r._root);
}
 
PUGI__FN bool xml_node::operator<(const xml_node &r) const
{
    return (_root < r._root);
}
 
PUGI__FN bool xml_node::operator>(const xml_node &r) const
{
    return (_root > r._root);
}
 
PUGI__FN bool xml_node::operator<=(const xml_node &r) const
{
    return (_root <= r._root);
}
 
PUGI__FN bool xml_node::operator>=(const xml_node &r) const
{
    return (_root >= r._root);
}
 
PUGI__FN bool xml_node::empty() const
{
    return !_root;
}
 
PUGI__FN const char_t *xml_node::name() const
{
    return (_root && _root->name) ? _root->name : PUGIXML_TEXT("");
}
 
PUGI__FN xml_node_type xml_node::type() const
{
    return _root ? static_cast<xml_node_type>((_root->header & impl::xml_memory_page_type_mask) + 1) : node_null;
}
 
PUGI__FN const char_t *xml_node::value() const
{
    return (_root && _root->value) ? _root->value : PUGIXML_TEXT("");
}
 
PUGI__FN xml_node xml_node::child(const char_t *name_) const
{
    if (!_root)
        return xml_node();
 
    for (xml_node_struct *i = _root->first_child; i; i = i->next_sibling)
        if (i->name && impl::strequal(name_, i->name))
            return xml_node(i);
 
    return xml_node();
}
 
PUGI__FN xml_attribute xml_node::attribute(const char_t *name_) const
{
    if (!_root)
        return xml_attribute();
 
    for (xml_attribute_struct *i = _root->first_attribute; i; i = i->next_attribute)
        if (i->name && impl::strequal(name_, i->name))
            return xml_attribute(i);
 
    return xml_attribute();
}
 
PUGI__FN xml_node xml_node::next_sibling(const char_t *name_) const
{
    if (!_root)
        return xml_node();
 
    for (xml_node_struct *i = _root->next_sibling; i; i = i->next_sibling)
        if (i->name && impl::strequal(name_, i->name))
            return xml_node(i);
 
    return xml_node();
}
 
PUGI__FN xml_node xml_node::next_sibling() const
{
    if (!_root)
        return xml_node();
 
    if (_root->next_sibling)
        return xml_node(_root->next_sibling);
    else
        return xml_node();
}
 
PUGI__FN xml_node xml_node::previous_sibling(const char_t *name_) const
{
    if (!_root)
        return xml_node();
 
    for (xml_node_struct *i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
        if (i->name && impl::strequal(name_, i->name))
            return xml_node(i);
 
    return xml_node();
}
 
PUGI__FN xml_node xml_node::previous_sibling() const
{
    if (!_root)
        return xml_node();
 
    if (_root->prev_sibling_c->next_sibling)
        return xml_node(_root->prev_sibling_c);
    else
        return xml_node();
}
 
PUGI__FN xml_node xml_node::parent() const
{
    return _root ? xml_node(_root->parent) : xml_node();
}
 
PUGI__FN xml_node xml_node::root() const
{
    if (!_root)
        return xml_node();
 
    impl::xml_memory_page *page =
        reinterpret_cast<impl::xml_memory_page *>(_root->header & impl::xml_memory_page_pointer_mask);
 
    return xml_node(static_cast<impl::xml_document_struct *>(page->allocator));
}
 
PUGI__FN xml_text xml_node::text() const
{
    return xml_text(_root);
}
 
PUGI__FN const char_t *xml_node::child_value() const
{
    if (!_root)
        return PUGIXML_TEXT("");
 
    for (xml_node_struct *i = _root->first_child; i; i = i->next_sibling)
        if (i->value && impl::is_text_node(i))
            return i->value;
 
    return PUGIXML_TEXT("");
}
 
PUGI__FN const char_t *xml_node::child_value(const char_t *name_) const
{
    return child(name_).child_value();
}
 
PUGI__FN xml_attribute xml_node::first_attribute() const
{
    return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
}
 
PUGI__FN xml_attribute xml_node::last_attribute() const
{
    return _root && _root->first_attribute ? xml_attribute(_root->first_attribute->prev_attribute_c) : xml_attribute();
}
 
PUGI__FN xml_node xml_node::first_child() const
{
    return _root ? xml_node(_root->first_child) : xml_node();
}
 
PUGI__FN xml_node xml_node::last_child() const
{
    return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
}
 
PUGI__FN bool xml_node::set_name(const char_t *rhs)
{
    switch (type())
    {
    case node_pi:
    case node_declaration:
    case node_element:
        return impl::strcpy_insitu(_root->name, _root->header, impl::xml_memory_page_name_allocated_mask, rhs);
 
    default:
        return false;
    }
}
 
PUGI__FN bool xml_node::set_value(const char_t *rhs)
{
    switch (type())
    {
    case node_pi:
    case node_cdata:
    case node_pcdata:
    case node_comment:
    case node_doctype:
        return impl::strcpy_insitu(_root->value, _root->header, impl::xml_memory_page_value_allocated_mask, rhs);
 
    default:
        return false;
    }
}
 
PUGI__FN xml_attribute xml_node::append_attribute(const char_t *name_)
{
    if (type() != node_element && type() != node_declaration)
        return xml_attribute();
 
    xml_attribute a(impl::append_attribute_ll(_root, impl::get_allocator(_root)));
    a.set_name(name_);
 
    return a;
}
 
PUGI__FN xml_attribute xml_node::prepend_attribute(const char_t *name_)
{
    if (type() != node_element && type() != node_declaration)
        return xml_attribute();
 
    xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
    if (!a)
        return xml_attribute();
 
    a.set_name(name_);
 
    xml_attribute_struct *head = _root->first_attribute;
 
    if (head)
    {
        a._attr->prev_attribute_c = head->prev_attribute_c;
        head->prev_attribute_c = a._attr;
    }
    else
        a._attr->prev_attribute_c = a._attr;
 
    a._attr->next_attribute = head;
    _root->first_attribute = a._attr;
 
    return a;
}
 
PUGI__FN xml_attribute xml_node::insert_attribute_before(const char_t *name_, const xml_attribute &attr)
{
    if ((type() != node_element && type() != node_declaration) || attr.empty())
        return xml_attribute();
 
    // check that attribute belongs to *this
    xml_attribute_struct *cur = attr._attr;
 
    while (cur->prev_attribute_c->next_attribute)
        cur = cur->prev_attribute_c;
 
    if (cur != _root->first_attribute)
        return xml_attribute();
 
    xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
    if (!a)
        return xml_attribute();
 
    a.set_name(name_);
 
    if (attr._attr->prev_attribute_c->next_attribute)
        attr._attr->prev_attribute_c->next_attribute = a._attr;
    else
        _root->first_attribute = a._attr;
 
    a._attr->prev_attribute_c = attr._attr->prev_attribute_c;
    a._attr->next_attribute = attr._attr;
    attr._attr->prev_attribute_c = a._attr;
 
    return a;
}
 
PUGI__FN xml_attribute xml_node::insert_attribute_after(const char_t *name_, const xml_attribute &attr)
{
    if ((type() != node_element && type() != node_declaration) || attr.empty())
        return xml_attribute();
 
    // check that attribute belongs to *this
    xml_attribute_struct *cur = attr._attr;
 
    while (cur->prev_attribute_c->next_attribute)
        cur = cur->prev_attribute_c;
 
    if (cur != _root->first_attribute)
        return xml_attribute();
 
    xml_attribute a(impl::allocate_attribute(impl::get_allocator(_root)));
    if (!a)
        return xml_attribute();
 
    a.set_name(name_);
 
    if (attr._attr->next_attribute)
        attr._attr->next_attribute->prev_attribute_c = a._attr;
    else
        _root->first_attribute->prev_attribute_c = a._attr;
 
    a._attr->next_attribute = attr._attr->next_attribute;
    a._attr->prev_attribute_c = attr._attr;
    attr._attr->next_attribute = a._attr;
 
    return a;
}
 
PUGI__FN xml_attribute xml_node::append_copy(const xml_attribute &proto)
{
    if (!proto)
        return xml_attribute();
 
    xml_attribute result = append_attribute(proto.name());
    result.set_value(proto.value());
 
    return result;
}
 
PUGI__FN xml_attribute xml_node::prepend_copy(const xml_attribute &proto)
{
    if (!proto)
        return xml_attribute();
 
    xml_attribute result = prepend_attribute(proto.name());
    result.set_value(proto.value());
 
    return result;
}
 
PUGI__FN xml_attribute xml_node::insert_copy_after(const xml_attribute &proto, const xml_attribute &attr)
{
    if (!proto)
        return xml_attribute();
 
    xml_attribute result = insert_attribute_after(proto.name(), attr);
    result.set_value(proto.value());
 
    return result;
}
 
PUGI__FN xml_attribute xml_node::insert_copy_before(const xml_attribute &proto, const xml_attribute &attr)
{
    if (!proto)
        return xml_attribute();
 
    xml_attribute result = insert_attribute_before(proto.name(), attr);
    result.set_value(proto.value());
 
    return result;
}
 
PUGI__FN xml_node xml_node::append_child(xml_node_type type_)
{
    if (!impl::allow_insert_child(this->type(), type_))
        return xml_node();
 
    xml_node n(impl::append_node(_root, impl::get_allocator(_root), type_));
 
    if (type_ == node_declaration)
        n.set_name(PUGIXML_TEXT("xml"));
 
    return n;
}
 
PUGI__FN xml_node xml_node::prepend_child(xml_node_type type_)
{
    if (!impl::allow_insert_child(this->type(), type_))
        return xml_node();
 
    xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
    if (!n)
        return xml_node();
 
    n._root->parent = _root;
 
    xml_node_struct *head = _root->first_child;
 
    if (head)
    {
        n._root->prev_sibling_c = head->prev_sibling_c;
        head->prev_sibling_c = n._root;
    }
    else
        n._root->prev_sibling_c = n._root;
 
    n._root->next_sibling = head;
    _root->first_child = n._root;
 
    if (type_ == node_declaration)
        n.set_name(PUGIXML_TEXT("xml"));
 
    return n;
}
 
PUGI__FN xml_node xml_node::insert_child_before(xml_node_type type_, const xml_node &node)
{
    if (!impl::allow_insert_child(this->type(), type_))
        return xml_node();
    if (!node._root || node._root->parent != _root)
        return xml_node();
 
    xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
    if (!n)
        return xml_node();
 
    n._root->parent = _root;
 
    if (node._root->prev_sibling_c->next_sibling)
        node._root->prev_sibling_c->next_sibling = n._root;
    else
        _root->first_child = n._root;
 
    n._root->prev_sibling_c = node._root->prev_sibling_c;
    n._root->next_sibling = node._root;
    node._root->prev_sibling_c = n._root;
 
    if (type_ == node_declaration)
        n.set_name(PUGIXML_TEXT("xml"));
 
    return n;
}
 
PUGI__FN xml_node xml_node::insert_child_after(xml_node_type type_, const xml_node &node)
{
    if (!impl::allow_insert_child(this->type(), type_))
        return xml_node();
    if (!node._root || node._root->parent != _root)
        return xml_node();
 
    xml_node n(impl::allocate_node(impl::get_allocator(_root), type_));
    if (!n)
        return xml_node();
 
    n._root->parent = _root;
 
    if (node._root->next_sibling)
        node._root->next_sibling->prev_sibling_c = n._root;
    else
        _root->first_child->prev_sibling_c = n._root;
 
    n._root->next_sibling = node._root->next_sibling;
    n._root->prev_sibling_c = node._root;
    node._root->next_sibling = n._root;
 
    if (type_ == node_declaration)
        n.set_name(PUGIXML_TEXT("xml"));
 
    return n;
}
 
PUGI__FN xml_node xml_node::append_child(const char_t *name_)
{
    xml_node result = append_child(node_element);
 
    result.set_name(name_);
 
    return result;
}
 
PUGI__FN xml_node xml_node::prepend_child(const char_t *name_)
{
    xml_node result = prepend_child(node_element);
 
    result.set_name(name_);
 
    return result;
}
 
PUGI__FN xml_node xml_node::insert_child_after(const char_t *name_, const xml_node &node)
{
    xml_node result = insert_child_after(node_element, node);
 
    result.set_name(name_);
 
    return result;
}
 
PUGI__FN xml_node xml_node::insert_child_before(const char_t *name_, const xml_node &node)
{
    xml_node result = insert_child_before(node_element, node);
 
    result.set_name(name_);
 
    return result;
}
 
PUGI__FN xml_node xml_node::append_copy(const xml_node &proto)
{
    xml_node result = append_child(proto.type());
 
    if (result)
        impl::recursive_copy_skip(result, proto, result);
 
    return result;
}
 
PUGI__FN xml_node xml_node::prepend_copy(const xml_node &proto)
{
    xml_node result = prepend_child(proto.type());
 
    if (result)
        impl::recursive_copy_skip(result, proto, result);
 
    return result;
}
 
PUGI__FN xml_node xml_node::insert_copy_after(const xml_node &proto, const xml_node &node)
{
    xml_node result = insert_child_after(proto.type(), node);
 
    if (result)
        impl::recursive_copy_skip(result, proto, result);
 
    return result;
}
 
PUGI__FN xml_node xml_node::insert_copy_before(const xml_node &proto, const xml_node &node)
{
    xml_node result = insert_child_before(proto.type(), node);
 
    if (result)
        impl::recursive_copy_skip(result, proto, result);
 
    return result;
}
 
PUGI__FN bool xml_node::remove_attribute(const char_t *name_)
{
    return remove_attribute(attribute(name_));
}
 
PUGI__FN bool xml_node::remove_attribute(const xml_attribute &a)
{
    if (!_root || !a._attr)
        return false;
 
    // check that attribute belongs to *this
    xml_attribute_struct *attr = a._attr;
 
    while (attr->prev_attribute_c->next_attribute)
        attr = attr->prev_attribute_c;
 
    if (attr != _root->first_attribute)
        return false;
 
    if (a._attr->next_attribute)
        a._attr->next_attribute->prev_attribute_c = a._attr->prev_attribute_c;
    else if (_root->first_attribute)
        _root->first_attribute->prev_attribute_c = a._attr->prev_attribute_c;
 
    if (a._attr->prev_attribute_c->next_attribute)
        a._attr->prev_attribute_c->next_attribute = a._attr->next_attribute;
    else
        _root->first_attribute = a._attr->next_attribute;
 
    impl::destroy_attribute(a._attr, impl::get_allocator(_root));
 
    return true;
}
 
PUGI__FN bool xml_node::remove_child(const char_t *name_)
{
    return remove_child(child(name_));
}
 
PUGI__FN bool xml_node::remove_child(const xml_node &n)
{
    if (!_root || !n._root || n._root->parent != _root)
        return false;
 
    if (n._root->next_sibling)
        n._root->next_sibling->prev_sibling_c = n._root->prev_sibling_c;
    else if (_root->first_child)
        _root->first_child->prev_sibling_c = n._root->prev_sibling_c;
 
    if (n._root->prev_sibling_c->next_sibling)
        n._root->prev_sibling_c->next_sibling = n._root->next_sibling;
    else
        _root->first_child = n._root->next_sibling;
 
    impl::destroy_node(n._root, impl::get_allocator(_root));
 
    return true;
}
 
PUGI__FN xml_parse_result xml_node::append_buffer(const void *contents, size_t size, unsigned int options,
                                                  xml_encoding encoding)
{
    // append_buffer is only valid for elements/documents
    if (!impl::allow_insert_child(type(), node_element))
        return impl::make_parse_result(status_append_invalid_root);
 
    // get document node
    impl::xml_document_struct *doc = static_cast<impl::xml_document_struct *>(root()._root);
    assert(doc);
 
    // get extra buffer element (we'll store the document fragment buffer there so that we can deallocate it later)
    impl::xml_memory_page *page = 0;
    impl::xml_extra_buffer *extra =
        static_cast<impl::xml_extra_buffer *>(doc->allocate_memory(sizeof(impl::xml_extra_buffer), page));
    (void)page;
 
    if (!extra)
        return impl::make_parse_result(status_out_of_memory);
 
    // save name; name of the root has to be NULL before parsing - otherwise closing node mismatches will not be
    // detected at the top level
    char_t *rootname = _root->name;
    _root->name = 0;
 
    // parse
    char_t *buffer = 0;
    xml_parse_result res = impl::load_buffer_impl(doc, _root, const_cast<void *>(contents), size, options, encoding,
                                                  false, false, &buffer);
 
    // restore name
    _root->name = rootname;
 
    // add extra buffer to the list
    extra->buffer = buffer;
    extra->next = doc->extra_buffers;
    doc->extra_buffers = extra;
 
    return res;
}
 
PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t *name_, const char_t *attr_name,
                                                    const char_t *attr_value) const
{
    if (!_root)
        return xml_node();
 
    for (xml_node_struct *i = _root->first_child; i; i = i->next_sibling)
        if (i->name && impl::strequal(name_, i->name))
        {
            for (xml_attribute_struct *a = i->first_attribute; a; a = a->next_attribute)
                if (a->name && impl::strequal(attr_name, a->name) &&
                    impl::strequal(attr_value, a->value ? a->value : PUGIXML_TEXT("")))
                    return xml_node(i);
        }
 
    return xml_node();
}
 
PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t *attr_name, const char_t *attr_value) const
{
    if (!_root)
        return xml_node();
 
    for (xml_node_struct *i = _root->first_child; i; i = i->next_sibling)
        for (xml_attribute_struct *a = i->first_attribute; a; a = a->next_attribute)
            if (a->name && impl::strequal(attr_name, a->name) &&
                impl::strequal(attr_value, a->value ? a->value : PUGIXML_TEXT("")))
                return xml_node(i);
 
    return xml_node();
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN string_t xml_node::path(char_t delimiter) const
{
    xml_node cursor = *this; // Make a copy.
 
    string_t result = cursor.name();
 
    while (cursor.parent())
    {
        cursor = cursor.parent();
 
        string_t temp = cursor.name();
        temp += delimiter;
        temp += result;
        result.swap(temp);
    }
 
    return result;
}
#endif
 
PUGI__FN xml_node xml_node::first_element_by_path(const char_t *path_, char_t delimiter) const
{
    xml_node found = *this; // Current search context.
 
    if (!_root || !path_ || !path_[0])
        return found;
 
    if (path_[0] == delimiter)
    {
        // Absolute path; e.g. '/foo/bar'
        found = found.root();
        ++path_;
    }
 
    const char_t *path_segment = path_;
 
    while (*path_segment == delimiter)
        ++path_segment;
 
    const char_t *path_segment_end = path_segment;
 
    while (*path_segment_end && *path_segment_end != delimiter)
        ++path_segment_end;
 
    if (path_segment == path_segment_end)
        return found;
 
    const char_t *next_segment = path_segment_end;
 
    while (*next_segment == delimiter)
        ++next_segment;
 
    if (*path_segment == '.' && path_segment + 1 == path_segment_end)
        return found.first_element_by_path(next_segment, delimiter);
    else if (*path_segment == '.' && *(path_segment + 1) == '.' && path_segment + 2 == path_segment_end)
        return found.parent().first_element_by_path(next_segment, delimiter);
    else
    {
        for (xml_node_struct *j = found._root->first_child; j; j = j->next_sibling)
        {
            if (j->name &&
                impl::strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
            {
                xml_node subsearch = xml_node(j).first_element_by_path(next_segment, delimiter);
 
                if (subsearch)
                    return subsearch;
            }
        }
 
        return xml_node();
    }
}
 
PUGI__FN bool xml_node::traverse(xml_tree_walker &walker)
{
    walker._depth = -1;
 
    xml_node arg_begin = *this;
    if (!walker.begin(arg_begin))
        return false;
 
    xml_node cur = first_child();
 
    if (cur)
    {
        ++walker._depth;
 
        do
        {
            xml_node arg_for_each = cur;
            if (!walker.for_each(arg_for_each))
                return false;
 
            if (cur.first_child())
            {
                ++walker._depth;
                cur = cur.first_child();
            }
            else if (cur.next_sibling())
                cur = cur.next_sibling();
            else
            {
                // Borland C++ workaround
                while (!cur.next_sibling() && cur != *this && !cur.parent().empty())
                {
                    --walker._depth;
                    cur = cur.parent();
                }
 
                if (cur != *this)
                    cur = cur.next_sibling();
            }
        } while (cur && cur != *this);
    }
 
    assert(walker._depth == -1);
 
    xml_node arg_end = *this;
    return walker.end(arg_end);
}
 
PUGI__FN size_t xml_node::hash_value() const
{
    return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
}
 
PUGI__FN xml_node_struct *xml_node::internal_object() const
{
    return _root;
}
 
PUGI__FN void xml_node::print(xml_writer &writer, const char_t *indent, unsigned int flags, xml_encoding encoding,
                              unsigned int depth) const
{
    if (!_root)
        return;
 
    impl::xml_buffered_writer buffered_writer(writer, encoding);
 
    impl::node_output(buffered_writer, *this, indent, flags, depth);
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN void xml_node::print(std::basic_ostream<char, std::char_traits<char>> &stream, const char_t *indent,
                              unsigned int flags, xml_encoding encoding, unsigned int depth) const
{
    xml_writer_stream writer(stream);
 
    print(writer, indent, flags, encoding, depth);
}
 
PUGI__FN void xml_node::print(std::basic_ostream<wchar_t, std::char_traits<wchar_t>> &stream, const char_t *indent,
                              unsigned int flags, unsigned int depth) const
{
    xml_writer_stream writer(stream);
 
    print(writer, indent, flags, encoding_wchar, depth);
}
#endif
 
PUGI__FN ptrdiff_t xml_node::offset_debug() const
{
    xml_node_struct *r = root()._root;
 
    if (!r)
        return -1;
 
    const char_t *buffer = static_cast<impl::xml_document_struct *>(r)->buffer;
 
    if (!buffer)
        return -1;
 
    switch (type())
    {
    case node_document:
        return 0;
 
    case node_element:
    case node_declaration:
    case node_pi:
        return (_root->header & impl::xml_memory_page_name_allocated_mask) ? -1 : _root->name - buffer;
 
    case node_pcdata:
    case node_cdata:
    case node_comment:
    case node_doctype:
        return (_root->header & impl::xml_memory_page_value_allocated_mask) ? -1 : _root->value - buffer;
 
    default:
        return -1;
    }
}
 
#ifdef __BORLANDC__
PUGI__FN bool operator&&(const xml_node &lhs, bool rhs)
{
    return (bool)lhs && rhs;
}
 
PUGI__FN bool operator||(const xml_node &lhs, bool rhs)
{
    return (bool)lhs || rhs;
}
#endif
 
PUGI__FN xml_text::xml_text(xml_node_struct *root) : _root(root) {}
 
PUGI__FN xml_node_struct *xml_text::_data() const
{
    if (!_root || impl::is_text_node(_root))
        return _root;
 
    for (xml_node_struct *node = _root->first_child; node; node = node->next_sibling)
        if (impl::is_text_node(node))
            return node;
 
    return 0;
}
 
PUGI__FN xml_node_struct *xml_text::_data_new()
{
    xml_node_struct *d = _data();
    if (d)
        return d;
 
    return xml_node(_root).append_child(node_pcdata).internal_object();
}
 
PUGI__FN xml_text::xml_text() : _root(0) {}
 
PUGI__FN static void unspecified_bool_xml_text(xml_text ***) {}
 
PUGI__FN xml_text::operator xml_text::unspecified_bool_type() const
{
    return _data() ? unspecified_bool_xml_text : 0;
}
 
PUGI__FN bool xml_text::operator!() const
{
    return !_data();
}
 
PUGI__FN bool xml_text::empty() const
{
    return _data() == 0;
}
 
PUGI__FN const char_t *xml_text::get() const
{
    xml_node_struct *d = _data();
 
    return (d && d->value) ? d->value : PUGIXML_TEXT("");
}
 
PUGI__FN const char_t *xml_text::as_string(const char_t *def) const
{
    xml_node_struct *d = _data();
 
    return (d && d->value) ? d->value : def;
}
 
PUGI__FN int xml_text::as_int(int def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_int(d ? d->value : 0, def);
}
 
PUGI__FN unsigned int xml_text::as_uint(unsigned int def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_uint(d ? d->value : 0, def);
}
 
PUGI__FN double xml_text::as_double(double def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_double(d ? d->value : 0, def);
}
 
PUGI__FN float xml_text::as_float(float def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_float(d ? d->value : 0, def);
}
 
PUGI__FN bool xml_text::as_bool(bool def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_bool(d ? d->value : 0, def);
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN long long xml_text::as_llong(long long def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_llong(d ? d->value : 0, def);
}
 
PUGI__FN unsigned long long xml_text::as_ullong(unsigned long long def) const
{
    xml_node_struct *d = _data();
 
    return impl::get_value_ullong(d ? d->value : 0, def);
}
#endif
 
PUGI__FN bool xml_text::set(const char_t *rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::strcpy_insitu(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
PUGI__FN bool xml_text::set(int rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
PUGI__FN bool xml_text::set(unsigned int rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
PUGI__FN bool xml_text::set(double rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
PUGI__FN bool xml_text::set(bool rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN bool xml_text::set(long long rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
 
PUGI__FN bool xml_text::set(unsigned long long rhs)
{
    xml_node_struct *dn = _data_new();
 
    return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
}
#endif
 
PUGI__FN xml_text &xml_text::operator=(const char_t *rhs)
{
    set(rhs);
    return *this;
}
 
PUGI__FN xml_text &xml_text::operator=(int rhs)
{
    set(rhs);
    return *this;
}
 
PUGI__FN xml_text &xml_text::operator=(unsigned int rhs)
{
    set(rhs);
    return *this;
}
 
PUGI__FN xml_text &xml_text::operator=(double rhs)
{
    set(rhs);
    return *this;
}
 
PUGI__FN xml_text &xml_text::operator=(bool rhs)
{
    set(rhs);
    return *this;
}
 
#ifdef PUGIXML_HAS_LONG_LONG
PUGI__FN xml_text &xml_text::operator=(long long rhs)
{
    set(rhs);
    return *this;
}
 
PUGI__FN xml_text &xml_text::operator=(unsigned long long rhs)
{
    set(rhs);
    return *this;
}
#endif
 
PUGI__FN xml_node xml_text::data() const
{
    return xml_node(_data());
}
 
#ifdef __BORLANDC__
PUGI__FN bool operator&&(const xml_text &lhs, bool rhs)
{
    return (bool)lhs && rhs;
}
 
PUGI__FN bool operator||(const xml_text &lhs, bool rhs)
{
    return (bool)lhs || rhs;
}
#endif
 
PUGI__FN xml_node_iterator::xml_node_iterator() {}
 
PUGI__FN xml_node_iterator::xml_node_iterator(const xml_node &node) : _wrap(node), _parent(node.parent()) {}
 
PUGI__FN xml_node_iterator::xml_node_iterator(xml_node_struct *ref, xml_node_struct *parent)
    : _wrap(ref), _parent(parent)
{
}
 
PUGI__FN bool xml_node_iterator::operator==(const xml_node_iterator &rhs) const
{
    return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
}
 
PUGI__FN bool xml_node_iterator::operator!=(const xml_node_iterator &rhs) const
{
    return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
}
 
PUGI__FN xml_node &xml_node_iterator::operator*() const
{
    assert(_wrap._root);
    return _wrap;
}
 
PUGI__FN xml_node *xml_node_iterator::operator->() const
{
    assert(_wrap._root);
    return const_cast<xml_node *>(&_wrap); // BCC32 workaround
}
 
PUGI__FN const xml_node_iterator &xml_node_iterator::operator++()
{
    assert(_wrap._root);
    _wrap._root = _wrap._root->next_sibling;
    return *this;
}
 
PUGI__FN xml_node_iterator xml_node_iterator::operator++(int)
{
    xml_node_iterator temp = *this;
    ++*this;
    return temp;
}
 
PUGI__FN const xml_node_iterator &xml_node_iterator::operator--()
{
    _wrap = _wrap._root ? _wrap.previous_sibling() : _parent.last_child();
    return *this;
}
 
PUGI__FN xml_node_iterator xml_node_iterator::operator--(int)
{
    xml_node_iterator temp = *this;
    --*this;
    return temp;
}
 
PUGI__FN xml_attribute_iterator::xml_attribute_iterator() {}
 
PUGI__FN xml_attribute_iterator::xml_attribute_iterator(const xml_attribute &attr, const xml_node &parent)
    : _wrap(attr), _parent(parent)
{
}
 
PUGI__FN xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct *ref, xml_node_struct *parent)
    : _wrap(ref), _parent(parent)
{
}
 
PUGI__FN bool xml_attribute_iterator::operator==(const xml_attribute_iterator &rhs) const
{
    return _wrap._attr == rhs._wrap._attr && _parent._root == rhs._parent._root;
}
 
PUGI__FN bool xml_attribute_iterator::operator!=(const xml_attribute_iterator &rhs) const
{
    return _wrap._attr != rhs._wrap._attr || _parent._root != rhs._parent._root;
}
 
PUGI__FN xml_attribute &xml_attribute_iterator::operator*() const
{
    assert(_wrap._attr);
    return _wrap;
}
 
PUGI__FN xml_attribute *xml_attribute_iterator::operator->() const
{
    assert(_wrap._attr);
    return const_cast<xml_attribute *>(&_wrap); // BCC32 workaround
}
 
PUGI__FN const xml_attribute_iterator &xml_attribute_iterator::operator++()
{
    assert(_wrap._attr);
    _wrap._attr = _wrap._attr->next_attribute;
    return *this;
}
 
PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator++(int)
{
    xml_attribute_iterator temp = *this;
    ++*this;
    return temp;
}
 
PUGI__FN const xml_attribute_iterator &xml_attribute_iterator::operator--()
{
    _wrap = _wrap._attr ? _wrap.previous_attribute() : _parent.last_attribute();
    return *this;
}
 
PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator--(int)
{
    xml_attribute_iterator temp = *this;
    --*this;
    return temp;
}
 
PUGI__FN xml_named_node_iterator::xml_named_node_iterator() : _name(0) {}
 
PUGI__FN xml_named_node_iterator::xml_named_node_iterator(const xml_node &node, const char_t *name)
    : _wrap(node), _parent(node.parent()), _name(name)
{
}
 
PUGI__FN xml_named_node_iterator::xml_named_node_iterator(xml_node_struct *ref, xml_node_struct *parent,
                                                          const char_t *name)
    : _wrap(ref), _parent(parent), _name(name)
{
}
 
PUGI__FN bool xml_named_node_iterator::operator==(const xml_named_node_iterator &rhs) const
{
    return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
}
 
PUGI__FN bool xml_named_node_iterator::operator!=(const xml_named_node_iterator &rhs) const
{
    return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
}
 
PUGI__FN xml_node &xml_named_node_iterator::operator*() const
{
    assert(_wrap._root);
    return _wrap;
}
 
PUGI__FN xml_node *xml_named_node_iterator::operator->() const
{
    assert(_wrap._root);
    return const_cast<xml_node *>(&_wrap); // BCC32 workaround
}
 
PUGI__FN const xml_named_node_iterator &xml_named_node_iterator::operator++()
{
    assert(_wrap._root);
    _wrap = _wrap.next_sibling(_name);
    return *this;
}
 
PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator++(int)
{
    xml_named_node_iterator temp = *this;
    ++*this;
    return temp;
}
 
PUGI__FN const xml_named_node_iterator &xml_named_node_iterator::operator--()
{
    if (_wrap._root)
        _wrap = _wrap.previous_sibling(_name);
    else
    {
        _wrap = _parent.last_child();
 
        if (!impl::strequal(_wrap.name(), _name))
            _wrap = _wrap.previous_sibling(_name);
    }
 
    return *this;
}
 
PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator--(int)
{
    xml_named_node_iterator temp = *this;
    --*this;
    return temp;
}
 
PUGI__FN xml_parse_result::xml_parse_result() : status(status_internal_error), offset(0), encoding(encoding_auto) {}
 
PUGI__FN xml_parse_result::operator bool() const
{
    return status == status_ok;
}
 
PUGI__FN const char *xml_parse_result::description() const
{
    switch (status)
    {
    case status_ok:
        return "No error";
 
    case status_file_not_found:
        return "File was not found";
    case status_io_error:
        return "Error reading from file/stream";
    case status_out_of_memory:
        return "Could not allocate memory";
    case status_internal_error:
        return "Internal error occurred";
 
    case status_unrecognized_tag:
        return "Could not determine tag type";
 
    case status_bad_pi:
        return "Error parsing document declaration/processing instruction";
    case status_bad_comment:
        return "Error parsing comment";
    case status_bad_cdata:
        return "Error parsing CDATA section";
    case status_bad_doctype:
        return "Error parsing document type declaration";
    case status_bad_pcdata:
        return "Error parsing PCDATA section";
    case status_bad_start_element:
        return "Error parsing start element tag";
    case status_bad_attribute:
        return "Error parsing element attribute";
    case status_bad_end_element:
        return "Error parsing end element tag";
    case status_end_element_mismatch:
        return "Start-end tags mismatch";
 
    case status_append_invalid_root:
        return "Unable to append nodes: root is not an element or document";
 
    case status_no_document_element:
        return "No document element found";
 
    default:
        return "Unknown error";
    }
}
 
PUGI__FN xml_document::xml_document() : _buffer(0)
{
    create();
}
 
PUGI__FN xml_document::~xml_document()
{
    destroy();
}
 
PUGI__FN void xml_document::reset()
{
    destroy();
    create();
}
 
PUGI__FN void xml_document::reset(const xml_document &proto)
{
    reset();
 
    for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
        append_copy(cur);
}
 
PUGI__FN void xml_document::create()
{
    assert(!_root);
 
    // initialize sentinel page
    PUGI__STATIC_ASSERT(sizeof(impl::xml_memory_page) + sizeof(impl::xml_document_struct) +
                            impl::xml_memory_page_alignment <=
                        sizeof(_memory));
 
    // align upwards to page boundary
    void *page_memory =
        reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(_memory) + (impl::xml_memory_page_alignment - 1)) &
                                 ~(impl::xml_memory_page_alignment - 1));
 
    // prepare page structure
    impl::xml_memory_page *page = impl::xml_memory_page::construct(page_memory);
    assert(page);
 
    page->busy_size = impl::xml_memory_page_size;
 
    // allocate new root
    _root = new (page->data) impl::xml_document_struct(page);
    _root->prev_sibling_c = _root;
 
    // setup sentinel page
    page->allocator = static_cast<impl::xml_document_struct *>(_root);
}
 
PUGI__FN void xml_document::destroy()
{
    assert(_root);
 
    // destroy static storage
    if (_buffer)
    {
        impl::xml_memory::deallocate(_buffer);
        _buffer = 0;
    }
 
    // destroy extra buffers (note: no need to destroy linked list nodes, they're allocated using document allocator)
    for (impl::xml_extra_buffer *extra = static_cast<impl::xml_document_struct *>(_root)->extra_buffers; extra;
         extra = extra->next)
    {
        if (extra->buffer)
            impl::xml_memory::deallocate(extra->buffer);
    }
 
    // destroy dynamic storage, leave sentinel page (it's in static memory)
    impl::xml_memory_page *root_page =
        reinterpret_cast<impl::xml_memory_page *>(_root->header & impl::xml_memory_page_pointer_mask);
    assert(root_page && !root_page->prev && !root_page->memory);
 
    for (impl::xml_memory_page *page = root_page->next; page;)
    {
        impl::xml_memory_page *next = page->next;
 
        impl::xml_allocator::deallocate_page(page);
 
        page = next;
    }
 
    _root = 0;
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN xml_parse_result xml_document::load(std::basic_istream<char, std::char_traits<char>> &stream,
                                             unsigned int options, xml_encoding encoding)
{
    reset();
 
    return impl::load_stream_impl(*this, stream, options, encoding);
}
 
PUGI__FN xml_parse_result xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t>> &stream,
                                             unsigned int options)
{
    reset();
 
    return impl::load_stream_impl(*this, stream, options, encoding_wchar);
}
#endif
 
PUGI__FN xml_parse_result xml_document::load(const char_t *contents, unsigned int options)
{
    // Force native encoding (skip autodetection)
#ifdef PUGIXML_WCHAR_MODE
    xml_encoding encoding = encoding_wchar;
#else
    xml_encoding encoding = encoding_utf8;
#endif
 
    return load_buffer(contents, impl::strlength(contents) * sizeof(char_t), options, encoding);
}
 
PUGI__FN xml_parse_result xml_document::load_file(const char *path_, unsigned int options, xml_encoding encoding)
{
    reset();
 
    FILE *file = fopen(path_, "rb");
 
    return impl::load_file_impl(*this, file, options, encoding);
}
 
PUGI__FN xml_parse_result xml_document::load_file(const wchar_t *path_, unsigned int options, xml_encoding encoding)
{
    reset();
 
    FILE *file = impl::open_file_wide(path_, L"rb");
 
    return impl::load_file_impl(*this, file, options, encoding);
}
 
PUGI__FN xml_parse_result xml_document::load_buffer(const void *contents, size_t size, unsigned int options,
                                                    xml_encoding encoding)
{
    reset();
 
    return impl::load_buffer_impl(static_cast<impl::xml_document_struct *>(_root), _root, const_cast<void *>(contents),
                                  size, options, encoding, false, false, &_buffer);
}
 
PUGI__FN xml_parse_result xml_document::load_buffer_inplace(void *contents, size_t size, unsigned int options,
                                                            xml_encoding encoding)
{
    reset();
 
    return impl::load_buffer_impl(static_cast<impl::xml_document_struct *>(_root), _root, contents, size, options,
                                  encoding, true, false, &_buffer);
}
 
PUGI__FN xml_parse_result xml_document::load_buffer_inplace_own(void *contents, size_t size, unsigned int options,
                                                                xml_encoding encoding)
{
    reset();
 
    return impl::load_buffer_impl(static_cast<impl::xml_document_struct *>(_root), _root, contents, size, options,
                                  encoding, true, true, &_buffer);
}
 
PUGI__FN void xml_document::save(xml_writer &writer, const char_t *indent, unsigned int flags,
                                 xml_encoding encoding) const
{
    impl::xml_buffered_writer buffered_writer(writer, encoding);
 
    if ((flags & format_write_bom) && encoding != encoding_latin1)
    {
        // BOM always represents the codepoint U+FEFF, so just write it in native encoding
#ifdef PUGIXML_WCHAR_MODE
        unsigned int bom = 0xfeff;
        buffered_writer.write(static_cast<wchar_t>(bom));
#else
        buffered_writer.write('\xef', '\xbb', '\xbf');
#endif
    }
 
    if (!(flags & format_no_declaration) && !impl::has_declaration(*this))
    {
        buffered_writer.write(PUGIXML_TEXT("<?xml version=\"1.0\""));
        if (encoding == encoding_latin1)
            buffered_writer.write(PUGIXML_TEXT(" encoding=\"ISO-8859-1\""));
        buffered_writer.write('?', '>');
        if (!(flags & format_raw))
            buffered_writer.write('\n');
    }
 
    impl::node_output(buffered_writer, *this, indent, flags, 0);
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN void xml_document::save(std::basic_ostream<char, std::char_traits<char>> &stream, const char_t *indent,
                                 unsigned int flags, xml_encoding encoding) const
{
    xml_writer_stream writer(stream);
 
    save(writer, indent, flags, encoding);
}
 
PUGI__FN void xml_document::save(std::basic_ostream<wchar_t, std::char_traits<wchar_t>> &stream, const char_t *indent,
                                 unsigned int flags) const
{
    xml_writer_stream writer(stream);
 
    save(writer, indent, flags, encoding_wchar);
}
#endif
 
PUGI__FN bool xml_document::save_file(const char *path_, const char_t *indent, unsigned int flags,
                                      xml_encoding encoding) const
{
    FILE *file = fopen(path_, (flags & format_save_file_text) ? "w" : "wb");
    return impl::save_file_impl(*this, file, indent, flags, encoding);
}
 
PUGI__FN bool xml_document::save_file(const wchar_t *path_, const char_t *indent, unsigned int flags,
                                      xml_encoding encoding) const
{
    FILE *file = impl::open_file_wide(path_, (flags & format_save_file_text) ? L"w" : L"wb");
    return impl::save_file_impl(*this, file, indent, flags, encoding);
}
 
PUGI__FN xml_node xml_document::document_element() const
{
    assert(_root);
 
    for (xml_node_struct *i = _root->first_child; i; i = i->next_sibling)
        if ((i->header & impl::xml_memory_page_type_mask) + 1 == node_element)
            return xml_node(i);
 
    return xml_node();
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const wchar_t *str)
{
    assert(str);
 
    return impl::as_utf8_impl(str, impl::strlength_wide(str));
}
 
PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const std::basic_string<wchar_t> &str)
{
    return impl::as_utf8_impl(str.c_str(), str.size());
}
 
PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const char *str)
{
    assert(str);
 
    return impl::as_wide_impl(str, strlen(str));
}
 
PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const std::string &str)
{
    return impl::as_wide_impl(str.c_str(), str.size());
}
#endif
 
PUGI__FN void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate,
                                                               deallocation_function deallocate)
{
    impl::xml_memory::allocate = allocate;
    impl::xml_memory::deallocate = deallocate;
}
 
PUGI__FN allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
{
    return impl::xml_memory::allocate;
}
 
PUGI__FN deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
{
    return impl::xml_memory::deallocate;
}
} // namespace pugi
 
#if !defined(PUGIXML_NO_STL) && (defined(_MSC_VER) || defined(__ICC))
namespace std
{
// Workarounds for (non-standard) iterator category detection for older versions (MSVC7/IC8 and earlier)
PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_node_iterator &)
{
    return std::bidirectional_iterator_tag();
}
 
PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_attribute_iterator &)
{
    return std::bidirectional_iterator_tag();
}
 
PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_named_node_iterator &)
{
    return std::bidirectional_iterator_tag();
}
} // namespace std
#endif
 
#if !defined(PUGIXML_NO_STL) && defined(__SUNPRO_CC)
namespace std
{
// Workarounds for (non-standard) iterator category detection
PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_node_iterator &)
{
    return std::bidirectional_iterator_tag();
}
 
PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_attribute_iterator &)
{
    return std::bidirectional_iterator_tag();
}
 
PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_named_node_iterator &)
{
    return std::bidirectional_iterator_tag();
}
} // namespace std
#endif
 
#ifndef PUGIXML_NO_XPATH
 
// STL replacements
PUGI__NS_BEGIN
struct equal_to
{
    template <typename T>
    bool operator()(const T &lhs, const T &rhs) const
    {
        return lhs == rhs;
    }
};
 
struct not_equal_to
{
    template <typename T>
    bool operator()(const T &lhs, const T &rhs) const
    {
        return lhs != rhs;
    }
};
 
struct less
{
    template <typename T>
    bool operator()(const T &lhs, const T &rhs) const
    {
        return lhs < rhs;
    }
};
 
struct less_equal
{
    template <typename T>
    bool operator()(const T &lhs, const T &rhs) const
    {
        return lhs <= rhs;
    }
};
 
template <typename T>
void swap(T &lhs, T &rhs)
{
    T temp = lhs;
    lhs = rhs;
    rhs = temp;
}
 
template <typename I, typename Pred>
I min_element(I begin, I end, const Pred &pred)
{
    I result = begin;
 
    for (I it = begin + 1; it != end; ++it)
        if (pred(*it, *result))
            result = it;
 
    return result;
}
 
template <typename I>
void reverse(I begin, I end)
{
    while (end - begin > 1)
        swap(*begin++, *--end);
}
 
template <typename I>
I unique(I begin, I end)
{
    // fast skip head
    while (end - begin > 1 && *begin != *(begin + 1))
        begin++;
 
    if (begin == end)
        return begin;
 
    // last written element
    I write = begin++;
 
    // merge unique elements
    while (begin != end)
    {
        if (*begin != *write)
            *++write = *begin++;
        else
            begin++;
    }
 
    // past-the-end (write points to live element)
    return write + 1;
}
 
template <typename I>
void copy_backwards(I begin, I end, I target)
{
    while (begin != end)
        *--target = *--end;
}
 
template <typename I, typename Pred, typename T>
void insertion_sort(I begin, I end, const Pred &pred, T *)
{
    assert(begin != end);
 
    for (I it = begin + 1; it != end; ++it)
    {
        T val = *it;
 
        if (pred(val, *begin))
        {
            // move to front
            copy_backwards(begin, it, it + 1);
            *begin = val;
        }
        else
        {
            I hole = it;
 
            // move hole backwards
            while (pred(val, *(hole - 1)))
            {
                *hole = *(hole - 1);
                hole--;
            }
 
            // fill hole with element
            *hole = val;
        }
    }
}
 
// std variant for elements with ==
template <typename I, typename Pred>
void partition(I begin, I middle, I end, const Pred &pred, I *out_eqbeg, I *out_eqend)
{
    I eqbeg = middle, eqend = middle + 1;
 
    // expand equal range
    while (eqbeg != begin && *(eqbeg - 1) == *eqbeg)
        --eqbeg;
    while (eqend != end && *eqend == *eqbeg)
        ++eqend;
 
    // process outer elements
    I ltend = eqbeg, gtbeg = eqend;
 
    for (;;)
    {
        // find the element from the right side that belongs to the left one
        for (; gtbeg != end; ++gtbeg)
            if (!pred(*eqbeg, *gtbeg))
            {
                if (*gtbeg == *eqbeg)
                    swap(*gtbeg, *eqend++);
                else
                    break;
            }
 
        // find the element from the left side that belongs to the right one
        for (; ltend != begin; --ltend)
            if (!pred(*(ltend - 1), *eqbeg))
            {
                if (*eqbeg == *(ltend - 1))
                    swap(*(ltend - 1), *--eqbeg);
                else
                    break;
            }
 
        // scanned all elements
        if (gtbeg == end && ltend == begin)
        {
            *out_eqbeg = eqbeg;
            *out_eqend = eqend;
            return;
        }
 
        // make room for elements by moving equal area
        if (gtbeg == end)
        {
            if (--ltend != --eqbeg)
                swap(*ltend, *eqbeg);
            swap(*eqbeg, *--eqend);
        }
        else if (ltend == begin)
        {
            if (eqend != gtbeg)
                swap(*eqbeg, *eqend);
            ++eqend;
            swap(*gtbeg++, *eqbeg++);
        }
        else
            swap(*gtbeg++, *--ltend);
    }
}
 
template <typename I, typename Pred>
void median3(I first, I middle, I last, const Pred &pred)
{
    if (pred(*middle, *first))
        swap(*middle, *first);
    if (pred(*last, *middle))
        swap(*last, *middle);
    if (pred(*middle, *first))
        swap(*middle, *first);
}
 
template <typename I, typename Pred>
void median(I first, I middle, I last, const Pred &pred)
{
    if (last - first <= 40)
    {
        // median of three for small chunks
        median3(first, middle, last, pred);
    }
    else
    {
        // median of nine
        size_t step = (last - first + 1) / 8;
 
        median3(first, first + step, first + 2 * step, pred);
        median3(middle - step, middle, middle + step, pred);
        median3(last - 2 * step, last - step, last, pred);
        median3(first + step, middle, last - step, pred);
    }
}
 
template <typename I, typename Pred>
void sort(I begin, I end, const Pred &pred)
{
    // sort large chunks
    while (end - begin > 32)
    {
        // find median element
        I middle = begin + (end - begin) / 2;
        median(begin, middle, end - 1, pred);
 
        // partition in three chunks (< = >)
        I eqbeg, eqend;
        partition(begin, middle, end, pred, &eqbeg, &eqend);
 
        // loop on larger half
        if (eqbeg - begin > end - eqend)
        {
            sort(eqend, end, pred);
            end = eqbeg;
        }
        else
        {
            sort(begin, eqbeg, pred);
            begin = eqend;
        }
    }
 
    // insertion sort small chunk
    if (begin != end)
        insertion_sort(begin, end, pred, &*begin);
}
PUGI__NS_END
 
// Allocator used for AST and evaluation stacks
PUGI__NS_BEGIN
struct xpath_memory_block
{
    xpath_memory_block *next;
 
    char data[
#ifdef PUGIXML_MEMORY_XPATH_PAGE_SIZE
        PUGIXML_MEMORY_XPATH_PAGE_SIZE
#else
        4096
#endif
    ];
};
 
class xpath_allocator
{
    xpath_memory_block *_root;
    size_t _root_size;
 
  public:
#ifdef PUGIXML_NO_EXCEPTIONS
    jmp_buf *error_handler;
#endif
 
    xpath_allocator(xpath_memory_block *root, size_t root_size = 0) : _root(root), _root_size(root_size)
    {
#ifdef PUGIXML_NO_EXCEPTIONS
        error_handler = 0;
#endif
    }
 
    void *allocate_nothrow(size_t size)
    {
        const size_t block_capacity = sizeof(_root->data);
 
        // align size so that we're able to store pointers in subsequent blocks
        size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
 
        if (_root_size + size <= block_capacity)
        {
            void *buf = _root->data + _root_size;
            _root_size += size;
            return buf;
        }
        else
        {
            size_t block_data_size = (size > block_capacity) ? size : block_capacity;
            size_t block_size = block_data_size + offsetof(xpath_memory_block, data);
 
            xpath_memory_block *block = static_cast<xpath_memory_block *>(xml_memory::allocate(block_size));
            if (!block)
                return 0;
 
            block->next = _root;
 
            _root = block;
            _root_size = size;
 
            return block->data;
        }
    }
 
    void *allocate(size_t size)
    {
        void *result = allocate_nothrow(size);
 
        if (!result)
        {
#ifdef PUGIXML_NO_EXCEPTIONS
            assert(error_handler);
            longjmp(*error_handler, 1);
#else
            throw std::bad_alloc();
#endif
        }
 
        return result;
    }
 
    void *reallocate(void *ptr, size_t old_size, size_t new_size)
    {
        // align size so that we're able to store pointers in subsequent blocks
        old_size = (old_size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
        new_size = (new_size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
 
        // we can only reallocate the last object
        assert(ptr == 0 || static_cast<char *>(ptr) + old_size == _root->data + _root_size);
 
        // adjust root size so that we have not allocated the object at all
        bool only_object = (_root_size == old_size);
 
        if (ptr)
            _root_size -= old_size;
 
        // allocate a new version (this will obviously reuse the memory if possible)
        void *result = allocate(new_size);
        assert(result);
 
        // we have a new block
        if (result != ptr && ptr)
        {
            // copy old data
            assert(new_size >= old_size);
            memcpy(result, ptr, old_size);
 
            // free the previous page if it had no other objects
            if (only_object)
            {
                assert(_root->data == result);
                assert(_root->next);
 
                xpath_memory_block *next = _root->next->next;
 
                if (next)
                {
                    // deallocate the whole page, unless it was the first one
                    xml_memory::deallocate(_root->next);
                    _root->next = next;
                }
            }
        }
 
        return result;
    }
 
    void revert(const xpath_allocator &state)
    {
        // free all new pages
        xpath_memory_block *cur = _root;
 
        while (cur != state._root)
        {
            xpath_memory_block *next = cur->next;
 
            xml_memory::deallocate(cur);
 
            cur = next;
        }
 
        // restore state
        _root = state._root;
        _root_size = state._root_size;
    }
 
    void release()
    {
        xpath_memory_block *cur = _root;
        assert(cur);
 
        while (cur->next)
        {
            xpath_memory_block *next = cur->next;
 
            xml_memory::deallocate(cur);
 
            cur = next;
        }
    }
};
 
struct xpath_allocator_capture
{
    xpath_allocator_capture(xpath_allocator *alloc) : _target(alloc), _state(*alloc) {}
 
    ~xpath_allocator_capture() { _target->revert(_state); }
 
    xpath_allocator *_target;
    xpath_allocator _state;
};
 
struct xpath_stack
{
    xpath_allocator *result;
    xpath_allocator *temp;
};
 
struct xpath_stack_data
{
    xpath_memory_block blocks[2];
    xpath_allocator result;
    xpath_allocator temp;
    xpath_stack stack;
 
#ifdef PUGIXML_NO_EXCEPTIONS
    jmp_buf error_handler;
#endif
 
    xpath_stack_data() : result(blocks + 0), temp(blocks + 1)
    {
        blocks[0].next = blocks[1].next = 0;
 
        stack.result = &result;
        stack.temp = &temp;
 
#ifdef PUGIXML_NO_EXCEPTIONS
        result.error_handler = temp.error_handler = &error_handler;
#endif
    }
 
    ~xpath_stack_data()
    {
        result.release();
        temp.release();
    }
};
PUGI__NS_END
 
// String class
PUGI__NS_BEGIN
class xpath_string
{
    const char_t *_buffer;
    bool _uses_heap;
 
    static char_t *duplicate_string(const char_t *string, size_t length, xpath_allocator *alloc)
    {
        char_t *result = static_cast<char_t *>(alloc->allocate((length + 1) * sizeof(char_t)));
        assert(result);
 
        memcpy(result, string, length * sizeof(char_t));
        result[length] = 0;
 
        return result;
    }
 
    static char_t *duplicate_string(const char_t *string, xpath_allocator *alloc)
    {
        return duplicate_string(string, strlength(string), alloc);
    }
 
  public:
    xpath_string() : _buffer(PUGIXML_TEXT("")), _uses_heap(false) {}
 
    explicit xpath_string(const char_t *str, xpath_allocator *alloc)
    {
        bool empty_ = (*str == 0);
 
        _buffer = empty_ ? PUGIXML_TEXT("") : duplicate_string(str, alloc);
        _uses_heap = !empty_;
    }
 
    explicit xpath_string(const char_t *str, bool use_heap) : _buffer(str), _uses_heap(use_heap) {}
 
    xpath_string(const char_t *begin, const char_t *end, xpath_allocator *alloc)
    {
        assert(begin <= end);
 
        bool empty_ = (begin == end);
 
        _buffer = empty_ ? PUGIXML_TEXT("") : duplicate_string(begin, static_cast<size_t>(end - begin), alloc);
        _uses_heap = !empty_;
    }
 
    void append(const xpath_string &o, xpath_allocator *alloc)
    {
        // skip empty sources
        if (!*o._buffer)
            return;
 
        // fast append for constant empty target and constant source
        if (!*_buffer && !_uses_heap && !o._uses_heap)
        {
            _buffer = o._buffer;
        }
        else
        {
            // need to make heap copy
            size_t target_length = strlength(_buffer);
            size_t source_length = strlength(o._buffer);
            size_t result_length = target_length + source_length;
 
            // allocate new buffer
            char_t *result = static_cast<char_t *>(alloc->reallocate(_uses_heap ? const_cast<char_t *>(_buffer) : 0,
                                                                     (target_length + 1) * sizeof(char_t),
                                                                     (result_length + 1) * sizeof(char_t)));
            assert(result);
 
            // append first string to the new buffer in case there was no reallocation
            if (!_uses_heap)
                memcpy(result, _buffer, target_length * sizeof(char_t));
 
            // append second string to the new buffer
            memcpy(result + target_length, o._buffer, source_length * sizeof(char_t));
            result[result_length] = 0;
 
            // finalize
            _buffer = result;
            _uses_heap = true;
        }
    }
 
    const char_t *c_str() const { return _buffer; }
 
    size_t length() const { return strlength(_buffer); }
 
    char_t *data(xpath_allocator *alloc)
    {
        // make private heap copy
        if (!_uses_heap)
        {
            _buffer = duplicate_string(_buffer, alloc);
            _uses_heap = true;
        }
 
        return const_cast<char_t *>(_buffer);
    }
 
    bool empty() const { return *_buffer == 0; }
 
    bool operator==(const xpath_string &o) const { return strequal(_buffer, o._buffer); }
 
    bool operator!=(const xpath_string &o) const { return !strequal(_buffer, o._buffer); }
 
    bool uses_heap() const { return _uses_heap; }
};
 
PUGI__FN xpath_string xpath_string_const(const char_t *str)
{
    return xpath_string(str, false);
}
PUGI__NS_END
 
PUGI__NS_BEGIN
PUGI__FN bool starts_with(const char_t *string, const char_t *pattern)
{
    while (*pattern && *string == *pattern)
    {
        string++;
        pattern++;
    }
 
    return *pattern == 0;
}
 
PUGI__FN const char_t *find_char(const char_t *s, char_t c)
{
#ifdef PUGIXML_WCHAR_MODE
    return wcschr(s, c);
#else
    return strchr(s, c);
#endif
}
 
PUGI__FN const char_t *find_substring(const char_t *s, const char_t *p)
{
#ifdef PUGIXML_WCHAR_MODE
    // MSVC6 wcsstr bug workaround (if s is empty it always returns 0)
    return (*p == 0) ? s : wcsstr(s, p);
#else
    return strstr(s, p);
#endif
}
 
// Converts symbol to lower case, if it is an ASCII one
PUGI__FN char_t tolower_ascii(char_t ch)
{
    return static_cast<unsigned int>(ch - 'A') < 26 ? static_cast<char_t>(ch | ' ') : ch;
}
 
PUGI__FN xpath_string string_value(const xpath_node &na, xpath_allocator *alloc)
{
    if (na.attribute())
        return xpath_string_const(na.attribute().value());
    else
    {
        const xml_node &n = na.node();
 
        switch (n.type())
        {
        case node_pcdata:
        case node_cdata:
        case node_comment:
        case node_pi:
            return xpath_string_const(n.value());
 
        case node_document:
        case node_element:
        {
            xpath_string result;
 
            xml_node cur = n.first_child();
 
            while (cur && cur != n)
            {
                if (cur.type() == node_pcdata || cur.type() == node_cdata)
                    result.append(xpath_string_const(cur.value()), alloc);
 
                if (cur.first_child())
                    cur = cur.first_child();
                else if (cur.next_sibling())
                    cur = cur.next_sibling();
                else
                {
                    while (!cur.next_sibling() && cur != n)
                        cur = cur.parent();
 
                    if (cur != n)
                        cur = cur.next_sibling();
                }
            }
 
            return result;
        }
 
        default:
            return xpath_string();
        }
    }
}
 
PUGI__FN unsigned int node_height(xml_node n)
{
    unsigned int result = 0;
 
    while (n)
    {
        ++result;
        n = n.parent();
    }
 
    return result;
}
 
PUGI__FN bool node_is_before(xml_node ln, unsigned int lh, xml_node rn, unsigned int rh)
{
    // normalize heights
    for (unsigned int i = rh; i < lh; i++)
        ln = ln.parent();
    for (unsigned int j = lh; j < rh; j++)
        rn = rn.parent();
 
    // one node is the ancestor of the other
    if (ln == rn)
        return lh < rh;
 
    // find common ancestor
    while (ln.parent() != rn.parent())
    {
        ln = ln.parent();
        rn = rn.parent();
    }
 
    // there is no common ancestor (the shared parent is null), nodes are from different documents
    if (!ln.parent())
        return ln < rn;
 
    // determine sibling order
    for (; ln; ln = ln.next_sibling())
        if (ln == rn)
            return true;
 
    return false;
}
 
PUGI__FN bool node_is_ancestor(xml_node parent, xml_node node)
{
    while (node && node != parent)
        node = node.parent();
 
    return parent && node == parent;
}
 
PUGI__FN const void *document_order(const xpath_node &xnode)
{
    xml_node_struct *node = xnode.node().internal_object();
 
    if (node)
    {
        if (node->name && (node->header & xml_memory_page_name_allocated_mask) == 0)
            return node->name;
        if (node->value && (node->header & xml_memory_page_value_allocated_mask) == 0)
            return node->value;
        return 0;
    }
 
    xml_attribute_struct *attr = xnode.attribute().internal_object();
 
    if (attr)
    {
        if ((attr->header & xml_memory_page_name_allocated_mask) == 0)
            return attr->name;
        if ((attr->header & xml_memory_page_value_allocated_mask) == 0)
            return attr->value;
        return 0;
    }
 
    return 0;
}
 
struct document_order_comparator
{
    bool operator()(const xpath_node &lhs, const xpath_node &rhs) const
    {
        // optimized document order based check
        const void *lo = document_order(lhs);
        const void *ro = document_order(rhs);
 
        if (lo && ro)
            return lo < ro;
 
        // slow comparison
        xml_node ln = lhs.node(), rn = rhs.node();
 
        // compare attributes
        if (lhs.attribute() && rhs.attribute())
        {
            // shared parent
            if (lhs.parent() == rhs.parent())
            {
                // determine sibling order
                for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
                    if (a == rhs.attribute())
                        return true;
 
                return false;
            }
 
            // compare attribute parents
            ln = lhs.parent();
            rn = rhs.parent();
        }
        else if (lhs.attribute())
        {
            // attributes go after the parent element
            if (lhs.parent() == rhs.node())
                return false;
 
            ln = lhs.parent();
        }
        else if (rhs.attribute())
        {
            // attributes go after the parent element
            if (rhs.parent() == lhs.node())
                return true;
 
            rn = rhs.parent();
        }
 
        if (ln == rn)
            return false;
 
        unsigned int lh = node_height(ln);
        unsigned int rh = node_height(rn);
 
        return node_is_before(ln, lh, rn, rh);
    }
};
 
struct duplicate_comparator
{
    bool operator()(const xpath_node &lhs, const xpath_node &rhs) const
    {
        if (lhs.attribute())
            return rhs.attribute() ? lhs.attribute() < rhs.attribute() : true;
        else
            return rhs.attribute() ? false : lhs.node() < rhs.node();
    }
};
 
PUGI__FN double gen_nan()
{
#if defined(__STDC_IEC_559__) || ((FLT_RADIX - 0 == 2) && (FLT_MAX_EXP - 0 == 128) && (FLT_MANT_DIG - 0 == 24))
    union {
        float f;
        uint32_t i;
    } u[sizeof(float) == sizeof(uint32_t) ? 1 : -1];
    u[0].i = 0x7fc00000;
    return u[0].f;
#else
    // fallback
    const volatile double zero = 0.0;
    return zero / zero;
#endif
}
 
PUGI__FN bool is_nan(double value)
{
#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
    return !!_isnan(value);
#elif defined(fpclassify) && defined(FP_NAN)
    return fpclassify(value) == FP_NAN;
#else
    // fallback
    const volatile double v = value;
    return v != v;
#endif
}
 
PUGI__FN const char_t *convert_number_to_string_special(double value)
{
#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
    if (_finite(value))
        return (value == 0) ? PUGIXML_TEXT("0") : 0;
    if (_isnan(value))
        return PUGIXML_TEXT("NaN");
    return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
#elif defined(fpclassify) && defined(FP_NAN) && defined(FP_INFINITE) && defined(FP_ZERO)
    switch (fpclassify(value))
    {
    case FP_NAN:
        return PUGIXML_TEXT("NaN");
 
    case FP_INFINITE:
        return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
 
    case FP_ZERO:
        return PUGIXML_TEXT("0");
 
    default:
        return 0;
    }
#else
    // fallback
    const volatile double v = value;
 
    if (v == 0)
        return PUGIXML_TEXT("0");
    if (v != v)
        return PUGIXML_TEXT("NaN");
    if (v * 2 == v)
        return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
    return 0;
#endif
}
 
PUGI__FN bool convert_number_to_boolean(double value)
{
    return (value != 0 && !is_nan(value));
}
 
PUGI__FN void truncate_zeros(char *begin, char *end)
{
    while (begin != end && end[-1] == '0')
        end--;
 
    *end = 0;
}
 
// gets mantissa digits in the form of 0.xxxxx with 0. implied and the exponent
#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
PUGI__FN void convert_number_to_mantissa_exponent(double value, char *buffer, size_t buffer_size, char **out_mantissa,
                                                  int *out_exponent)
{
    // get base values
    int sign, exponent;
    _ecvt_s(buffer, buffer_size, value, DBL_DIG + 1, &exponent, &sign);
 
    // truncate redundant zeros
    truncate_zeros(buffer, buffer + strlen(buffer));
 
    // fill results
    *out_mantissa = buffer;
    *out_exponent = exponent;
}
#else
PUGI__FN void convert_number_to_mantissa_exponent(double value, char *buffer, size_t buffer_size, char **out_mantissa,
                                                  int *out_exponent)
{
    // get a scientific notation value with IEEE DBL_DIG decimals
    sprintf(buffer, "%.*e", DBL_DIG, value);
    assert(strlen(buffer) < buffer_size);
    (void)!buffer_size;
 
    // get the exponent (possibly negative)
    char *exponent_string = strchr(buffer, 'e');
    assert(exponent_string);
 
    int exponent = atoi(exponent_string + 1);
 
    // extract mantissa string: skip sign
    char *mantissa = buffer[0] == '-' ? buffer + 1 : buffer;
    assert(mantissa[0] != '0' && mantissa[1] == '.');
 
    // divide mantissa by 10 to eliminate integer part
    mantissa[1] = mantissa[0];
    mantissa++;
    exponent++;
 
    // remove extra mantissa digits and zero-terminate mantissa
    truncate_zeros(mantissa, exponent_string);
 
    // fill results
    *out_mantissa = mantissa;
    *out_exponent = exponent;
}
#endif
 
PUGI__FN xpath_string convert_number_to_string(double value, xpath_allocator *alloc)
{
    // try special number conversion
    const char_t *special = convert_number_to_string_special(value);
    if (special)
        return xpath_string_const(special);
 
    // get mantissa + exponent form
    char mantissa_buffer[32];
 
    char *mantissa;
    int exponent;
    convert_number_to_mantissa_exponent(value, mantissa_buffer, sizeof(mantissa_buffer), &mantissa, &exponent);
 
    // allocate a buffer of suitable length for the number
    size_t result_size = strlen(mantissa_buffer) + (exponent > 0 ? exponent : -exponent) + 4;
    char_t *result = static_cast<char_t *>(alloc->allocate(sizeof(char_t) * result_size));
    assert(result);
 
    // make the number!
    char_t *s = result;
 
    // sign
    if (value < 0)
        *s++ = '-';
 
    // integer part
    if (exponent <= 0)
    {
        *s++ = '0';
    }
    else
    {
        while (exponent > 0)
        {
            assert(*mantissa == 0 || static_cast<unsigned int>(static_cast<unsigned int>(*mantissa) - '0') <= 9);
            *s++ = *mantissa ? *mantissa++ : '0';
            exponent--;
        }
    }
 
    // fractional part
    if (*mantissa)
    {
        // decimal point
        *s++ = '.';
 
        // extra zeroes from negative exponent
        while (exponent < 0)
        {
            *s++ = '0';
            exponent++;
        }
 
        // extra mantissa digits
        while (*mantissa)
        {
            assert(static_cast<unsigned int>(*mantissa - '0') <= 9);
            *s++ = *mantissa++;
        }
    }
 
    // zero-terminate
    assert(s < result + result_size);
    *s = 0;
 
    return xpath_string(result, true);
}
 
PUGI__FN bool check_string_to_number_format(const char_t *string)
{
    // parse leading whitespace
    while (PUGI__IS_CHARTYPE(*string, ct_space))
        ++string;
 
    // parse sign
    if (*string == '-')
        ++string;
 
    if (!*string)
        return false;
 
    // if there is no integer part, there should be a decimal part with at least one digit
    if (!PUGI__IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !PUGI__IS_CHARTYPEX(string[1], ctx_digit)))
        return false;
 
    // parse integer part
    while (PUGI__IS_CHARTYPEX(*string, ctx_digit))
        ++string;
 
    // parse decimal part
    if (*string == '.')
    {
        ++string;
 
        while (PUGI__IS_CHARTYPEX(*string, ctx_digit))
            ++string;
    }
 
    // parse trailing whitespace
    while (PUGI__IS_CHARTYPE(*string, ct_space))
        ++string;
 
    return *string == 0;
}
 
PUGI__FN double convert_string_to_number(const char_t *string)
{
    // check string format
    if (!check_string_to_number_format(string))
        return gen_nan();
 
        // parse string
#ifdef PUGIXML_WCHAR_MODE
    return wcstod(string, 0);
#else
    return atof(string);
#endif
}
 
PUGI__FN bool convert_string_to_number_scratch(char_t (&buffer)[32], const char_t *begin, const char_t *end,
                                               double *out_result)
{
    size_t length = static_cast<size_t>(end - begin);
    char_t *scratch = buffer;
 
    if (length >= sizeof(buffer) / sizeof(buffer[0]))
    {
        // need to make dummy on-heap copy
        scratch = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
        if (!scratch)
            return false;
    }
 
    // copy string to zero-terminated buffer and perform conversion
    memcpy(scratch, begin, length * sizeof(char_t));
    scratch[length] = 0;
 
    *out_result = convert_string_to_number(scratch);
 
    // free dummy buffer
    if (scratch != buffer)
        xml_memory::deallocate(scratch);
 
    return true;
}
 
PUGI__FN double round_nearest(double value)
{
    return floor(value + 0.5);
}
 
PUGI__FN double round_nearest_nzero(double value)
{
    // same as round_nearest, but returns -0 for [-0.5, -0]
    // ceil is used to differentiate between +0 and -0 (we return -0 for [-0.5, -0] and +0 for +0)
    return (value >= -0.5 && value <= 0) ? ceil(value) : floor(value + 0.5);
}
 
PUGI__FN const char_t *qualified_name(const xpath_node &node)
{
    return node.attribute() ? node.attribute().name() : node.node().name();
}
 
PUGI__FN const char_t *local_name(const xpath_node &node)
{
    const char_t *name = qualified_name(node);
    const char_t *p = find_char(name, ':');
 
    return p ? p + 1 : name;
}
 
struct namespace_uri_predicate
{
    const char_t *prefix;
    size_t prefix_length;
 
    namespace_uri_predicate(const char_t *name)
    {
        const char_t *pos = find_char(name, ':');
 
        prefix = pos ? name : 0;
        prefix_length = pos ? static_cast<size_t>(pos - name) : 0;
    }
 
    bool operator()(const xml_attribute &a) const
    {
        const char_t *name = a.name();
 
        if (!starts_with(name, PUGIXML_TEXT("xmlns")))
            return false;
 
        return prefix ? name[5] == ':' && strequalrange(name + 6, prefix, prefix_length) : name[5] == 0;
    }
};
 
PUGI__FN const char_t *namespace_uri(const xml_node &node)
{
    namespace_uri_predicate pred = node.name();
 
    xml_node p = node;
 
    while (p)
    {
        xml_attribute a = p.find_attribute(pred);
 
        if (a)
            return a.value();
 
        p = p.parent();
    }
 
    return PUGIXML_TEXT("");
}
 
PUGI__FN const char_t *namespace_uri(const xml_attribute &attr, const xml_node &parent)
{
    namespace_uri_predicate pred = attr.name();
 
    // Default namespace does not apply to attributes
    if (!pred.prefix)
        return PUGIXML_TEXT("");
 
    xml_node p = parent;
 
    while (p)
    {
        xml_attribute a = p.find_attribute(pred);
 
        if (a)
            return a.value();
 
        p = p.parent();
    }
 
    return PUGIXML_TEXT("");
}
 
PUGI__FN const char_t *namespace_uri(const xpath_node &node)
{
    return node.attribute() ? namespace_uri(node.attribute(), node.parent()) : namespace_uri(node.node());
}
 
PUGI__FN void normalize_space(char_t *buffer)
{
    char_t *write = buffer;
 
    for (char_t *it = buffer; *it;)
    {
        char_t ch = *it++;
 
        if (PUGI__IS_CHARTYPE(ch, ct_space))
        {
            // replace whitespace sequence with single space
            while (PUGI__IS_CHARTYPE(*it, ct_space))
                it++;
 
            // avoid leading spaces
            if (write != buffer)
                *write++ = ' ';
        }
        else
            *write++ = ch;
    }
 
    // remove trailing space
    if (write != buffer && PUGI__IS_CHARTYPE(write[-1], ct_space))
        write--;
 
    // zero-terminate
    *write = 0;
}
 
PUGI__FN void translate(char_t *buffer, const char_t *from, const char_t *to)
{
    size_t to_length = strlength(to);
 
    char_t *write = buffer;
 
    while (*buffer)
    {
        PUGI__DMC_VOLATILE char_t ch = *buffer++;
 
        const char_t *pos = find_char(from, ch);
 
        if (!pos)
            *write++ = ch; // do not process
        else if (static_cast<size_t>(pos - from) < to_length)
            *write++ = to[pos - from]; // replace
    }
 
    // zero-terminate
    *write = 0;
}
 
struct xpath_variable_boolean : xpath_variable
{
    xpath_variable_boolean() : value(false) {}
 
    bool value;
    char_t name[1];
};
 
struct xpath_variable_number : xpath_variable
{
    xpath_variable_number() : value(0) {}
 
    double value;
    char_t name[1];
};
 
struct xpath_variable_string : xpath_variable
{
    xpath_variable_string() : value(0) {}
 
    ~xpath_variable_string()
    {
        if (value)
            xml_memory::deallocate(value);
    }
 
    char_t *value;
    char_t name[1];
};
 
struct xpath_variable_node_set : xpath_variable
{
    xpath_node_set value;
    char_t name[1];
};
 
static const xpath_node_set dummy_node_set;
 
PUGI__FN unsigned int hash_string(const char_t *str)
{
    // Jenkins one-at-a-time hash (http://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time)
    unsigned int result = 0;
 
    while (*str)
    {
        result += static_cast<unsigned int>(*str++);
        result += result << 10;
        result ^= result >> 6;
    }
 
    result += result << 3;
    result ^= result >> 11;
    result += result << 15;
 
    return result;
}
 
template <typename T>
PUGI__FN T *new_xpath_variable(const char_t *name)
{
    size_t length = strlength(name);
    if (length == 0)
        return 0; // empty variable names are invalid
 
    // $$ we can't use offsetof(T, name) because T is non-POD, so we just allocate additional length characters
    void *memory = xml_memory::allocate(sizeof(T) + length * sizeof(char_t));
    if (!memory)
        return 0;
 
    T *result = new (memory) T();
 
    memcpy(result->name, name, (length + 1) * sizeof(char_t));
 
    return result;
}
 
PUGI__FN xpath_variable *new_xpath_variable(xpath_value_type type, const char_t *name)
{
    switch (type)
    {
    case xpath_type_node_set:
        return new_xpath_variable<xpath_variable_node_set>(name);
 
    case xpath_type_number:
        return new_xpath_variable<xpath_variable_number>(name);
 
    case xpath_type_string:
        return new_xpath_variable<xpath_variable_string>(name);
 
    case xpath_type_boolean:
        return new_xpath_variable<xpath_variable_boolean>(name);
 
    default:
        return 0;
    }
}
 
template <typename T>
PUGI__FN void delete_xpath_variable(T *var)
{
    var->~T();
    xml_memory::deallocate(var);
}
 
PUGI__FN void delete_xpath_variable(xpath_value_type type, xpath_variable *var)
{
    switch (type)
    {
    case xpath_type_node_set:
        delete_xpath_variable(static_cast<xpath_variable_node_set *>(var));
        break;
 
    case xpath_type_number:
        delete_xpath_variable(static_cast<xpath_variable_number *>(var));
        break;
 
    case xpath_type_string:
        delete_xpath_variable(static_cast<xpath_variable_string *>(var));
        break;
 
    case xpath_type_boolean:
        delete_xpath_variable(static_cast<xpath_variable_boolean *>(var));
        break;
 
    default:
        assert(!"Invalid variable type");
    }
}
 
PUGI__FN xpath_variable *get_variable_scratch(char_t (&buffer)[32], xpath_variable_set *set, const char_t *begin,
                                              const char_t *end)
{
    size_t length = static_cast<size_t>(end - begin);
    char_t *scratch = buffer;
 
    if (length >= sizeof(buffer) / sizeof(buffer[0]))
    {
        // need to make dummy on-heap copy
        scratch = static_cast<char_t *>(xml_memory::allocate((length + 1) * sizeof(char_t)));
        if (!scratch)
            return 0;
    }
 
    // copy string to zero-terminated buffer and perform lookup
    memcpy(scratch, begin, length * sizeof(char_t));
    scratch[length] = 0;
 
    xpath_variable *result = set->get(scratch);
 
    // free dummy buffer
    if (scratch != buffer)
        xml_memory::deallocate(scratch);
 
    return result;
}
PUGI__NS_END
 
// Internal node set class
PUGI__NS_BEGIN
PUGI__FN xpath_node_set::type_t xpath_sort(xpath_node *begin, xpath_node *end, xpath_node_set::type_t type, bool rev)
{
    xpath_node_set::type_t order = rev ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;
 
    if (type == xpath_node_set::type_unsorted)
    {
        sort(begin, end, document_order_comparator());
 
        type = xpath_node_set::type_sorted;
    }
 
    if (type != order)
        reverse(begin, end);
 
    return order;
}
 
PUGI__FN xpath_node xpath_first(const xpath_node *begin, const xpath_node *end, xpath_node_set::type_t type)
{
    if (begin == end)
        return xpath_node();
 
    switch (type)
    {
    case xpath_node_set::type_sorted:
        return *begin;
 
    case xpath_node_set::type_sorted_reverse:
        return *(end - 1);
 
    case xpath_node_set::type_unsorted:
        return *min_element(begin, end, document_order_comparator());
 
    default:
        assert(!"Invalid node set type");
        return xpath_node();
    }
}
 
class xpath_node_set_raw
{
    xpath_node_set::type_t _type;
 
    xpath_node *_begin;
    xpath_node *_end;
    xpath_node *_eos;
 
  public:
    xpath_node_set_raw() : _type(xpath_node_set::type_unsorted), _begin(0), _end(0), _eos(0) {}
 
    xpath_node *begin() const { return _begin; }
 
    xpath_node *end() const { return _end; }
 
    bool empty() const { return _begin == _end; }
 
    size_t size() const { return static_cast<size_t>(_end - _begin); }
 
    xpath_node first() const { return xpath_first(_begin, _end, _type); }
 
    void push_back(const xpath_node &node, xpath_allocator *alloc)
    {
        if (_end == _eos)
        {
            size_t capacity = static_cast<size_t>(_eos - _begin);
 
            // get new capacity (1.5x rule)
            size_t new_capacity = capacity + capacity / 2 + 1;
 
            // reallocate the old array or allocate a new one
            xpath_node *data = static_cast<xpath_node *>(
                alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
            assert(data);
 
            // finalize
            _begin = data;
            _end = data + capacity;
            _eos = data + new_capacity;
        }
 
        *_end++ = node;
    }
 
    void append(const xpath_node *begin_, const xpath_node *end_, xpath_allocator *alloc)
    {
        size_t size_ = static_cast<size_t>(_end - _begin);
        size_t capacity = static_cast<size_t>(_eos - _begin);
        size_t count = static_cast<size_t>(end_ - begin_);
 
        if (size_ + count > capacity)
        {
            // reallocate the old array or allocate a new one
            xpath_node *data = static_cast<xpath_node *>(
                alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size_ + count) * sizeof(xpath_node)));
            assert(data);
 
            // finalize
            _begin = data;
            _end = data + size_;
            _eos = data + size_ + count;
        }
 
        memcpy(_end, begin_, count * sizeof(xpath_node));
        _end += count;
    }
 
    void sort_do() { _type = xpath_sort(_begin, _end, _type, false); }
 
    void truncate(xpath_node *pos)
    {
        assert(_begin <= pos && pos <= _end);
 
        _end = pos;
    }
 
    void remove_duplicates()
    {
        if (_type == xpath_node_set::type_unsorted)
            sort(_begin, _end, duplicate_comparator());
 
        _end = unique(_begin, _end);
    }
 
    xpath_node_set::type_t type() const { return _type; }
 
    void set_type(xpath_node_set::type_t value) { _type = value; }
};
PUGI__NS_END
 
PUGI__NS_BEGIN
struct xpath_context
{
    xpath_node n;
    size_t position, size;
 
    xpath_context(const xpath_node &n_, size_t position_, size_t size_) : n(n_), position(position_), size(size_) {}
};
 
enum lexeme_t
{
    lex_none = 0,
    lex_equal,
    lex_not_equal,
    lex_less,
    lex_greater,
    lex_less_or_equal,
    lex_greater_or_equal,
    lex_plus,
    lex_minus,
    lex_multiply,
    lex_union,
    lex_var_ref,
    lex_open_brace,
    lex_close_brace,
    lex_quoted_string,
    lex_number,
    lex_slash,
    lex_double_slash,
    lex_open_square_brace,
    lex_close_square_brace,
    lex_string,
    lex_comma,
    lex_axis_attribute,
    lex_dot,
    lex_double_dot,
    lex_double_colon,
    lex_eof
};
 
struct xpath_lexer_string
{
    const char_t *begin;
    const char_t *end;
 
    xpath_lexer_string() : begin(0), end(0) {}
 
    bool operator==(const char_t *other) const
    {
        size_t length = static_cast<size_t>(end - begin);
 
        return strequalrange(other, begin, length);
    }
};
 
class xpath_lexer
{
    const char_t *_cur;
    const char_t *_cur_lexeme_pos;
    xpath_lexer_string _cur_lexeme_contents;
 
    lexeme_t _cur_lexeme;
 
  public:
    explicit xpath_lexer(const char_t *query) : _cur(query) { next(); }
 
    const char_t *state() const { return _cur; }
 
    void next()
    {
        const char_t *cur = _cur;
 
        while (PUGI__IS_CHARTYPE(*cur, ct_space))
            ++cur;
 
        // save lexeme position for error reporting
        _cur_lexeme_pos = cur;
 
        switch (*cur)
        {
        case 0:
            _cur_lexeme = lex_eof;
            break;
 
        case '>':
            if (*(cur + 1) == '=')
            {
                cur += 2;
                _cur_lexeme = lex_greater_or_equal;
            }
            else
            {
                cur += 1;
                _cur_lexeme = lex_greater;
            }
            break;
 
        case '<':
            if (*(cur + 1) == '=')
            {
                cur += 2;
                _cur_lexeme = lex_less_or_equal;
            }
            else
            {
                cur += 1;
                _cur_lexeme = lex_less;
            }
            break;
 
        case '!':
            if (*(cur + 1) == '=')
            {
                cur += 2;
                _cur_lexeme = lex_not_equal;
            }
            else
            {
                _cur_lexeme = lex_none;
            }
            break;
 
        case '=':
            cur += 1;
            _cur_lexeme = lex_equal;
 
            break;
 
        case '+':
            cur += 1;
            _cur_lexeme = lex_plus;
 
            break;
 
        case '-':
            cur += 1;
            _cur_lexeme = lex_minus;
 
            break;
 
        case '*':
            cur += 1;
            _cur_lexeme = lex_multiply;
 
            break;
 
        case '|':
            cur += 1;
            _cur_lexeme = lex_union;
 
            break;
 
        case '$':
            cur += 1;
 
            if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
            {
                _cur_lexeme_contents.begin = cur;
 
                while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
                    cur++;
 
                if (cur[0] == ':' && PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
                {
                    cur++; // :
 
                    while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
                        cur++;
                }
 
                _cur_lexeme_contents.end = cur;
 
                _cur_lexeme = lex_var_ref;
            }
            else
            {
                _cur_lexeme = lex_none;
            }
 
            break;
 
        case '(':
            cur += 1;
            _cur_lexeme = lex_open_brace;
 
            break;
 
        case ')':
            cur += 1;
            _cur_lexeme = lex_close_brace;
 
            break;
 
        case '[':
            cur += 1;
            _cur_lexeme = lex_open_square_brace;
 
            break;
 
        case ']':
            cur += 1;
            _cur_lexeme = lex_close_square_brace;
 
            break;
 
        case ',':
            cur += 1;
            _cur_lexeme = lex_comma;
 
            break;
 
        case '/':
            if (*(cur + 1) == '/')
            {
                cur += 2;
                _cur_lexeme = lex_double_slash;
            }
            else
            {
                cur += 1;
                _cur_lexeme = lex_slash;
            }
            break;
 
        case '.':
            if (*(cur + 1) == '.')
            {
                cur += 2;
                _cur_lexeme = lex_double_dot;
            }
            else if (PUGI__IS_CHARTYPEX(*(cur + 1), ctx_digit))
            {
                _cur_lexeme_contents.begin = cur; // .
 
                ++cur;
 
                while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
                    cur++;
 
                _cur_lexeme_contents.end = cur;
 
                _cur_lexeme = lex_number;
            }
            else
            {
                cur += 1;
                _cur_lexeme = lex_dot;
            }
            break;
 
        case '@':
            cur += 1;
            _cur_lexeme = lex_axis_attribute;
 
            break;
 
        case '"':
        case '\'':
        {
            char_t terminator = *cur;
 
            ++cur;
 
            _cur_lexeme_contents.begin = cur;
            while (*cur && *cur != terminator)
                cur++;
            _cur_lexeme_contents.end = cur;
 
            if (!*cur)
                _cur_lexeme = lex_none;
            else
            {
                cur += 1;
                _cur_lexeme = lex_quoted_string;
            }
 
            break;
        }
 
        case ':':
            if (*(cur + 1) == ':')
            {
                cur += 2;
                _cur_lexeme = lex_double_colon;
            }
            else
            {
                _cur_lexeme = lex_none;
            }
            break;
 
        default:
            if (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
            {
                _cur_lexeme_contents.begin = cur;
 
                while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
                    cur++;
 
                if (*cur == '.')
                {
                    cur++;
 
                    while (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
                        cur++;
                }
 
                _cur_lexeme_contents.end = cur;
 
                _cur_lexeme = lex_number;
            }
            else if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
            {
                _cur_lexeme_contents.begin = cur;
 
                while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
                    cur++;
 
                if (cur[0] == ':')
                {
                    if (cur[1] == '*') // namespace test ncname:*
                    {
                        cur += 2; // :*
                    }
                    else if (PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
                    {
                        cur++; // :
 
                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol))
                            cur++;
                    }
                }
 
                _cur_lexeme_contents.end = cur;
 
                _cur_lexeme = lex_string;
            }
            else
            {
                _cur_lexeme = lex_none;
            }
        }
 
        _cur = cur;
    }
 
    lexeme_t current() const { return _cur_lexeme; }
 
    const char_t *current_pos() const { return _cur_lexeme_pos; }
 
    const xpath_lexer_string &contents() const
    {
        assert(_cur_lexeme == lex_var_ref || _cur_lexeme == lex_number || _cur_lexeme == lex_string ||
               _cur_lexeme == lex_quoted_string);
 
        return _cur_lexeme_contents;
    }
};
 
enum ast_type_t
{
    ast_unknown,
    ast_op_or,                  // left or right
    ast_op_and,                 // left and right
    ast_op_equal,               // left = right
    ast_op_not_equal,           // left != right
    ast_op_less,                // left < right
    ast_op_greater,             // left > right
    ast_op_less_or_equal,       // left <= right
    ast_op_greater_or_equal,    // left >= right
    ast_op_add,                 // left + right
    ast_op_subtract,            // left - right
    ast_op_multiply,            // left * right
    ast_op_divide,              // left / right
    ast_op_mod,                 // left % right
    ast_op_negate,              // left - right
    ast_op_union,               // left | right
    ast_predicate,              // apply predicate to set; next points to next predicate
    ast_filter,                 // select * from left where right
    ast_filter_posinv,          // select * from left where right; proximity position invariant
    ast_string_constant,        // string constant
    ast_number_constant,        // number constant
    ast_variable,               // variable
    ast_func_last,              // last()
    ast_func_position,          // position()
    ast_func_count,             // count(left)
    ast_func_id,                // id(left)
    ast_func_local_name_0,      // local-name()
    ast_func_local_name_1,      // local-name(left)
    ast_func_namespace_uri_0,   // namespace-uri()
    ast_func_namespace_uri_1,   // namespace-uri(left)
    ast_func_name_0,            // name()
    ast_func_name_1,            // name(left)
    ast_func_string_0,          // string()
    ast_func_string_1,          // string(left)
    ast_func_concat,            // concat(left, right, siblings)
    ast_func_starts_with,       // starts_with(left, right)
    ast_func_contains,          // contains(left, right)
    ast_func_substring_before,  // substring-before(left, right)
    ast_func_substring_after,   // substring-after(left, right)
    ast_func_substring_2,       // substring(left, right)
    ast_func_substring_3,       // substring(left, right, third)
    ast_func_string_length_0,   // string-length()
    ast_func_string_length_1,   // string-length(left)
    ast_func_normalize_space_0, // normalize-space()
    ast_func_normalize_space_1, // normalize-space(left)
    ast_func_translate,         // translate(left, right, third)
    ast_func_boolean,           // boolean(left)
    ast_func_not,               // not(left)
    ast_func_true,              // true()
    ast_func_false,             // false()
    ast_func_lang,              // lang(left)
    ast_func_number_0,          // number()
    ast_func_number_1,          // number(left)
    ast_func_sum,               // sum(left)
    ast_func_floor,             // floor(left)
    ast_func_ceiling,           // ceiling(left)
    ast_func_round,             // round(left)
    ast_step,                   // process set left with step
    ast_step_root               // select root node
};
 
enum axis_t
{
    axis_ancestor,
    axis_ancestor_or_self,
    axis_attribute,
    axis_child,
    axis_descendant,
    axis_descendant_or_self,
    axis_following,
    axis_following_sibling,
    axis_namespace,
    axis_parent,
    axis_preceding,
    axis_preceding_sibling,
    axis_self
};
 
enum nodetest_t
{
    nodetest_none,
    nodetest_name,
    nodetest_type_node,
    nodetest_type_comment,
    nodetest_type_pi,
    nodetest_type_text,
    nodetest_pi,
    nodetest_all,
    nodetest_all_in_namespace
};
 
template <axis_t N>
struct axis_to_type
{
    static const axis_t axis;
};
 
template <axis_t N>
const axis_t axis_to_type<N>::axis = N;
 
class xpath_ast_node
{
  private:
    // node type
    char _type;
    char _rettype;
 
    // for ast_step / ast_predicate
    char _axis;
    char _test;
 
    // tree node structure
    xpath_ast_node *_left;
    xpath_ast_node *_right;
    xpath_ast_node *_next;
 
    union {
        // value for ast_string_constant
        const char_t *string;
        // value for ast_number_constant
        double number;
        // variable for ast_variable
        xpath_variable *variable;
        // node test for ast_step (node name/namespace/node type/pi target)
        const char_t *nodetest;
    } _data;
 
    xpath_ast_node(const xpath_ast_node &);
    xpath_ast_node &operator=(const xpath_ast_node &);
 
    template <class Comp>
    static bool compare_eq(xpath_ast_node *lhs, xpath_ast_node *rhs, const xpath_context &c, const xpath_stack &stack,
                           const Comp &comp)
    {
        xpath_value_type lt = lhs->rettype(), rt = rhs->rettype();
 
        if (lt != xpath_type_node_set && rt != xpath_type_node_set)
        {
            if (lt == xpath_type_boolean || rt == xpath_type_boolean)
                return comp(lhs->eval_boolean(c, stack), rhs->eval_boolean(c, stack));
            else if (lt == xpath_type_number || rt == xpath_type_number)
                return comp(lhs->eval_number(c, stack), rhs->eval_number(c, stack));
            else if (lt == xpath_type_string || rt == xpath_type_string)
            {
                xpath_allocator_capture cr(stack.result);
 
                xpath_string ls = lhs->eval_string(c, stack);
                xpath_string rs = rhs->eval_string(c, stack);
 
                return comp(ls, rs);
            }
        }
        else if (lt == xpath_type_node_set && rt == xpath_type_node_set)
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
            xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
 
            for (const xpath_node *li = ls.begin(); li != ls.end(); ++li)
                for (const xpath_node *ri = rs.begin(); ri != rs.end(); ++ri)
                {
                    xpath_allocator_capture cri(stack.result);
 
                    if (comp(string_value(*li, stack.result), string_value(*ri, stack.result)))
                        return true;
                }
 
            return false;
        }
        else
        {
            if (lt == xpath_type_node_set)
            {
                swap(lhs, rhs);
                swap(lt, rt);
            }
 
            if (lt == xpath_type_boolean)
                return comp(lhs->eval_boolean(c, stack), rhs->eval_boolean(c, stack));
            else if (lt == xpath_type_number)
            {
                xpath_allocator_capture cr(stack.result);
 
                double l = lhs->eval_number(c, stack);
                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
 
                for (const xpath_node *ri = rs.begin(); ri != rs.end(); ++ri)
                {
                    xpath_allocator_capture cri(stack.result);
 
                    if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
                        return true;
                }
 
                return false;
            }
            else if (lt == xpath_type_string)
            {
                xpath_allocator_capture cr(stack.result);
 
                xpath_string l = lhs->eval_string(c, stack);
                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
 
                for (const xpath_node *ri = rs.begin(); ri != rs.end(); ++ri)
                {
                    xpath_allocator_capture cri(stack.result);
 
                    if (comp(l, string_value(*ri, stack.result)))
                        return true;
                }
 
                return false;
            }
        }
 
        assert(!"Wrong types");
        return false;
    }
 
    template <class Comp>
    static bool compare_rel(xpath_ast_node *lhs, xpath_ast_node *rhs, const xpath_context &c, const xpath_stack &stack,
                            const Comp &comp)
    {
        xpath_value_type lt = lhs->rettype(), rt = rhs->rettype();
 
        if (lt != xpath_type_node_set && rt != xpath_type_node_set)
            return comp(lhs->eval_number(c, stack), rhs->eval_number(c, stack));
        else if (lt == xpath_type_node_set && rt == xpath_type_node_set)
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
            xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
 
            for (const xpath_node *li = ls.begin(); li != ls.end(); ++li)
            {
                xpath_allocator_capture cri(stack.result);
 
                double l = convert_string_to_number(string_value(*li, stack.result).c_str());
 
                for (const xpath_node *ri = rs.begin(); ri != rs.end(); ++ri)
                {
                    xpath_allocator_capture crii(stack.result);
 
                    if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
                        return true;
                }
            }
 
            return false;
        }
        else if (lt != xpath_type_node_set && rt == xpath_type_node_set)
        {
            xpath_allocator_capture cr(stack.result);
 
            double l = lhs->eval_number(c, stack);
            xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
 
            for (const xpath_node *ri = rs.begin(); ri != rs.end(); ++ri)
            {
                xpath_allocator_capture cri(stack.result);
 
                if (comp(l, convert_string_to_number(string_value(*ri, stack.result).c_str())))
                    return true;
            }
 
            return false;
        }
        else if (lt == xpath_type_node_set && rt != xpath_type_node_set)
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
            double r = rhs->eval_number(c, stack);
 
            for (const xpath_node *li = ls.begin(); li != ls.end(); ++li)
            {
                xpath_allocator_capture cri(stack.result);
 
                if (comp(convert_string_to_number(string_value(*li, stack.result).c_str()), r))
                    return true;
            }
 
            return false;
        }
        else
        {
            assert(!"Wrong types");
            return false;
        }
    }
 
    void apply_predicate(xpath_node_set_raw &ns, size_t first, xpath_ast_node *expr, const xpath_stack &stack)
    {
        assert(ns.size() >= first);
 
        size_t i = 1;
        size_t size = ns.size() - first;
 
        xpath_node *last = ns.begin() + first;
 
        // remove_if... or well, sort of
        for (xpath_node *it = last; it != ns.end(); ++it, ++i)
        {
            xpath_context c(*it, i, size);
 
            if (expr->rettype() == xpath_type_number)
            {
                if (expr->eval_number(c, stack) == i)
                    *last++ = *it;
            }
            else if (expr->eval_boolean(c, stack))
                *last++ = *it;
        }
 
        ns.truncate(last);
    }
 
    void apply_predicates(xpath_node_set_raw &ns, size_t first, const xpath_stack &stack)
    {
        if (ns.size() == first)
            return;
 
        for (xpath_ast_node *pred = _right; pred; pred = pred->_next)
        {
            apply_predicate(ns, first, pred->_left, stack);
        }
    }
 
    void step_push(xpath_node_set_raw &ns, const xml_attribute &a, const xml_node &parent, xpath_allocator *alloc)
    {
        if (!a)
            return;
 
        const char_t *name = a.name();
 
        // There are no attribute nodes corresponding to attributes that declare namespaces
        // That is, "xmlns:..." or "xmlns"
        if (starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':'))
            return;
 
        switch (_test)
        {
        case nodetest_name:
            if (strequal(name, _data.nodetest))
                ns.push_back(xpath_node(a, parent), alloc);
            break;
 
        case nodetest_type_node:
        case nodetest_all:
            ns.push_back(xpath_node(a, parent), alloc);
            break;
 
        case nodetest_all_in_namespace:
            if (starts_with(name, _data.nodetest))
                ns.push_back(xpath_node(a, parent), alloc);
            break;
 
        default:;
        }
    }
 
    void step_push(xpath_node_set_raw &ns, const xml_node &n, xpath_allocator *alloc)
    {
        if (!n)
            return;
 
        switch (_test)
        {
        case nodetest_name:
            if (n.type() == node_element && strequal(n.name(), _data.nodetest))
                ns.push_back(n, alloc);
            break;
 
        case nodetest_type_node:
            ns.push_back(n, alloc);
            break;
 
        case nodetest_type_comment:
            if (n.type() == node_comment)
                ns.push_back(n, alloc);
            break;
 
        case nodetest_type_text:
            if (n.type() == node_pcdata || n.type() == node_cdata)
                ns.push_back(n, alloc);
            break;
 
        case nodetest_type_pi:
            if (n.type() == node_pi)
                ns.push_back(n, alloc);
            break;
 
        case nodetest_pi:
            if (n.type() == node_pi && strequal(n.name(), _data.nodetest))
                ns.push_back(n, alloc);
            break;
 
        case nodetest_all:
            if (n.type() == node_element)
                ns.push_back(n, alloc);
            break;
 
        case nodetest_all_in_namespace:
            if (n.type() == node_element && starts_with(n.name(), _data.nodetest))
                ns.push_back(n, alloc);
            break;
 
        default:
            assert(!"Unknown axis");
        }
    }
 
    template <class T>
    void step_fill(xpath_node_set_raw &ns, const xml_node &n, xpath_allocator *alloc, T)
    {
        const axis_t axis = T::axis;
 
        switch (axis)
        {
        case axis_attribute:
        {
            for (xml_attribute a = n.first_attribute(); a; a = a.next_attribute())
                step_push(ns, a, n, alloc);
 
            break;
        }
 
        case axis_child:
        {
            for (xml_node c = n.first_child(); c; c = c.next_sibling())
                step_push(ns, c, alloc);
 
            break;
        }
 
        case axis_descendant:
        case axis_descendant_or_self:
        {
            if (axis == axis_descendant_or_self)
                step_push(ns, n, alloc);
 
            xml_node cur = n.first_child();
 
            while (cur && cur != n)
            {
                step_push(ns, cur, alloc);
 
                if (cur.first_child())
                    cur = cur.first_child();
                else if (cur.next_sibling())
                    cur = cur.next_sibling();
                else
                {
                    while (!cur.next_sibling() && cur != n)
                        cur = cur.parent();
 
                    if (cur != n)
                        cur = cur.next_sibling();
                }
            }
 
            break;
        }
 
        case axis_following_sibling:
        {
            for (xml_node c = n.next_sibling(); c; c = c.next_sibling())
                step_push(ns, c, alloc);
 
            break;
        }
 
        case axis_preceding_sibling:
        {
            for (xml_node c = n.previous_sibling(); c; c = c.previous_sibling())
                step_push(ns, c, alloc);
 
            break;
        }
 
        case axis_following:
        {
            xml_node cur = n;
 
            // exit from this node so that we don't include descendants
            while (cur && !cur.next_sibling())
                cur = cur.parent();
            cur = cur.next_sibling();
 
            for (;;)
            {
                step_push(ns, cur, alloc);
 
                if (cur.first_child())
                    cur = cur.first_child();
                else if (cur.next_sibling())
                    cur = cur.next_sibling();
                else
                {
                    while (cur && !cur.next_sibling())
                        cur = cur.parent();
                    cur = cur.next_sibling();
 
                    if (!cur)
                        break;
                }
            }
 
            break;
        }
 
        case axis_preceding:
        {
            xml_node cur = n;
 
            while (cur && !cur.previous_sibling())
                cur = cur.parent();
            cur = cur.previous_sibling();
 
            for (;;)
            {
                if (cur.last_child())
                    cur = cur.last_child();
                else
                {
                    // leaf node, can't be ancestor
                    step_push(ns, cur, alloc);
 
                    if (cur.previous_sibling())
                        cur = cur.previous_sibling();
                    else
                    {
                        do
                        {
                            cur = cur.parent();
                            if (!cur)
                                break;
 
                            if (!node_is_ancestor(cur, n))
                                step_push(ns, cur, alloc);
                        } while (!cur.previous_sibling());
 
                        cur = cur.previous_sibling();
 
                        if (!cur)
                            break;
                    }
                }
            }
 
            break;
        }
 
        case axis_ancestor:
        case axis_ancestor_or_self:
        {
            if (axis == axis_ancestor_or_self)
                step_push(ns, n, alloc);
 
            xml_node cur = n.parent();
 
            while (cur)
            {
                step_push(ns, cur, alloc);
 
                cur = cur.parent();
            }
 
            break;
        }
 
        case axis_self:
        {
            step_push(ns, n, alloc);
 
            break;
        }
 
        case axis_parent:
        {
            if (n.parent())
                step_push(ns, n.parent(), alloc);
 
            break;
        }
 
        default:
            assert(!"Unimplemented axis");
        }
    }
 
    template <class T>
    void step_fill(xpath_node_set_raw &ns, const xml_attribute &a, const xml_node &p, xpath_allocator *alloc, T v)
    {
        const axis_t axis = T::axis;
 
        switch (axis)
        {
        case axis_ancestor:
        case axis_ancestor_or_self:
        {
            if (axis == axis_ancestor_or_self &&
                _test == nodetest_type_node) // reject attributes based on principal node type test
                step_push(ns, a, p, alloc);
 
            xml_node cur = p;
 
            while (cur)
            {
                step_push(ns, cur, alloc);
 
                cur = cur.parent();
            }
 
            break;
        }
 
        case axis_descendant_or_self:
        case axis_self:
        {
            if (_test == nodetest_type_node) // reject attributes based on principal node type test
                step_push(ns, a, p, alloc);
 
            break;
        }
 
        case axis_following:
        {
            xml_node cur = p;
 
            for (;;)
            {
                if (cur.first_child())
                    cur = cur.first_child();
                else if (cur.next_sibling())
                    cur = cur.next_sibling();
                else
                {
                    while (cur && !cur.next_sibling())
                        cur = cur.parent();
                    cur = cur.next_sibling();
 
                    if (!cur)
                        break;
                }
 
                step_push(ns, cur, alloc);
            }
 
            break;
        }
 
        case axis_parent:
        {
            step_push(ns, p, alloc);
 
            break;
        }
 
        case axis_preceding:
        {
            // preceding:: axis does not include attribute nodes and attribute ancestors (they are the same as parent's
            // ancestors), so we can reuse node preceding
            step_fill(ns, p, alloc, v);
            break;
        }
 
        default:
            assert(!"Unimplemented axis");
        }
    }
 
    template <class T>
    xpath_node_set_raw step_do(const xpath_context &c, const xpath_stack &stack, T v)
    {
        const axis_t axis = T::axis;
        bool attributes =
            (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self ||
             axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);
 
        xpath_node_set_raw ns;
        ns.set_type((axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding ||
                     axis == axis_preceding_sibling)
                        ? xpath_node_set::type_sorted_reverse
                        : xpath_node_set::type_sorted);
 
        if (_left)
        {
            xpath_node_set_raw s = _left->eval_node_set(c, stack);
 
            // self axis preserves the original order
            if (axis == axis_self)
                ns.set_type(s.type());
 
            for (const xpath_node *it = s.begin(); it != s.end(); ++it)
            {
                size_t size = ns.size();
 
                // in general, all axes generate elements in a particular order, but there is no order guarantee if axis
                // is applied to two nodes
                if (axis != axis_self && size != 0)
                    ns.set_type(xpath_node_set::type_unsorted);
 
                if (it->node())
                    step_fill(ns, it->node(), stack.result, v);
                else if (attributes)
                    step_fill(ns, it->attribute(), it->parent(), stack.result, v);
 
                apply_predicates(ns, size, stack);
            }
        }
        else
        {
            if (c.n.node())
                step_fill(ns, c.n.node(), stack.result, v);
            else if (attributes)
                step_fill(ns, c.n.attribute(), c.n.parent(), stack.result, v);
 
            apply_predicates(ns, 0, stack);
        }
 
        // child, attribute and self axes always generate unique set of nodes
        // for other axis, if the set stayed sorted, it stayed unique because the traversal algorithms do not visit the
        // same node twice
        if (axis != axis_child && axis != axis_attribute && axis != axis_self &&
            ns.type() == xpath_node_set::type_unsorted)
            ns.remove_duplicates();
 
        return ns;
    }
 
  public:
    xpath_ast_node(ast_type_t type, xpath_value_type rettype_, const char_t *value)
        : _type(static_cast<char>(type))
        , _rettype(static_cast<char>(rettype_))
        , _axis(0)
        , _test(0)
        , _left(0)
        , _right(0)
        , _next(0)
    {
        assert(type == ast_string_constant);
        _data.string = value;
    }
 
    xpath_ast_node(ast_type_t type, xpath_value_type rettype_, double value)
        : _type(static_cast<char>(type))
        , _rettype(static_cast<char>(rettype_))
        , _axis(0)
        , _test(0)
        , _left(0)
        , _right(0)
        , _next(0)
    {
        assert(type == ast_number_constant);
        _data.number = value;
    }
 
    xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_variable *value)
        : _type(static_cast<char>(type))
        , _rettype(static_cast<char>(rettype_))
        , _axis(0)
        , _test(0)
        , _left(0)
        , _right(0)
        , _next(0)
    {
        assert(type == ast_variable);
        _data.variable = value;
    }
 
    xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_ast_node *left = 0, xpath_ast_node *right = 0)
        : _type(static_cast<char>(type))
        , _rettype(static_cast<char>(rettype_))
        , _axis(0)
        , _test(0)
        , _left(left)
        , _right(right)
        , _next(0)
    {
    }
 
    xpath_ast_node(ast_type_t type, xpath_ast_node *left, axis_t axis, nodetest_t test, const char_t *contents)
        : _type(static_cast<char>(type))
        , _rettype(xpath_type_node_set)
        , _axis(static_cast<char>(axis))
        , _test(static_cast<char>(test))
        , _left(left)
        , _right(0)
        , _next(0)
    {
        _data.nodetest = contents;
    }
 
    void set_next(xpath_ast_node *value) { _next = value; }
 
    void set_right(xpath_ast_node *value) { _right = value; }
 
    bool eval_boolean(const xpath_context &c, const xpath_stack &stack)
    {
        switch (_type)
        {
        case ast_op_or:
            return _left->eval_boolean(c, stack) || _right->eval_boolean(c, stack);
 
        case ast_op_and:
            return _left->eval_boolean(c, stack) && _right->eval_boolean(c, stack);
 
        case ast_op_equal:
            return compare_eq(_left, _right, c, stack, equal_to());
 
        case ast_op_not_equal:
            return compare_eq(_left, _right, c, stack, not_equal_to());
 
        case ast_op_less:
            return compare_rel(_left, _right, c, stack, less());
 
        case ast_op_greater:
            return compare_rel(_right, _left, c, stack, less());
 
        case ast_op_less_or_equal:
            return compare_rel(_left, _right, c, stack, less_equal());
 
        case ast_op_greater_or_equal:
            return compare_rel(_right, _left, c, stack, less_equal());
 
        case ast_func_starts_with:
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_string lr = _left->eval_string(c, stack);
            xpath_string rr = _right->eval_string(c, stack);
 
            return starts_with(lr.c_str(), rr.c_str());
        }
 
        case ast_func_contains:
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_string lr = _left->eval_string(c, stack);
            xpath_string rr = _right->eval_string(c, stack);
 
            return find_substring(lr.c_str(), rr.c_str()) != 0;
        }
 
        case ast_func_boolean:
            return _left->eval_boolean(c, stack);
 
        case ast_func_not:
            return !_left->eval_boolean(c, stack);
 
        case ast_func_true:
            return true;
 
        case ast_func_false:
            return false;
 
        case ast_func_lang:
        {
            if (c.n.attribute())
                return false;
 
            xpath_allocator_capture cr(stack.result);
 
            xpath_string lang = _left->eval_string(c, stack);
 
            for (xml_node n = c.n.node(); n; n = n.parent())
            {
                xml_attribute a = n.attribute(PUGIXML_TEXT("xml:lang"));
 
                if (a)
                {
                    const char_t *value = a.value();
 
                    // strnicmp / strncasecmp is not portable
                    for (const char_t *lit = lang.c_str(); *lit; ++lit)
                    {
                        if (tolower_ascii(*lit) != tolower_ascii(*value))
                            return false;
                        ++value;
                    }
 
                    return *value == 0 || *value == '-';
                }
            }
 
            return false;
        }
 
        case ast_variable:
        {
            assert(_rettype == _data.variable->type());
 
            if (_rettype == xpath_type_boolean)
                return _data.variable->get_boolean();
 
            // fallthrough to type conversion
        }
 
        default:
        {
            switch (_rettype)
            {
            case xpath_type_number:
                return convert_number_to_boolean(eval_number(c, stack));
 
            case xpath_type_string:
            {
                xpath_allocator_capture cr(stack.result);
 
                return !eval_string(c, stack).empty();
            }
 
            case xpath_type_node_set:
            {
                xpath_allocator_capture cr(stack.result);
 
                return !eval_node_set(c, stack).empty();
            }
 
            default:
                assert(!"Wrong expression for return type boolean");
                return false;
            }
        }
        }
    }
 
    double eval_number(const xpath_context &c, const xpath_stack &stack)
    {
        switch (_type)
        {
        case ast_op_add:
            return _left->eval_number(c, stack) + _right->eval_number(c, stack);
 
        case ast_op_subtract:
            return _left->eval_number(c, stack) - _right->eval_number(c, stack);
 
        case ast_op_multiply:
            return _left->eval_number(c, stack) * _right->eval_number(c, stack);
 
        case ast_op_divide:
            return _left->eval_number(c, stack) / _right->eval_number(c, stack);
 
        case ast_op_mod:
            return fmod(_left->eval_number(c, stack), _right->eval_number(c, stack));
 
        case ast_op_negate:
            return -_left->eval_number(c, stack);
 
        case ast_number_constant:
            return _data.number;
 
        case ast_func_last:
            return static_cast<double>(c.size);
 
        case ast_func_position:
            return static_cast<double>(c.position);
 
        case ast_func_count:
        {
            xpath_allocator_capture cr(stack.result);
 
            return static_cast<double>(_left->eval_node_set(c, stack).size());
        }
 
        case ast_func_string_length_0:
        {
            xpath_allocator_capture cr(stack.result);
 
            return static_cast<double>(string_value(c.n, stack.result).length());
        }
 
        case ast_func_string_length_1:
        {
            xpath_allocator_capture cr(stack.result);
 
            return static_cast<double>(_left->eval_string(c, stack).length());
        }
 
        case ast_func_number_0:
        {
            xpath_allocator_capture cr(stack.result);
 
            return convert_string_to_number(string_value(c.n, stack.result).c_str());
        }
 
        case ast_func_number_1:
            return _left->eval_number(c, stack);
 
        case ast_func_sum:
        {
            xpath_allocator_capture cr(stack.result);
 
            double r = 0;
 
            xpath_node_set_raw ns = _left->eval_node_set(c, stack);
 
            for (const xpath_node *it = ns.begin(); it != ns.end(); ++it)
            {
                xpath_allocator_capture cri(stack.result);
 
                r += convert_string_to_number(string_value(*it, stack.result).c_str());
            }
 
            return r;
        }
 
        case ast_func_floor:
        {
            double r = _left->eval_number(c, stack);
 
            return r == r ? floor(r) : r;
        }
 
        case ast_func_ceiling:
        {
            double r = _left->eval_number(c, stack);
 
            return r == r ? ceil(r) : r;
        }
 
        case ast_func_round:
            return round_nearest_nzero(_left->eval_number(c, stack));
 
        case ast_variable:
        {
            assert(_rettype == _data.variable->type());
 
            if (_rettype == xpath_type_number)
                return _data.variable->get_number();
 
            // fallthrough to type conversion
        }
 
        default:
        {
            switch (_rettype)
            {
            case xpath_type_boolean:
                return eval_boolean(c, stack) ? 1 : 0;
 
            case xpath_type_string:
            {
                xpath_allocator_capture cr(stack.result);
 
                return convert_string_to_number(eval_string(c, stack).c_str());
            }
 
            case xpath_type_node_set:
            {
                xpath_allocator_capture cr(stack.result);
 
                return convert_string_to_number(eval_string(c, stack).c_str());
            }
 
            default:
                assert(!"Wrong expression for return type number");
                return 0;
            }
        }
        }
    }
 
    xpath_string eval_string_concat(const xpath_context &c, const xpath_stack &stack)
    {
        assert(_type == ast_func_concat);
 
        xpath_allocator_capture ct(stack.temp);
 
        // count the string number
        size_t count = 1;
        for (xpath_ast_node *nc = _right; nc; nc = nc->_next)
            count++;
 
        // gather all strings
        xpath_string static_buffer[4];
        xpath_string *buffer = static_buffer;
 
        // allocate on-heap for large concats
        if (count > sizeof(static_buffer) / sizeof(static_buffer[0]))
        {
            buffer = static_cast<xpath_string *>(stack.temp->allocate(count * sizeof(xpath_string)));
            assert(buffer);
        }
 
        // evaluate all strings to temporary stack
        xpath_stack swapped_stack = { stack.temp, stack.result };
 
        buffer[0] = _left->eval_string(c, swapped_stack);
 
        size_t pos = 1;
        for (xpath_ast_node *n = _right; n; n = n->_next, ++pos)
            buffer[pos] = n->eval_string(c, swapped_stack);
        assert(pos == count);
 
        // get total length
        size_t length = 0;
        for (size_t i = 0; i < count; ++i)
            length += buffer[i].length();
 
        // create final string
        char_t *result = static_cast<char_t *>(stack.result->allocate((length + 1) * sizeof(char_t)));
        assert(result);
 
        char_t *ri = result;
 
        for (size_t j = 0; j < count; ++j)
            for (const char_t *bi = buffer[j].c_str(); *bi; ++bi)
                *ri++ = *bi;
 
        *ri = 0;
 
        return xpath_string(result, true);
    }
 
    xpath_string eval_string(const xpath_context &c, const xpath_stack &stack)
    {
        switch (_type)
        {
        case ast_string_constant:
            return xpath_string_const(_data.string);
 
        case ast_func_local_name_0:
        {
            xpath_node na = c.n;
 
            return xpath_string_const(local_name(na));
        }
 
        case ast_func_local_name_1:
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ns = _left->eval_node_set(c, stack);
            xpath_node na = ns.first();
 
            return xpath_string_const(local_name(na));
        }
 
        case ast_func_name_0:
        {
            xpath_node na = c.n;
 
            return xpath_string_const(qualified_name(na));
        }
 
        case ast_func_name_1:
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ns = _left->eval_node_set(c, stack);
            xpath_node na = ns.first();
 
            return xpath_string_const(qualified_name(na));
        }
 
        case ast_func_namespace_uri_0:
        {
            xpath_node na = c.n;
 
            return xpath_string_const(namespace_uri(na));
        }
 
        case ast_func_namespace_uri_1:
        {
            xpath_allocator_capture cr(stack.result);
 
            xpath_node_set_raw ns = _left->eval_node_set(c, stack);
            xpath_node na = ns.first();
 
            return xpath_string_const(namespace_uri(na));
        }
 
        case ast_func_string_0:
            return string_value(c.n, stack.result);
 
        case ast_func_string_1:
            return _left->eval_string(c, stack);
 
        case ast_func_concat:
            return eval_string_concat(c, stack);
 
        case ast_func_substring_before:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_string s = _left->eval_string(c, swapped_stack);
            xpath_string p = _right->eval_string(c, swapped_stack);
 
            const char_t *pos = find_substring(s.c_str(), p.c_str());
 
            return pos ? xpath_string(s.c_str(), pos, stack.result) : xpath_string();
        }
 
        case ast_func_substring_after:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_string s = _left->eval_string(c, swapped_stack);
            xpath_string p = _right->eval_string(c, swapped_stack);
 
            const char_t *pos = find_substring(s.c_str(), p.c_str());
            if (!pos)
                return xpath_string();
 
            const char_t *result = pos + p.length();
 
            return s.uses_heap() ? xpath_string(result, stack.result) : xpath_string_const(result);
        }
 
        case ast_func_substring_2:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_string s = _left->eval_string(c, swapped_stack);
            size_t s_length = s.length();
 
            double first = round_nearest(_right->eval_number(c, stack));
 
            if (is_nan(first))
                return xpath_string(); // NaN
            else if (first >= s_length + 1)
                return xpath_string();
 
            size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
            assert(1 <= pos && pos <= s_length + 1);
 
            const char_t *rbegin = s.c_str() + (pos - 1);
 
            return s.uses_heap() ? xpath_string(rbegin, stack.result) : xpath_string_const(rbegin);
        }
 
        case ast_func_substring_3:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_string s = _left->eval_string(c, swapped_stack);
            size_t s_length = s.length();
 
            double first = round_nearest(_right->eval_number(c, stack));
            double last = first + round_nearest(_right->_next->eval_number(c, stack));
 
            if (is_nan(first) || is_nan(last))
                return xpath_string();
            else if (first >= s_length + 1)
                return xpath_string();
            else if (first >= last)
                return xpath_string();
            else if (last < 1)
                return xpath_string();
 
            size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
            size_t end = last >= s_length + 1 ? s_length + 1 : static_cast<size_t>(last);
 
            assert(1 <= pos && pos <= end && end <= s_length + 1);
            const char_t *rbegin = s.c_str() + (pos - 1);
            const char_t *rend = s.c_str() + (end - 1);
 
            return (end == s_length + 1 && !s.uses_heap()) ? xpath_string_const(rbegin)
                                                           : xpath_string(rbegin, rend, stack.result);
        }
 
        case ast_func_normalize_space_0:
        {
            xpath_string s = string_value(c.n, stack.result);
 
            normalize_space(s.data(stack.result));
 
            return s;
        }
 
        case ast_func_normalize_space_1:
        {
            xpath_string s = _left->eval_string(c, stack);
 
            normalize_space(s.data(stack.result));
 
            return s;
        }
 
        case ast_func_translate:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_string s = _left->eval_string(c, stack);
            xpath_string from = _right->eval_string(c, swapped_stack);
            xpath_string to = _right->_next->eval_string(c, swapped_stack);
 
            translate(s.data(stack.result), from.c_str(), to.c_str());
 
            return s;
        }
 
        case ast_variable:
        {
            assert(_rettype == _data.variable->type());
 
            if (_rettype == xpath_type_string)
                return xpath_string_const(_data.variable->get_string());
 
            // fallthrough to type conversion
        }
 
        default:
        {
            switch (_rettype)
            {
            case xpath_type_boolean:
                return xpath_string_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
 
            case xpath_type_number:
                return convert_number_to_string(eval_number(c, stack), stack.result);
 
            case xpath_type_node_set:
            {
                xpath_allocator_capture cr(stack.temp);
 
                xpath_stack swapped_stack = { stack.temp, stack.result };
 
                xpath_node_set_raw ns = eval_node_set(c, swapped_stack);
                return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result);
            }
 
            default:
                assert(!"Wrong expression for return type string");
                return xpath_string();
            }
        }
        }
    }
 
    xpath_node_set_raw eval_node_set(const xpath_context &c, const xpath_stack &stack)
    {
        switch (_type)
        {
        case ast_op_union:
        {
            xpath_allocator_capture cr(stack.temp);
 
            xpath_stack swapped_stack = { stack.temp, stack.result };
 
            xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack);
            xpath_node_set_raw rs = _right->eval_node_set(c, stack);
 
            // we can optimize merging two sorted sets, but this is a very rare operation, so don't bother
            rs.set_type(xpath_node_set::type_unsorted);
 
            rs.append(ls.begin(), ls.end(), stack.result);
            rs.remove_duplicates();
 
            return rs;
        }
 
        case ast_filter:
        case ast_filter_posinv:
        {
            xpath_node_set_raw set = _left->eval_node_set(c, stack);
 
            // either expression is a number or it contains position() call; sort by document order
            if (_type == ast_filter)
                set.sort_do();
 
            apply_predicate(set, 0, _right, stack);
 
            return set;
        }
 
        case ast_func_id:
            return xpath_node_set_raw();
 
        case ast_step:
        {
            switch (_axis)
            {
            case axis_ancestor:
                return step_do(c, stack, axis_to_type<axis_ancestor>());
 
            case axis_ancestor_or_self:
                return step_do(c, stack, axis_to_type<axis_ancestor_or_self>());
 
            case axis_attribute:
                return step_do(c, stack, axis_to_type<axis_attribute>());
 
            case axis_child:
                return step_do(c, stack, axis_to_type<axis_child>());
 
            case axis_descendant:
                return step_do(c, stack, axis_to_type<axis_descendant>());
 
            case axis_descendant_or_self:
                return step_do(c, stack, axis_to_type<axis_descendant_or_self>());
 
            case axis_following:
                return step_do(c, stack, axis_to_type<axis_following>());
 
            case axis_following_sibling:
                return step_do(c, stack, axis_to_type<axis_following_sibling>());
 
            case axis_namespace:
                // namespaced axis is not supported
                return xpath_node_set_raw();
 
            case axis_parent:
                return step_do(c, stack, axis_to_type<axis_parent>());
 
            case axis_preceding:
                return step_do(c, stack, axis_to_type<axis_preceding>());
 
            case axis_preceding_sibling:
                return step_do(c, stack, axis_to_type<axis_preceding_sibling>());
 
            case axis_self:
                return step_do(c, stack, axis_to_type<axis_self>());
 
            default:
                assert(!"Unknown axis");
                return xpath_node_set_raw();
            }
        }
 
        case ast_step_root:
        {
            assert(!_right); // root step can't have any predicates
 
            xpath_node_set_raw ns;
 
            ns.set_type(xpath_node_set::type_sorted);
 
            if (c.n.node())
                ns.push_back(c.n.node().root(), stack.result);
            else if (c.n.attribute())
                ns.push_back(c.n.parent().root(), stack.result);
 
            return ns;
        }
 
        case ast_variable:
        {
            assert(_rettype == _data.variable->type());
 
            if (_rettype == xpath_type_node_set)
            {
                const xpath_node_set &s = _data.variable->get_node_set();
 
                xpath_node_set_raw ns;
 
                ns.set_type(s.type());
                ns.append(s.begin(), s.end(), stack.result);
 
                return ns;
            }
 
            // fallthrough to type conversion
        }
 
        default:
            assert(!"Wrong expression for return type node set");
            return xpath_node_set_raw();
        }
    }
 
    bool is_posinv()
    {
        switch (_type)
        {
        case ast_func_position:
            return false;
 
        case ast_string_constant:
        case ast_number_constant:
        case ast_variable:
            return true;
 
        case ast_step:
        case ast_step_root:
            return true;
 
        case ast_predicate:
        case ast_filter:
        case ast_filter_posinv:
            return true;
 
        default:
            if (_left && !_left->is_posinv())
                return false;
 
            for (xpath_ast_node *n = _right; n; n = n->_next)
                if (!n->is_posinv())
                    return false;
 
            return true;
        }
    }
 
    xpath_value_type rettype() const { return static_cast<xpath_value_type>(_rettype); }
};
 
struct xpath_parser
{
    xpath_allocator *_alloc;
    xpath_lexer _lexer;
 
    const char_t *_query;
    xpath_variable_set *_variables;
 
    xpath_parse_result *_result;
 
    char_t _scratch[32];
 
#ifdef PUGIXML_NO_EXCEPTIONS
    jmp_buf _error_handler;
#endif
 
    void throw_error(const char *message)
    {
        _result->error = message;
        _result->offset = _lexer.current_pos() - _query;
 
#ifdef PUGIXML_NO_EXCEPTIONS
        longjmp(_error_handler, 1);
#else
        throw xpath_exception(*_result);
#endif
    }
 
    void throw_error_oom()
    {
#ifdef PUGIXML_NO_EXCEPTIONS
        throw_error("Out of memory");
#else
        throw std::bad_alloc();
#endif
    }
 
    void *alloc_node()
    {
        void *result = _alloc->allocate_nothrow(sizeof(xpath_ast_node));
 
        if (!result)
            throw_error_oom();
 
        return result;
    }
 
    const char_t *alloc_string(const xpath_lexer_string &value)
    {
        if (value.begin)
        {
            size_t length = static_cast<size_t>(value.end - value.begin);
 
            char_t *c = static_cast<char_t *>(_alloc->allocate_nothrow((length + 1) * sizeof(char_t)));
            if (!c)
                throw_error_oom();
            assert(c); // workaround for clang static analysis
 
            memcpy(c, value.begin, length * sizeof(char_t));
            c[length] = 0;
 
            return c;
        }
        else
            return 0;
    }
 
    xpath_ast_node *parse_function_helper(ast_type_t type0, ast_type_t type1, size_t argc, xpath_ast_node *args[2])
    {
        assert(argc <= 1);
 
        if (argc == 1 && args[0]->rettype() != xpath_type_node_set)
            throw_error("Function has to be applied to node set");
 
        return new (alloc_node()) xpath_ast_node(argc == 0 ? type0 : type1, xpath_type_string, args[0]);
    }
 
    xpath_ast_node *parse_function(const xpath_lexer_string &name, size_t argc, xpath_ast_node *args[2])
    {
        switch (name.begin[0])
        {
        case 'b':
            if (name == PUGIXML_TEXT("boolean") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_boolean, xpath_type_boolean, args[0]);
 
            break;
 
        case 'c':
            if (name == PUGIXML_TEXT("count") && argc == 1)
            {
                if (args[0]->rettype() != xpath_type_node_set)
                    throw_error("Function has to be applied to node set");
                return new (alloc_node()) xpath_ast_node(ast_func_count, xpath_type_number, args[0]);
            }
            else if (name == PUGIXML_TEXT("contains") && argc == 2)
                return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_boolean, args[0], args[1]);
            else if (name == PUGIXML_TEXT("concat") && argc >= 2)
                return new (alloc_node()) xpath_ast_node(ast_func_concat, xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("ceiling") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_ceiling, xpath_type_number, args[0]);
 
            break;
 
        case 'f':
            if (name == PUGIXML_TEXT("false") && argc == 0)
                return new (alloc_node()) xpath_ast_node(ast_func_false, xpath_type_boolean);
            else if (name == PUGIXML_TEXT("floor") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_floor, xpath_type_number, args[0]);
 
            break;
 
        case 'i':
            if (name == PUGIXML_TEXT("id") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_id, xpath_type_node_set, args[0]);
 
            break;
 
        case 'l':
            if (name == PUGIXML_TEXT("last") && argc == 0)
                return new (alloc_node()) xpath_ast_node(ast_func_last, xpath_type_number);
            else if (name == PUGIXML_TEXT("lang") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_lang, xpath_type_boolean, args[0]);
            else if (name == PUGIXML_TEXT("local-name") && argc <= 1)
                return parse_function_helper(ast_func_local_name_0, ast_func_local_name_1, argc, args);
 
            break;
 
        case 'n':
            if (name == PUGIXML_TEXT("name") && argc <= 1)
                return parse_function_helper(ast_func_name_0, ast_func_name_1, argc, args);
            else if (name == PUGIXML_TEXT("namespace-uri") && argc <= 1)
                return parse_function_helper(ast_func_namespace_uri_0, ast_func_namespace_uri_1, argc, args);
            else if (name == PUGIXML_TEXT("normalize-space") && argc <= 1)
                return new (alloc_node())
                    xpath_ast_node(argc == 0 ? ast_func_normalize_space_0 : ast_func_normalize_space_1,
                                   xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("not") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_not, xpath_type_boolean, args[0]);
            else if (name == PUGIXML_TEXT("number") && argc <= 1)
                return new (alloc_node())
                    xpath_ast_node(argc == 0 ? ast_func_number_0 : ast_func_number_1, xpath_type_number, args[0]);
 
            break;
 
        case 'p':
            if (name == PUGIXML_TEXT("position") && argc == 0)
                return new (alloc_node()) xpath_ast_node(ast_func_position, xpath_type_number);
 
            break;
 
        case 'r':
            if (name == PUGIXML_TEXT("round") && argc == 1)
                return new (alloc_node()) xpath_ast_node(ast_func_round, xpath_type_number, args[0]);
 
            break;
 
        case 's':
            if (name == PUGIXML_TEXT("string") && argc <= 1)
                return new (alloc_node())
                    xpath_ast_node(argc == 0 ? ast_func_string_0 : ast_func_string_1, xpath_type_string, args[0]);
            else if (name == PUGIXML_TEXT("string-length") && argc <= 1)
                return new (alloc_node()) xpath_ast_node(
                    argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1, xpath_type_number, args[0]);
            else if (name == PUGIXML_TEXT("starts-with") && argc == 2)
                return new (alloc_node()) xpath_ast_node(ast_func_starts_with, xpath_type_boolean, args[0], args[1]);
            else if (name == PUGIXML_TEXT("substring-before") && argc == 2)
                return new (alloc_node())
                    xpath_ast_node(ast_func_substring_before, xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("substring-after") && argc == 2)
                return new (alloc_node()) xpath_ast_node(ast_func_substring_after, xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("substring") && (argc == 2 || argc == 3))
                return new (alloc_node()) xpath_ast_node(argc == 2 ? ast_func_substring_2 : ast_func_substring_3,
                                                         xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("sum") && argc == 1)
            {
                if (args[0]->rettype() != xpath_type_node_set)
                    throw_error("Function has to be applied to node set");
                return new (alloc_node()) xpath_ast_node(ast_func_sum, xpath_type_number, args[0]);
            }
 
            break;
 
        case 't':
            if (name == PUGIXML_TEXT("translate") && argc == 3)
                return new (alloc_node()) xpath_ast_node(ast_func_translate, xpath_type_string, args[0], args[1]);
            else if (name == PUGIXML_TEXT("true") && argc == 0)
                return new (alloc_node()) xpath_ast_node(ast_func_true, xpath_type_boolean);
 
            break;
 
        default:
            break;
        }
 
        throw_error("Unrecognized function or wrong parameter count");
 
        return 0;
    }
 
    axis_t parse_axis_name(const xpath_lexer_string &name, bool &specified)
    {
        specified = true;
 
        switch (name.begin[0])
        {
        case 'a':
            if (name == PUGIXML_TEXT("ancestor"))
                return axis_ancestor;
            else if (name == PUGIXML_TEXT("ancestor-or-self"))
                return axis_ancestor_or_self;
            else if (name == PUGIXML_TEXT("attribute"))
                return axis_attribute;
 
            break;
 
        case 'c':
            if (name == PUGIXML_TEXT("child"))
                return axis_child;
 
            break;
 
        case 'd':
            if (name == PUGIXML_TEXT("descendant"))
                return axis_descendant;
            else if (name == PUGIXML_TEXT("descendant-or-self"))
                return axis_descendant_or_self;
 
            break;
 
        case 'f':
            if (name == PUGIXML_TEXT("following"))
                return axis_following;
            else if (name == PUGIXML_TEXT("following-sibling"))
                return axis_following_sibling;
 
            break;
 
        case 'n':
            if (name == PUGIXML_TEXT("namespace"))
                return axis_namespace;
 
            break;
 
        case 'p':
            if (name == PUGIXML_TEXT("parent"))
                return axis_parent;
            else if (name == PUGIXML_TEXT("preceding"))
                return axis_preceding;
            else if (name == PUGIXML_TEXT("preceding-sibling"))
                return axis_preceding_sibling;
 
            break;
 
        case 's':
            if (name == PUGIXML_TEXT("self"))
                return axis_self;
 
            break;
 
        default:
            break;
        }
 
        specified = false;
        return axis_child;
    }
 
    nodetest_t parse_node_test_type(const xpath_lexer_string &name)
    {
        switch (name.begin[0])
        {
        case 'c':
            if (name == PUGIXML_TEXT("comment"))
                return nodetest_type_comment;
 
            break;
 
        case 'n':
            if (name == PUGIXML_TEXT("node"))
                return nodetest_type_node;
 
            break;
 
        case 'p':
            if (name == PUGIXML_TEXT("processing-instruction"))
                return nodetest_type_pi;
 
            break;
 
        case 't':
            if (name == PUGIXML_TEXT("text"))
                return nodetest_type_text;
 
            break;
 
        default:
            break;
        }
 
        return nodetest_none;
    }
 
    // PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
    xpath_ast_node *parse_primary_expression()
    {
        switch (_lexer.current())
        {
        case lex_var_ref:
        {
            xpath_lexer_string name = _lexer.contents();
 
            if (!_variables)
                throw_error("Unknown variable: variable set is not provided");
 
            xpath_variable *var = get_variable_scratch(_scratch, _variables, name.begin, name.end);
 
            if (!var)
                throw_error("Unknown variable: variable set does not contain the given name");
 
            _lexer.next();
 
            return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
        }
 
        case lex_open_brace:
        {
            _lexer.next();
 
            xpath_ast_node *n = parse_expression();
 
            if (_lexer.current() != lex_close_brace)
                throw_error("Unmatched braces");
 
            _lexer.next();
 
            return n;
        }
 
        case lex_quoted_string:
        {
            const char_t *value = alloc_string(_lexer.contents());
 
            xpath_ast_node *n = new (alloc_node()) xpath_ast_node(ast_string_constant, xpath_type_string, value);
            _lexer.next();
 
            return n;
        }
 
        case lex_number:
        {
            double value = 0;
 
            if (!convert_string_to_number_scratch(_scratch, _lexer.contents().begin, _lexer.contents().end, &value))
                throw_error_oom();
 
            xpath_ast_node *n = new (alloc_node()) xpath_ast_node(ast_number_constant, xpath_type_number, value);
            _lexer.next();
 
            return n;
        }
 
        case lex_string:
        {
            xpath_ast_node *args[2] = { 0 };
            size_t argc = 0;
 
            xpath_lexer_string function = _lexer.contents();
            _lexer.next();
 
            xpath_ast_node *last_arg = 0;
 
            if (_lexer.current() != lex_open_brace)
                throw_error("Unrecognized function call");
            _lexer.next();
 
            if (_lexer.current() != lex_close_brace)
                args[argc++] = parse_expression();
 
            while (_lexer.current() != lex_close_brace)
            {
                if (_lexer.current() != lex_comma)
                    throw_error("No comma between function arguments");
                _lexer.next();
 
                xpath_ast_node *n = parse_expression();
 
                if (argc < 2)
                    args[argc] = n;
                else
                    last_arg->set_next(n);
 
                argc++;
                last_arg = n;
            }
 
            _lexer.next();
 
            return parse_function(function, argc, args);
        }
 
        default:
            throw_error("Unrecognizable primary expression");
 
            return 0;
        }
    }
 
    // FilterExpr ::= PrimaryExpr | FilterExpr Predicate
    // Predicate ::= '[' PredicateExpr ']'
    // PredicateExpr ::= Expr
    xpath_ast_node *parse_filter_expression()
    {
        xpath_ast_node *n = parse_primary_expression();
 
        while (_lexer.current() == lex_open_square_brace)
        {
            _lexer.next();
 
            xpath_ast_node *expr = parse_expression();
 
            if (n->rettype() != xpath_type_node_set)
                throw_error("Predicate has to be applied to node set");
 
            bool posinv = expr->rettype() != xpath_type_number && expr->is_posinv();
 
            n = new (alloc_node())
                xpath_ast_node(posinv ? ast_filter_posinv : ast_filter, xpath_type_node_set, n, expr);
 
            if (_lexer.current() != lex_close_square_brace)
                throw_error("Unmatched square brace");
 
            _lexer.next();
        }
 
        return n;
    }
 
    // Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
    // AxisSpecifier ::= AxisName '::' | '@'?
    // NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
    // NameTest ::= '*' | NCName ':' '*' | QName
    // AbbreviatedStep ::= '.' | '..'
    xpath_ast_node *parse_step(xpath_ast_node *set)
    {
        if (set && set->rettype() != xpath_type_node_set)
            throw_error("Step has to be applied to node set");
 
        bool axis_specified = false;
        axis_t axis = axis_child; // implied child axis
 
        if (_lexer.current() == lex_axis_attribute)
        {
            axis = axis_attribute;
            axis_specified = true;
 
            _lexer.next();
        }
        else if (_lexer.current() == lex_dot)
        {
            _lexer.next();
 
            return new (alloc_node()) xpath_ast_node(ast_step, set, axis_self, nodetest_type_node, 0);
        }
        else if (_lexer.current() == lex_double_dot)
        {
            _lexer.next();
 
            return new (alloc_node()) xpath_ast_node(ast_step, set, axis_parent, nodetest_type_node, 0);
        }
 
        nodetest_t nt_type = nodetest_none;
        xpath_lexer_string nt_name;
 
        if (_lexer.current() == lex_string)
        {
            // node name test
            nt_name = _lexer.contents();
            _lexer.next();
 
            // was it an axis name?
            if (_lexer.current() == lex_double_colon)
            {
                // parse axis name
                if (axis_specified)
                    throw_error("Two axis specifiers in one step");
 
                axis = parse_axis_name(nt_name, axis_specified);
 
                if (!axis_specified)
                    throw_error("Unknown axis");
 
                // read actual node test
                _lexer.next();
 
                if (_lexer.current() == lex_multiply)
                {
                    nt_type = nodetest_all;
                    nt_name = xpath_lexer_string();
                    _lexer.next();
                }
                else if (_lexer.current() == lex_string)
                {
                    nt_name = _lexer.contents();
                    _lexer.next();
                }
                else
                    throw_error("Unrecognized node test");
            }
 
            if (nt_type == nodetest_none)
            {
                // node type test or processing-instruction
                if (_lexer.current() == lex_open_brace)
                {
                    _lexer.next();
 
                    if (_lexer.current() == lex_close_brace)
                    {
                        _lexer.next();
 
                        nt_type = parse_node_test_type(nt_name);
 
                        if (nt_type == nodetest_none)
                            throw_error("Unrecognized node type");
 
                        nt_name = xpath_lexer_string();
                    }
                    else if (nt_name == PUGIXML_TEXT("processing-instruction"))
                    {
                        if (_lexer.current() != lex_quoted_string)
                            throw_error("Only literals are allowed as arguments to processing-instruction()");
 
                        nt_type = nodetest_pi;
                        nt_name = _lexer.contents();
                        _lexer.next();
 
                        if (_lexer.current() != lex_close_brace)
                            throw_error("Unmatched brace near processing-instruction()");
                        _lexer.next();
                    }
                    else
                        throw_error("Unmatched brace near node type test");
                }
                // QName or NCName:*
                else
                {
                    if (nt_name.end - nt_name.begin > 2 && nt_name.end[-2] == ':' && nt_name.end[-1] == '*') // NCName:*
                    {
                        nt_name.end--; // erase *
 
                        nt_type = nodetest_all_in_namespace;
                    }
                    else
                        nt_type = nodetest_name;
                }
            }
        }
        else if (_lexer.current() == lex_multiply)
        {
            nt_type = nodetest_all;
            _lexer.next();
        }
        else
            throw_error("Unrecognized node test");
 
        xpath_ast_node *n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, alloc_string(nt_name));
 
        xpath_ast_node *last = 0;
 
        while (_lexer.current() == lex_open_square_brace)
        {
            _lexer.next();
 
            xpath_ast_node *expr = parse_expression();
 
            xpath_ast_node *pred = new (alloc_node()) xpath_ast_node(ast_predicate, xpath_type_node_set, expr);
 
            if (_lexer.current() != lex_close_square_brace)
                throw_error("Unmatched square brace");
            _lexer.next();
 
            if (last)
                last->set_next(pred);
            else
                n->set_right(pred);
 
            last = pred;
        }
 
        return n;
    }
 
    // RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
    xpath_ast_node *parse_relative_location_path(xpath_ast_node *set)
    {
        xpath_ast_node *n = parse_step(set);
 
        while (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
        {
            lexeme_t l = _lexer.current();
            _lexer.next();
 
            if (l == lex_double_slash)
                n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
 
            n = parse_step(n);
        }
 
        return n;
    }
 
    // LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
    // AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
    xpath_ast_node *parse_location_path()
    {
        if (_lexer.current() == lex_slash)
        {
            _lexer.next();
 
            xpath_ast_node *n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
 
            // relative location path can start from axis_attribute, dot, double_dot, multiply and string lexemes; any
            // other lexeme means standalone root path
            lexeme_t l = _lexer.current();
 
            if (l == lex_string || l == lex_axis_attribute || l == lex_dot || l == lex_double_dot || l == lex_multiply)
                return parse_relative_location_path(n);
            else
                return n;
        }
        else if (_lexer.current() == lex_double_slash)
        {
            _lexer.next();
 
            xpath_ast_node *n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
            n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
 
            return parse_relative_location_path(n);
        }
 
        // else clause moved outside of if because of bogus warning 'control may reach end of non-void function being
        // inlined' in gcc 4.0.1
        return parse_relative_location_path(0);
    }
 
    // PathExpr ::= LocationPath
    //                          | FilterExpr
    //                          | FilterExpr '/' RelativeLocationPath
    //                          | FilterExpr '//' RelativeLocationPath
    // UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
    // UnaryExpr ::= UnionExpr | '-' UnaryExpr
    xpath_ast_node *parse_path_or_unary_expression()
    {
        // Clarification.
        // PathExpr begins with either LocationPath or FilterExpr.
        // FilterExpr begins with PrimaryExpr
        // PrimaryExpr begins with '$' in case of it being a variable reference,
        // '(' in case of it being an expression, string literal, number constant or
        // function call.
 
        if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace ||
            _lexer.current() == lex_quoted_string || _lexer.current() == lex_number || _lexer.current() == lex_string)
        {
            if (_lexer.current() == lex_string)
            {
                // This is either a function call, or not - if not, we shall proceed with location path
                const char_t *state = _lexer.state();
 
                while (PUGI__IS_CHARTYPE(*state, ct_space))
                    ++state;
 
                if (*state != '(')
                    return parse_location_path();
 
                // This looks like a function call; however this still can be a node-test. Check it.
                if (parse_node_test_type(_lexer.contents()) != nodetest_none)
                    return parse_location_path();
            }
 
            xpath_ast_node *n = parse_filter_expression();
 
            if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
            {
                lexeme_t l = _lexer.current();
                _lexer.next();
 
                if (l == lex_double_slash)
                {
                    if (n->rettype() != xpath_type_node_set)
                        throw_error("Step has to be applied to node set");
 
                    n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
                }
 
                // select from location path
                return parse_relative_location_path(n);
            }
 
            return n;
        }
        else if (_lexer.current() == lex_minus)
        {
            _lexer.next();
 
            // precedence 7+ - only parses union expressions
            xpath_ast_node *expr = parse_expression_rec(parse_path_or_unary_expression(), 7);
 
            return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
        }
        else
            return parse_location_path();
    }
 
    struct binary_op_t
    {
        ast_type_t asttype;
        xpath_value_type rettype;
        int precedence;
 
        binary_op_t() : asttype(ast_unknown), rettype(xpath_type_none), precedence(0) {}
 
        binary_op_t(ast_type_t asttype_, xpath_value_type rettype_, int precedence_)
            : asttype(asttype_), rettype(rettype_), precedence(precedence_)
        {
        }
 
        static binary_op_t parse(xpath_lexer &lexer)
        {
            switch (lexer.current())
            {
            case lex_string:
                if (lexer.contents() == PUGIXML_TEXT("or"))
                    return binary_op_t(ast_op_or, xpath_type_boolean, 1);
                else if (lexer.contents() == PUGIXML_TEXT("and"))
                    return binary_op_t(ast_op_and, xpath_type_boolean, 2);
                else if (lexer.contents() == PUGIXML_TEXT("div"))
                    return binary_op_t(ast_op_divide, xpath_type_number, 6);
                else if (lexer.contents() == PUGIXML_TEXT("mod"))
                    return binary_op_t(ast_op_mod, xpath_type_number, 6);
                else
                    return binary_op_t();
 
            case lex_equal:
                return binary_op_t(ast_op_equal, xpath_type_boolean, 3);
 
            case lex_not_equal:
                return binary_op_t(ast_op_not_equal, xpath_type_boolean, 3);
 
            case lex_less:
                return binary_op_t(ast_op_less, xpath_type_boolean, 4);
 
            case lex_greater:
                return binary_op_t(ast_op_greater, xpath_type_boolean, 4);
 
            case lex_less_or_equal:
                return binary_op_t(ast_op_less_or_equal, xpath_type_boolean, 4);
 
            case lex_greater_or_equal:
                return binary_op_t(ast_op_greater_or_equal, xpath_type_boolean, 4);
 
            case lex_plus:
                return binary_op_t(ast_op_add, xpath_type_number, 5);
 
            case lex_minus:
                return binary_op_t(ast_op_subtract, xpath_type_number, 5);
 
            case lex_multiply:
                return binary_op_t(ast_op_multiply, xpath_type_number, 6);
 
            case lex_union:
                return binary_op_t(ast_op_union, xpath_type_node_set, 7);
 
            default:
                return binary_op_t();
            }
        }
    };
 
    xpath_ast_node *parse_expression_rec(xpath_ast_node *lhs, int limit)
    {
        binary_op_t op = binary_op_t::parse(_lexer);
 
        while (op.asttype != ast_unknown && op.precedence >= limit)
        {
            _lexer.next();
 
            xpath_ast_node *rhs = parse_path_or_unary_expression();
 
            binary_op_t nextop = binary_op_t::parse(_lexer);
 
            while (nextop.asttype != ast_unknown && nextop.precedence > op.precedence)
            {
                rhs = parse_expression_rec(rhs, nextop.precedence);
 
                nextop = binary_op_t::parse(_lexer);
            }
 
            if (op.asttype == ast_op_union &&
                (lhs->rettype() != xpath_type_node_set || rhs->rettype() != xpath_type_node_set))
                throw_error("Union operator has to be applied to node sets");
 
            lhs = new (alloc_node()) xpath_ast_node(op.asttype, op.rettype, lhs, rhs);
 
            op = binary_op_t::parse(_lexer);
        }
 
        return lhs;
    }
 
    // Expr ::= OrExpr
    // OrExpr ::= AndExpr | OrExpr 'or' AndExpr
    // AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
    // EqualityExpr ::= RelationalExpr
    //                                  | EqualityExpr '=' RelationalExpr
    //                                  | EqualityExpr '!=' RelationalExpr
    // RelationalExpr ::= AdditiveExpr
    //                                    | RelationalExpr '<' AdditiveExpr
    //                                    | RelationalExpr '>' AdditiveExpr
    //                                    | RelationalExpr '<=' AdditiveExpr
    //                                    | RelationalExpr '>=' AdditiveExpr
    // AdditiveExpr ::= MultiplicativeExpr
    //                                  | AdditiveExpr '+' MultiplicativeExpr
    //                                  | AdditiveExpr '-' MultiplicativeExpr
    // MultiplicativeExpr ::= UnaryExpr
    //                                            | MultiplicativeExpr '*' UnaryExpr
    //                                            | MultiplicativeExpr 'div' UnaryExpr
    //                                            | MultiplicativeExpr 'mod' UnaryExpr
    xpath_ast_node *parse_expression() { return parse_expression_rec(parse_path_or_unary_expression(), 0); }
 
    xpath_parser(const char_t *query, xpath_variable_set *variables, xpath_allocator *alloc, xpath_parse_result *result)
        : _alloc(alloc), _lexer(query), _query(query), _variables(variables), _result(result)
    {
    }
 
    xpath_ast_node *parse()
    {
        xpath_ast_node *result = parse_expression();
 
        if (_lexer.current() != lex_eof)
        {
            // there are still unparsed tokens left, error
            throw_error("Incorrect query");
        }
 
        return result;
    }
 
    static xpath_ast_node *parse(const char_t *query, xpath_variable_set *variables, xpath_allocator *alloc,
                                 xpath_parse_result *result)
    {
        xpath_parser parser(query, variables, alloc, result);
 
#ifdef PUGIXML_NO_EXCEPTIONS
        int error = setjmp(parser._error_handler);
 
        return (error == 0) ? parser.parse() : 0;
#else
        return parser.parse();
#endif
    }
};
 
struct xpath_query_impl
{
    static xpath_query_impl *create()
    {
        void *memory = xml_memory::allocate(sizeof(xpath_query_impl));
 
        return new (memory) xpath_query_impl();
    }
 
    static void destroy(void *ptr)
    {
        if (!ptr)
            return;
 
        // free all allocated pages
        static_cast<xpath_query_impl *>(ptr)->alloc.release();
 
        // free allocator memory (with the first page)
        xml_memory::deallocate(ptr);
    }
 
    xpath_query_impl() : root(0), alloc(&block) { block.next = 0; }
 
    xpath_ast_node *root;
    xpath_allocator alloc;
    xpath_memory_block block;
};
 
PUGI__FN xpath_string evaluate_string_impl(xpath_query_impl *impl, const xpath_node &n, xpath_stack_data &sd)
{
    if (!impl)
        return xpath_string();
 
#ifdef PUGIXML_NO_EXCEPTIONS
    if (setjmp(sd.error_handler))
        return xpath_string();
#endif
 
    xpath_context c(n, 1, 1);
 
    return impl->root->eval_string(c, sd.stack);
}
PUGI__NS_END
 
namespace pugi
{
#ifndef PUGIXML_NO_EXCEPTIONS
PUGI__FN xpath_exception::xpath_exception(const xpath_parse_result &result_) : _result(result_)
{
    assert(_result.error);
}
 
PUGI__FN const char *xpath_exception::what() const throw()
{
    return _result.error;
}
 
PUGI__FN const xpath_parse_result &xpath_exception::result() const
{
    return _result;
}
#endif
 
PUGI__FN xpath_node::xpath_node() {}
 
PUGI__FN xpath_node::xpath_node(const xml_node &node_) : _node(node_) {}
 
PUGI__FN xpath_node::xpath_node(const xml_attribute &attribute_, const xml_node &parent_)
    : _node(attribute_ ? parent_ : xml_node()), _attribute(attribute_)
{
}
 
PUGI__FN xml_node xpath_node::node() const
{
    return _attribute ? xml_node() : _node;
}
 
PUGI__FN xml_attribute xpath_node::attribute() const
{
    return _attribute;
}
 
PUGI__FN xml_node xpath_node::parent() const
{
    return _attribute ? _node : _node.parent();
}
 
PUGI__FN static void unspecified_bool_xpath_node(xpath_node ***) {}
 
PUGI__FN xpath_node::operator xpath_node::unspecified_bool_type() const
{
    return (_node || _attribute) ? unspecified_bool_xpath_node : 0;
}
 
PUGI__FN bool xpath_node::operator!() const
{
    return !(_node || _attribute);
}
 
PUGI__FN bool xpath_node::operator==(const xpath_node &n) const
{
    return _node == n._node && _attribute == n._attribute;
}
 
PUGI__FN bool xpath_node::operator!=(const xpath_node &n) const
{
    return _node != n._node || _attribute != n._attribute;
}
 
#ifdef __BORLANDC__
PUGI__FN bool operator&&(const xpath_node &lhs, bool rhs)
{
    return (bool)lhs && rhs;
}
 
PUGI__FN bool operator||(const xpath_node &lhs, bool rhs)
{
    return (bool)lhs || rhs;
}
#endif
 
PUGI__FN void xpath_node_set::_assign(const_iterator begin_, const_iterator end_)
{
    assert(begin_ <= end_);
 
    size_t size_ = static_cast<size_t>(end_ - begin_);
 
    if (size_ <= 1)
    {
        // deallocate old buffer
        if (_begin != &_storage)
            impl::xml_memory::deallocate(_begin);
 
        // use internal buffer
        if (begin_ != end_)
            _storage = *begin_;
 
        _begin = &_storage;
        _end = &_storage + size_;
    }
    else
    {
        // make heap copy
        xpath_node *storage = static_cast<xpath_node *>(impl::xml_memory::allocate(size_ * sizeof(xpath_node)));
 
        if (!storage)
        {
#ifdef PUGIXML_NO_EXCEPTIONS
            return;
#else
            throw std::bad_alloc();
#endif
        }
 
        memcpy(storage, begin_, size_ * sizeof(xpath_node));
 
        // deallocate old buffer
        if (_begin != &_storage)
            impl::xml_memory::deallocate(_begin);
 
        // finalize
        _begin = storage;
        _end = storage + size_;
    }
}
 
PUGI__FN xpath_node_set::xpath_node_set() : _type(type_unsorted), _begin(&_storage), _end(&_storage) {}
 
PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_)
    : _type(type_), _begin(&_storage), _end(&_storage)
{
    _assign(begin_, end_);
}
 
PUGI__FN xpath_node_set::~xpath_node_set()
{
    if (_begin != &_storage)
        impl::xml_memory::deallocate(_begin);
}
 
PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set &ns) : _type(ns._type), _begin(&_storage), _end(&_storage)
{
    _assign(ns._begin, ns._end);
}
 
PUGI__FN xpath_node_set &xpath_node_set::operator=(const xpath_node_set &ns)
{
    if (this == &ns)
        return *this;
 
    _type = ns._type;
    _assign(ns._begin, ns._end);
 
    return *this;
}
 
PUGI__FN xpath_node_set::type_t xpath_node_set::type() const
{
    return _type;
}
 
PUGI__FN size_t xpath_node_set::size() const
{
    return _end - _begin;
}
 
PUGI__FN bool xpath_node_set::empty() const
{
    return _begin == _end;
}
 
PUGI__FN const xpath_node &xpath_node_set::operator[](size_t index) const
{
    assert(index < size());
    return _begin[index];
}
 
PUGI__FN xpath_node_set::const_iterator xpath_node_set::begin() const
{
    return _begin;
}
 
PUGI__FN xpath_node_set::const_iterator xpath_node_set::end() const
{
    return _end;
}
 
PUGI__FN void xpath_node_set::sort(bool reverse)
{
    _type = impl::xpath_sort(_begin, _end, _type, reverse);
}
 
PUGI__FN xpath_node xpath_node_set::first() const
{
    return impl::xpath_first(_begin, _end, _type);
}
 
PUGI__FN xpath_parse_result::xpath_parse_result() : error("Internal error"), offset(0) {}
 
PUGI__FN xpath_parse_result::operator bool() const
{
    return error == 0;
}
 
PUGI__FN const char *xpath_parse_result::description() const
{
    return error ? error : "No error";
}
 
PUGI__FN xpath_variable::xpath_variable() : _type(xpath_type_none), _next(0) {}
 
PUGI__FN const char_t *xpath_variable::name() const
{
    switch (_type)
    {
    case xpath_type_node_set:
        return static_cast<const impl::xpath_variable_node_set *>(this)->name;
 
    case xpath_type_number:
        return static_cast<const impl::xpath_variable_number *>(this)->name;
 
    case xpath_type_string:
        return static_cast<const impl::xpath_variable_string *>(this)->name;
 
    case xpath_type_boolean:
        return static_cast<const impl::xpath_variable_boolean *>(this)->name;
 
    default:
        assert(!"Invalid variable type");
        return 0;
    }
}
 
PUGI__FN xpath_value_type xpath_variable::type() const
{
    return _type;
}
 
PUGI__FN bool xpath_variable::get_boolean() const
{
    return (_type == xpath_type_boolean) ? static_cast<const impl::xpath_variable_boolean *>(this)->value : false;
}
 
PUGI__FN double xpath_variable::get_number() const
{
    return (_type == xpath_type_number) ? static_cast<const impl::xpath_variable_number *>(this)->value
                                        : impl::gen_nan();
}
 
PUGI__FN const char_t *xpath_variable::get_string() const
{
    const char_t *value =
        (_type == xpath_type_string) ? static_cast<const impl::xpath_variable_string *>(this)->value : 0;
    return value ? value : PUGIXML_TEXT("");
}
 
PUGI__FN const xpath_node_set &xpath_variable::get_node_set() const
{
    return (_type == xpath_type_node_set) ? static_cast<const impl::xpath_variable_node_set *>(this)->value
                                          : impl::dummy_node_set;
}
 
PUGI__FN bool xpath_variable::set(bool value)
{
    if (_type != xpath_type_boolean)
        return false;
 
    static_cast<impl::xpath_variable_boolean *>(this)->value = value;
    return true;
}
 
PUGI__FN bool xpath_variable::set(double value)
{
    if (_type != xpath_type_number)
        return false;
 
    static_cast<impl::xpath_variable_number *>(this)->value = value;
    return true;
}
 
PUGI__FN bool xpath_variable::set(const char_t *value)
{
    if (_type != xpath_type_string)
        return false;
 
    impl::xpath_variable_string *var = static_cast<impl::xpath_variable_string *>(this);
 
    // duplicate string
    size_t size = (impl::strlength(value) + 1) * sizeof(char_t);
 
    char_t *copy = static_cast<char_t *>(impl::xml_memory::allocate(size));
    if (!copy)
        return false;
 
    memcpy(copy, value, size);
 
    // replace old string
    if (var->value)
        impl::xml_memory::deallocate(var->value);
    var->value = copy;
 
    return true;
}
 
PUGI__FN bool xpath_variable::set(const xpath_node_set &value)
{
    if (_type != xpath_type_node_set)
        return false;
 
    static_cast<impl::xpath_variable_node_set *>(this)->value = value;
    return true;
}
 
PUGI__FN xpath_variable_set::xpath_variable_set()
{
    for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
        _data[i] = 0;
}
 
PUGI__FN xpath_variable_set::~xpath_variable_set()
{
    for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
    {
        xpath_variable *var = _data[i];
 
        while (var)
        {
            xpath_variable *next = var->_next;
 
            impl::delete_xpath_variable(var->_type, var);
 
            var = next;
        }
    }
}
 
PUGI__FN xpath_variable *xpath_variable_set::find(const char_t *name) const
{
    const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
    size_t hash = impl::hash_string(name) % hash_size;
 
    // look for existing variable
    for (xpath_variable *var = _data[hash]; var; var = var->_next)
        if (impl::strequal(var->name(), name))
            return var;
 
    return 0;
}
 
PUGI__FN xpath_variable *xpath_variable_set::add(const char_t *name, xpath_value_type type)
{
    const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
    size_t hash = impl::hash_string(name) % hash_size;
 
    // look for existing variable
    for (xpath_variable *var = _data[hash]; var; var = var->_next)
        if (impl::strequal(var->name(), name))
            return var->type() == type ? var : 0;
 
    // add new variable
    xpath_variable *result = impl::new_xpath_variable(type, name);
 
    if (result)
    {
        result->_type = type;
        result->_next = _data[hash];
 
        _data[hash] = result;
    }
 
    return result;
}
 
PUGI__FN bool xpath_variable_set::set(const char_t *name, bool value)
{
    xpath_variable *var = add(name, xpath_type_boolean);
    return var ? var->set(value) : false;
}
 
PUGI__FN bool xpath_variable_set::set(const char_t *name, double value)
{
    xpath_variable *var = add(name, xpath_type_number);
    return var ? var->set(value) : false;
}
 
PUGI__FN bool xpath_variable_set::set(const char_t *name, const char_t *value)
{
    xpath_variable *var = add(name, xpath_type_string);
    return var ? var->set(value) : false;
}
 
PUGI__FN bool xpath_variable_set::set(const char_t *name, const xpath_node_set &value)
{
    xpath_variable *var = add(name, xpath_type_node_set);
    return var ? var->set(value) : false;
}
 
PUGI__FN xpath_variable *xpath_variable_set::get(const char_t *name)
{
    return find(name);
}
 
PUGI__FN const xpath_variable *xpath_variable_set::get(const char_t *name) const
{
    return find(name);
}
 
PUGI__FN xpath_query::xpath_query(const char_t *query, xpath_variable_set *variables) : _impl(0)
{
    impl::xpath_query_impl *qimpl = impl::xpath_query_impl::create();
 
    if (!qimpl)
    {
#ifdef PUGIXML_NO_EXCEPTIONS
        _result.error = "Out of memory";
#else
        throw std::bad_alloc();
#endif
    }
    else
    {
        impl::buffer_holder impl_holder(qimpl, impl::xpath_query_impl::destroy);
 
        qimpl->root = impl::xpath_parser::parse(query, variables, &qimpl->alloc, &_result);
 
        if (qimpl->root)
        {
            _impl = static_cast<impl::xpath_query_impl *>(impl_holder.release());
            _result.error = 0;
        }
    }
}
 
PUGI__FN xpath_query::~xpath_query()
{
    impl::xpath_query_impl::destroy(_impl);
}
 
PUGI__FN xpath_value_type xpath_query::return_type() const
{
    if (!_impl)
        return xpath_type_none;
 
    return static_cast<impl::xpath_query_impl *>(_impl)->root->rettype();
}
 
PUGI__FN bool xpath_query::evaluate_boolean(const xpath_node &n) const
{
    if (!_impl)
        return false;
 
    impl::xpath_context c(n, 1, 1);
    impl::xpath_stack_data sd;
 
#ifdef PUGIXML_NO_EXCEPTIONS
    if (setjmp(sd.error_handler))
        return false;
#endif
 
    return static_cast<impl::xpath_query_impl *>(_impl)->root->eval_boolean(c, sd.stack);
}
 
PUGI__FN double xpath_query::evaluate_number(const xpath_node &n) const
{
    if (!_impl)
        return impl::gen_nan();
 
    impl::xpath_context c(n, 1, 1);
    impl::xpath_stack_data sd;
 
#ifdef PUGIXML_NO_EXCEPTIONS
    if (setjmp(sd.error_handler))
        return impl::gen_nan();
#endif
 
    return static_cast<impl::xpath_query_impl *>(_impl)->root->eval_number(c, sd.stack);
}
 
#ifndef PUGIXML_NO_STL
PUGI__FN string_t xpath_query::evaluate_string(const xpath_node &n) const
{
    impl::xpath_stack_data sd;
 
    return impl::evaluate_string_impl(static_cast<impl::xpath_query_impl *>(_impl), n, sd).c_str();
}
#endif
 
PUGI__FN size_t xpath_query::evaluate_string(char_t *buffer, size_t capacity, const xpath_node &n) const
{
    impl::xpath_stack_data sd;
 
    impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl *>(_impl), n, sd);
 
    size_t full_size = r.length() + 1;
 
    if (capacity > 0)
    {
        size_t size = (full_size < capacity) ? full_size : capacity;
        assert(size > 0);
 
        memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
        buffer[size - 1] = 0;
    }
 
    return full_size;
}
 
PUGI__FN xpath_node_set xpath_query::evaluate_node_set(const xpath_node &n) const
{
    if (!_impl)
        return xpath_node_set();
 
    impl::xpath_ast_node *root = static_cast<impl::xpath_query_impl *>(_impl)->root;
 
    if (root->rettype() != xpath_type_node_set)
    {
#ifdef PUGIXML_NO_EXCEPTIONS
        return xpath_node_set();
#else
        xpath_parse_result res;
        res.error = "Expression does not evaluate to node set";
 
        throw xpath_exception(res);
#endif
    }
 
    impl::xpath_context c(n, 1, 1);
    impl::xpath_stack_data sd;
 
#ifdef PUGIXML_NO_EXCEPTIONS
    if (setjmp(sd.error_handler))
        return xpath_node_set();
#endif
 
    impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack);
 
    return xpath_node_set(r.begin(), r.end(), r.type());
}
 
PUGI__FN const xpath_parse_result &xpath_query::result() const
{
    return _result;
}
 
PUGI__FN static void unspecified_bool_xpath_query(xpath_query ***) {}
 
PUGI__FN xpath_query::operator xpath_query::unspecified_bool_type() const
{
    return _impl ? unspecified_bool_xpath_query : 0;
}
 
PUGI__FN bool xpath_query::operator!() const
{
    return !_impl;
}
 
PUGI__FN xpath_node xml_node::select_single_node(const char_t *query, xpath_variable_set *variables) const
{
    xpath_query q(query, variables);
    return select_single_node(q);
}
 
PUGI__FN xpath_node xml_node::select_single_node(const xpath_query &query) const
{
    xpath_node_set s = query.evaluate_node_set(*this);
    return s.empty() ? xpath_node() : s.first();
}
 
PUGI__FN xpath_node_set xml_node::select_nodes(const char_t *query, xpath_variable_set *variables) const
{
    xpath_query q(query, variables);
    return select_nodes(q);
}
 
PUGI__FN xpath_node_set xml_node::select_nodes(const xpath_query &query) const
{
    return query.evaluate_node_set(*this);
}
} // namespace pugi
 
#endif
 
#ifdef __BORLANDC__
#pragma option pop
#endif
 
// Intel C++ does not properly keep warning state for function templates,
// so popping warning state at the end of translation unit leads to warnings in
// the middle.
#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
#pragma warning(pop)
#endif
 
// Undefine all local macros (makes sure we're not leaking macros in header-only
// mode)
#undef PUGI__NO_INLINE
#undef PUGI__STATIC_ASSERT
#undef PUGI__DMC_VOLATILE
#undef PUGI__MSVC_CRT_VERSION
#undef PUGI__NS_BEGIN
#undef PUGI__NS_END
#undef PUGI__FN
#undef PUGI__FN_NO_INLINE
#undef PUGI__IS_CHARTYPE_IMPL
#undef PUGI__IS_CHARTYPE
#undef PUGI__IS_CHARTYPEX
#undef PUGI__SKIPWS
#undef PUGI__OPTSET
#undef PUGI__PUSHNODE
#undef PUGI__POPNODE
#undef PUGI__SCANFOR
#undef PUGI__SCANWHILE
#undef PUGI__ENDSEG
#undef PUGI__THROW_ERROR
#undef PUGI__CHECK_ERROR
 
#endif