record.c

/****************************************************************************

 \file      record.c
 \ingroup   Patterns
 \brief     Binary record support functions
 \author    David Mace

Binary Record Handling

This writes something a lot like XML, except it all doesn't have to be mapped
into memory at once, and we don't need to recursively parse text.  Everything
is an offset to an offset.  

This is every bit as robust as the XML that DataGen uses, except it's binary.

We store labels and enumerations and types as text, so that validation and
conversion can be done on data.  Generally the system still expects certain
things to be present at certain places, but the data types can change somewhat
without breaking.  Enumerations can have new members added.  Scalar types may
be translated.  Arrays may be grown.  Enum or structure members may be re-
arranged without breaking the stored data.  Most kinds of containers (i.e.
arrays, vectors, sets, etc.) are interchangeable, as long as they expect the
same members.  You can not change one type of structure to another.  You 
can not change a scalar to a structure or to a container.

Generally, this is the serialization meant for files, databases and archives
that last across versions of the application.  Where ctl_serial will melt 
down for minor changes to the application which affect its protocol (i.e. if
you add or change a member and send a packet to a previous version, that
older version will not be able to decode it), this is far more forgiving, 
assuming you follow the rules.

*****************************************************************************/

#include <stdio.h>
#include "ctl/dg/datagen.h"
#include "ctl/algorithm.temp.h"

size_t ctl_record_head_size( const char* label, const char* type )
{
    assertconst(label,1);
    assertconst(type,1);
    return 
        sizeof(uint32) +                /* Record size */
        sizeof(uint16) +                /* Header size */
        cstrlen(char)( label ) + 1 +    /* NUL terminated record label string */
        cstrlen(char)( type ) + 1;      /* NUL terminated record type string */
}

size_t ctl_record_objarray_head_size( const char* label, const char* type, const char* mbrtype )
{
    assertconst(label,1);
    assertconst(type,1);
    assertconst(mbrtype,1);
    return 
        sizeof(uint32) +                /* Record size */
        sizeof(uint16) +                /* Header size */
        cstrlen(char)( label ) + 1 +    /* NUL terminated record label string */
        cstrlen(char)( type ) + 1 +     /* NUL terminated record type string */
        cstrlen(char)( mbrtype ) + 1 +  /* NUL terminated record member type string */
        sizeof(uint32);                 /* Record count */
}

bool ctl_record_valid( const ctl_serial* serial )
{
    uint32 size;
    uint16 headersize;
    size_t labellen;
    size_t typelen;
    ctl_serial parse;
    assertobjconst(serial);
    parse = *serial;
    /* We should have AT LEAST a header with one-byte label/type */
    if( serial->end - serial->curr < 10 )
        return false;
    uint32_serial_read(&size, &parse );
    /* Size should be within bounds */
    if( serial->curr + size > serial->end )
        return false;
    uint16_serial_read(&headersize, &parse );
    /* Data offset should be within bounds */
    if( serial->curr + headersize > serial->end )
        return false;
    /* Data offset should be smaller than size */
    if( headersize > size )
        return false;
    labellen = cstrnlen(char)((char*)parse.curr,headersize-6);
    /* Label length isn't NUL terminated, or it consumes type string space */
    if( labellen >= (size_t)(headersize-6) )
        return false;
    parse.curr += labellen;
    /* Needed to be NUL terminated */
    if( *parse.curr )
        return false;
    parse.curr++;
    typelen = cstrnlen(char)((char*)parse.curr,headersize-(6+labellen));
    parse.curr += labellen;
    /* Needed to be NUL terminated */
    if( *parse.curr )
        return false;
    parse.curr++;
    /* Should add up to the header size */
    if( 6 + labellen + 1 + typelen + 1 != headersize )
        return false;

    /* Didn't find any problem */
    return true;
}

bool ctl_record_next( ctl_serial* serial )
{
    uint8* curr;
    uint32 offset;
    assertobjptr(serial);
    curr = serial->curr;
    uint32_serial_read(&offset, serial );
    if( curr + offset > serial->end )
        ctl_serial_except( serial, "ctl_record_next: Bad offset" );
    serial->curr = curr + offset;
    return serial->curr < serial->end;
}

bool ctl_record_find( ctl_serial* serial, const char* label )
{
    ctl_serial search;
    assertobjptr(serial);
    search = *serial;
    do
    {
        const char *szcurr = ctl_record_label( &search );
        if( !cstrcmp(char)( label, szcurr ) )
        {
            serial->curr = search.curr;
            return true;
        }
    }
    while( ctl_record_next( &search ) );
    return false;
}

bool ctl_record_find_next( ctl_serial* serial, const char* label )
{
    ctl_serial search;
    assertobjptr(serial);
    search = *serial;
    while( ctl_record_next( &search ) )
    {
        if( !cstrcmp(char)( label, ctl_record_label( &search ) ) )
        {
            serial->curr = search.curr;
            return true;
        }
    }
    return false;
}

size_t ctl_record_getdata( ctl_serial* serial )
{
    uint8* curr;
    uint32 offset;
    uint16 dataoffset;
    assertobjptr(serial);
    curr = serial->curr;
    uint32_serial_read(&offset, serial );
    uint16_serial_read(&dataoffset, serial );
    if( dataoffset > offset )
        ctl_serial_except( serial, "ctl_record_getdata: Bad offset" );
    if( curr + offset > serial->end )
        ctl_serial_except( serial, "ctl_record_getdata: Offset exceeds serial data" );
    serial->end = curr + offset;
    serial->curr = curr + dataoffset;
    return offset - dataoffset;
}

size_t ctl_record_container_validatedata( ctl_serial* serial, const char* type, const char* mbrtype )
{
    assertobjptr(serial);
    assertconst(type,1);
    assertconst(mbrtype,1);
    if( type && !ctl_record_type_is( type, serial ) )
        ctl_serial_except( serial, "ctl_record_container_validatedata: Mismatched container type" );
    if( mbrtype && !ctl_record_contained_type_is( mbrtype, serial ) )
        ctl_serial_except( serial, "ctl_record_container_validatedata: Mismatched content type" );
    {
        uint8* curr = serial->curr;
        uint32 offset;
        uint16 dataoffset;
        uint32 count;
        uint32_serial_read(&offset, serial );
        uint16_serial_read(&dataoffset, serial );
        if( dataoffset > offset )
            ctl_serial_except( serial, "ctl_record_container_validatedata: Bad offset" );
        if( curr + offset > serial->end )
            ctl_serial_except( serial, "ctl_record_container_validatedata: Offset exceeds serial data" );
        serial->end = curr + offset;
        serial->curr = curr + dataoffset - 4;
        uint32_serial_read(&count, serial );
        return count;
    }
}


const uint8* ctl_record_end( const ctl_serial* serial )
{
    uint32 offset;
    ctl_serial tmp;
    assertobjconst(serial);
    tmp = *serial;
    uint32_serial_read(&offset, &tmp );
    if( serial->curr + offset > serial->end )
        ctl_serial_except( serial, "ctl_record_next: Bad offset" );
    return serial->curr + offset;;
}

bool ctl_record_is( const char* label, const ctl_serial* serial )
{
    assertconst(label,1);
    assertobjconst(serial);
    return !cstrcmp(char)( label, ctl_record_label( serial ) );
}


const char* ctl_record_label( const ctl_serial* serial )
{
    assertobjconst(serial);
    if( serial->curr + 7 > serial->end )
        ctl_serial_except( serial, "ctl_record_label: Truncated" );
    return (const char*)(serial->curr + 6);
}

const char* ctl_record_type( const ctl_serial* serial )
{
    const char* label;
    assertobjconst(serial);
    label = ctl_record_label( serial );
    label += cstrnlen(char)(label,ctl_serial_remain(serial))+1;
    if( (uint8*)label >= serial->end )
        ctl_serial_except( serial, "ctl_record_type: Truncated" );
    return label;
}

bool ctl_record_type_is( const char* label, const ctl_serial* serial )
{
    assertconst(label,1);
    assertobjconst(serial);
    return !cstrcmp(char)( label, ctl_record_type( serial ) );
}

const char* ctl_record_contained_type( const ctl_serial* serial )
{
    const char* type;
    assertobjconst(serial);
    type = ctl_record_type( serial );
    type += cstrnlen(char)(type,ctl_serial_remain(serial))+1;
    if( (uint8*)type >= serial->end )
        ctl_serial_except( serial, "ctl_record_contained_type: Truncated" );
    return type;
}

bool ctl_record_contained_type_is( const char* label, const ctl_serial* serial )
{
    assertconst(label,1);
    assertobjconst(serial);
    return !cstrcmp(char)(label,ctl_record_contained_type( serial ) );
}

size_t ctl_record_count( const ctl_serial* serial )
{
    ctl_serial pserial;
    uint16 dataoffset;
    uint32 count;
    assertobjconst(serial);
    pserial = *serial;
    pserial.curr += 4;
    uint16_serial_read(&dataoffset, &pserial );
    pserial.curr = serial->curr + dataoffset - 4;
    uint32_serial_read(&count, &pserial );
    return count;
}

uint8* ctl_record_write_begin( const char* type, size_t size, ctl_serial* serial, const char* label )
{
    assertobjptr(serial);
    assertconst(type,1);
    assertconst(label,1);
    {
        uint8* ret = serial->curr;
        size_t lenLabel = cstrlen(char)(label)+1;
        size_t lenType = cstrlen(char)(type)+1;
        uint16 offData = (uint16)(lenLabel + lenType + 6);
        uint32 reserve = (uint32)size + offData;
        /* This should fit into the reported space */
        assert( serial->curr + offData + size <= serial->end );
        uint32_serial_write(&reserve, serial );
        uint16_serial_write(&offData, serial );
        memcpy(serial->curr, label, lenLabel);
        serial->curr += lenLabel;
        memcpy(serial->curr, type, lenType);
        serial->curr += lenType;
        return ret;
    }
}


uint8* ctl_record_write_container_begin( const char* containertype, const char* contenttype, size_t size, size_t count, ctl_serial* serial, const char* label )
{
    assertobjptr(serial);
    assertconst(containertype,1);
    assertconst(contenttype,1);
    assertconst(label,1);
    {
        uint8* ret = serial->curr;
        size_t lenLabel = cstrlen(char)(label)+1;
        size_t lenType = cstrlen(char)(containertype)+1;
        size_t lenContent = cstrlen(char)(contenttype)+1;
        uint16 offData = (uint16)(lenLabel + lenType + lenContent + 10);
        uint32 reserve = (uint32)size + offData;
        uint32 mbrcount = (uint32)count;
        /* This should fit into the reported space */
        assert( serial->curr + offData + size <= serial->end );
        uint32_serial_write(&reserve, serial );
        uint16_serial_write(&offData, serial );
        memcpy(serial->curr, label, lenLabel);
        serial->curr += lenLabel;
        memcpy(serial->curr, containertype, lenType);
        serial->curr += lenType;
        memcpy(serial->curr, contenttype, lenContent);
        serial->curr += lenContent;
        uint32_serial_write(&mbrcount, serial );
        return ret;
    }
}

void ctl_record_write_end( uint8* begret, ctl_serial* serial )
{
    assertobjptr(serial);
    assertptr(begret,1);
    {
        uint8* tmp = serial->curr;
        uint32 size = (uint32)(serial->curr - begret);
        serial->curr = begret;
        uint32_serial_write( &size, serial );
        serial->curr = tmp;
    }
}

size_t ctl_serial_record_size( const ctl_serial* serial, const char* label )
{
    size_t size;
    assertconst(label,1);
    assertobjconst(serial);
    size = ctl_serial_total( serial );
    if( 0 == size )
        return 0;
    size += ctl_record_head_size( label, "raw" );
    return size;
}
void ctl_serial_record_write( const ctl_serial* from, ctl_serial* to, const char* label )
{
    size_t total;
    assertobjconst(from);
    assertobjptr(to);
    assertconst(label,1);
    total = ctl_serial_total( from );
    if( 0 != total )
    {
        size_t total = ctl_serial_total( from );
        ctl_record_write_begin( "raw", total, to, label );
        memcpy( to->curr, from->begin, total );
        to->curr += total;
    }
}
bool ctl_serial_record_read( ctl_serial* to, ctl_serial* from, const char* label )
{
    ctl_serial tmp;
    assertobjptr(from);
    assertobjptr(to);
    assertconst(label,1);
    tmp = *from;
    if( ctl_record_find( &tmp, label ) )
    {
        const char* szcontainer = ctl_record_type( &tmp );
        if( cstrcmp(char)(szcontainer,"raw") )
            ctl_serial_except( from, "ctl_serial_record_read: Not raw data" );
        ctl_record_getdata( &tmp );
        ctl_serial_init( to, tmp.curr, tmp.end );
        return true;
    }
    return false;
}


size_t char_record_size_string( const char* str, size_t maxsize, const char* label )
{
    assertconst(label,1);
    assertconst(str,1);
    return ctl_record_objarray_head_size( label,"string","char" ) + chssize(char,cstrnlen(char)(str,maxsize));
}
size_t wchar_t_record_size_string( const wchar_t* str, size_t maxsize, const char* label )
{
    assertconst(label,1);
    assertconst(str,1);
    return ctl_record_objarray_head_size( label,"string","wchar_t" ) + chssize(wchar_t,cstrnlen(wchar_t)(str,maxsize));
}
size_t char_record_size_gstring( const ctl_gstring(char)* str, const char* label )
{
    assertconst(label,1);
    assertobjconst(str);
    return ctl_record_objarray_head_size( label,"string","char" ) + chssize(char,ctl_gstring_size(char,str));
}
size_t wchar_t_record_size_gstring( const ctl_gstring(wchar_t)* str, const char* label )
{
    assertconst(label,1);
    assertobjconst(str);
    return ctl_record_objarray_head_size( label,"string","wchar_t" ) + chssize(wchar_t,ctl_gstring_size(wchar_t,str));
}


void char_record_write_string( const char* sz, size_t size, ctl_serial* serial, const char* label )
{
    size_t storesize;
    assertconst(sz,1);
    assertconst(label,1);
    assertobjptr(serial);
    storesize = cstrnlen(char)(sz,size);
    assert( serial->curr + storesize <= serial->end );
    ctl_record_write_container_begin( "string", "char", storesize,storesize, serial, label );
    memcpy( serial->curr, sz, storesize );
    serial->curr += storesize;
}
void wchar_t_record_write_string( const wchar_t* sz, size_t size, ctl_serial* serial, const char* label )
{
    size_t storesize;
    assertconst(sz,1);
    assertconst(label,1);
    assertobjptr(serial);
    storesize = cstrnlen(wchar_t)(sz,size);
    ctl_record_write_container_begin( "string", "wchar_t", chssize(wchar_t,storesize),storesize, serial, label );
    {
        pointer_foreach_const( wchar_t, sz, storesize, curr )
        {
            uint16 ch = (uint16)*curr;
            uint16_serial_write( &ch, serial );
        }
    }
}

bool char_record_read_string( char* sz, size_t size, const ctl_serial* serial, const char* label )
{
    ctl_serial tmp;
    assertptr(sz,size);
    assertconst(label,1);
    assertobjconst(serial);
    tmp = *serial;
    if( ctl_record_find( &tmp, label ) )
    {
        const char* szcontainer = ctl_record_type( &tmp );
        if( cstrcmp(char)(szcontainer,"string") )
            ctl_serial_except( serial, "char_record_read_string: Not a string" );
        {
            const char* szt = ctl_record_contained_type( &tmp );
            size_t rsize = ctl_record_getdata( &tmp );
            if( !cstrcmp(char)(szt,"char") )
            {
                memcpy( sz, tmp.curr, min(size,rsize) );
                if( size > rsize )
                    sz[rsize] = 0;
            }
            else if( !cstrcmp(char)(szt,"type") )
            {
                while( size-- && tmp.curr < tmp.end )
                {
                    uint16 ch;
                    uint16_serial_read( &ch, &tmp );
                    *sz++ = (char)ch;
                }
                if( size > 0 )
                    *sz++ = 0;
            }
            else
            {
                ctl_serial_except( serial, "char_record_read_string: Unknown character type" );
            }
        }
        return true;
    }
    return false;
}

bool wchar_t_record_read_string( wchar_t* sz, size_t size, const ctl_serial* serial, const char* label )
{
    ctl_serial tmp;
    assertconst(sz,1);
    assertconst(label,1);
    assertobjconst(serial);
    tmp = *serial;
    if( ctl_record_find( &tmp, label ) )
    {
        const char* szcontainer = ctl_record_type( &tmp );
        if( cstrcmp(char)(szcontainer,"string") )
            ctl_serial_except( serial, "wchar_t_record_read_string: Not a string" );
        {
            const char* szwchar_t = ctl_record_contained_type( &tmp );
            ctl_record_getdata( &tmp );
            if( !cstrcmp(char)(szwchar_t,"char") )
            {
                const char* psz = (const char*)tmp.curr;
                while( size-- && tmp.curr < tmp.end )
                    *sz++ = (wchar_t)*psz++;
                if( size > 0 )
                    *sz++ = 0;
            }
            else if( !cstrcmp(char)(szwchar_t,"wchar_t") )
            {
                while( size-- && tmp.curr < tmp.end )
                {
                    uint16 ch;
                    uint16_serial_read( &ch, &tmp );
                    *sz++ = (wchar_t)ch;
                }
                if( size > 0 )
                    *sz++ = 0;
            }
            else
            {
                ctl_serial_except( serial, "wchar_t_record_read_string: Unknown character wchar_t" );
            }
        }
        return true;
    }
    return false;
}

bool char_record_read_gstring( ctl_gstring(char)* str, const ctl_serial* serial, const char* label )
{
    ctl_serial tmp;
    assertobjptr(str);
    assertconst(label,1);
    assertobjconst(serial);
    tmp = *serial;
    if( ctl_record_find( &tmp, label ) )
    {
        const char* szcontainer = ctl_record_type( &tmp );
        if( cstrcmp(char)(szcontainer,"string") )
            ctl_serial_except( serial, "char_record_read_string: Not a string" );
        {
            const char* sz = ctl_record_contained_type( &tmp );
            size_t rsize = ctl_record_getdata( &tmp );
            if( !cstrcmp(char)(sz,"char") )
            {
                ctl_gstring_resize(char, str, ctl_chlength(char,rsize) );
                memcpy( str->begin, tmp.curr, rsize );
            }
            else if( !cstrcmp(char)(sz,"wchar_t") )
            {
                char* sz;
                if( rsize & 1 )
                    ctl_serial_except( serial, "char_record_read_string: Odd string length" );
                ctl_gstring_resize(char, str, chslength(wchar_t,rsize) );
                sz = str->begin;
                while( tmp.curr < tmp.end )
                {
                    uint16 ch;
                    uint16_serial_read( &ch, &tmp );
                    *sz++ = (char)ch;
                }
            }
            else
            {
                ctl_serial_except( serial, "char_record_read_string: Unknown character type" );
            }
        }
        return true;
    }
    return false;
}

bool wchar_t_record_read_gstring( ctl_gstring(wchar_t)* str, const ctl_serial* serial, const char* label )
{
    ctl_serial tmp;
    assertobjptr(str);
    assertconst(label,1);
    assertobjconst(serial);
    tmp = *serial;
    if( ctl_record_find( &tmp, label ) )
    {
        const char* szcontainer = ctl_record_type( &tmp );
        if( cstrcmp(char)(szcontainer,"string") )
            ctl_serial_except( serial, "wchar_t_record_read_string: Not a string" );
        {
            const char* szwchar_t = ctl_record_contained_type( &tmp );
            size_t rsize = ctl_record_getdata( &tmp );
            if( !cstrcmp(char)(szwchar_t,"char") )
            {
                wchar_t* sz;
                const char* psz = (const char*)tmp.curr;
                ctl_gstring_resize(wchar_t, str, rsize );
                sz = str->begin;
                while( tmp.curr < tmp.end )
                    *sz++ = (wchar_t)*psz++;
            }
            else if( !cstrcmp(char)(szwchar_t,"wchar_t") )
            {
                wchar_t* sz;
                if( rsize & 1 )
                    ctl_serial_except( serial, "wchar_t_record_read_string: Odd string length" );
                rsize = chslength(wchar_t,rsize);
                ctl_gstring_resize(wchar_t, str, rsize );
                sz = str->begin;
                while( tmp.curr < tmp.end )
                {
                    uint16 ch;
                    uint16_serial_read( &ch, &tmp );
                    *sz++ = (wchar_t)ch;
                }
            }
            else
            {
                ctl_serial_except( serial, "wchar_t_record_read_string: Unknown character wchar_t" );
            }
        }
        return true;
    }
    return false;
}

#if 1

void spew1();

#define X_RECORD_READ(type,unused)\
\
bool ppConcat(type,_record_read)( type* pdata, const ctl_serial* serial, const char* label ) \
{\
    ctl_serial tmp;\
    assertconst(label,1);\
    assertobjconst(serial);\
    assertptr(pdata,sizeof(type));\
    tmp = *serial;\
    if( ctl_record_find( &tmp, label ) )\
    {\
        const char* sz = ctl_record_type( &tmp );\
        ctl_record_getdata( &tmp );\
        DEF_ENUMTYPES_INNER(X_SCALAR_TYPES,type)\
        {\
            ctl_serial_except( serial, #type "_record_read: Invalid scalar type" );\
        }\
    }\
    return false;\
}
#define X_SCALAR_TYPES(type,outertype)\
        if( !strcmp(sz,#type) )\
        {\
            type ppConcat(get_,type);\
            ppConcat(type,_serial_read)( &ppConcat(get_,type), &tmp );\
            ppConcat(outertype,_copy_t)(*pdata,type,ppConcat(get_,type));\
            return true;\
        }\
        else 

DEF_ENUMTYPES(X_RECORD_READ,unused)
#undef X_RECORD_READ
#undef X_SCALAR_TYPES



/* Write & write array of various scalar types */
#define X_RECORD_WRITE(type,unused) \
void ppConcat(type,_record_write)( const type* pdata, ctl_serial* serial, const char* label )\
{\
    assertconst(label,1);\
    assertobjptr(serial);\
    assertconst(pdata,sizeof(type));\
    ctl_record_write_begin( #type, ppConcat(type,_serial_size)( p ), serial, label );\
    ppConcat(type,_serial_write)( pdata, serial );\
}
DEF_ENUMTYPES(X_RECORD_WRITE,unused)
#undef X_RECORD_WRITE

#define X_RECORD_ARRAY_WRITE(type,unused) \
void ppConcat(type,_record_writearray)( const type* pdata, size_t count, ctl_serial* serial, const char* label )\
{\
    assertconst(label,1);\
    assertobjptr(serial);\
    assertconst(pdata,sizeof(type));\
    ctl_record_write_container_begin( "array", #type, ppConcat(type,_serial_size)( p )*count, count, serial, label );\
    {\
        pointer_foreach_const( type, pdata, count, curr )\
            ppConcat(type,_serial_write)( curr, serial );\
    }\
}
DEF_ENUMTYPES(X_RECORD_ARRAY_WRITE,unused)
#undef X_RECORD_ARRAY_WRITE

#define X_RECORD_ARRAY_READ(type,unused) \
\
bool ppConcat(type,_record_readarray)( type* pdata, size_t count, const ctl_serial* serial, const char* label )\
{\
    ctl_serial tmp;\
    assertconst(label,1);\
    assertobjconst(serial);\
    assertptr(pdata,count*sizeof(type));\
    tmp = *serial;\
    if( ctl_record_find( &tmp, label ) )\
    {\
        /*const char* szcontainer = ctl_record_type( &pserial, end );*/\
        const char* sz = ctl_record_contained_type( &tmp );\
        size_t rsize = ctl_record_getdata( &tmp );\
        DEF_ENUMTYPES_INNER(X_SCALAR_TYPES,type)\
        {\
            ctl_serial_except( serial, #type "_record_readarray: Invalid scalar type" );\
        }\
    }\
    return false;\
}
#define X_SCALAR_TYPES(type,outertype)\
        if( !strcmp(sz,#type) )\
        {\
            if( rsize % ppConcat(type,_serial_size)( 0 ) )\
                ctl_serial_except( serial, #outertype "_record_readarray: Odd length array" );\
            {\
                size_t scount = rsize / ppConcat(type,_serial_size)( 0 );\
                if( scount > count )\
                    scount = count;\
                {\
                    pointer_foreach( outertype, pdata, scount, curr )\
                    {\
                        type ppConcat(get_,type);\
                        ppConcat(type,_serial_read)( &ppConcat(get_,type), &tmp );\
                        ppConcat(outertype,_copy_t)(*pdata,type,ppConcat(get_,type));\
                        pdata++;\
                    }\
                    return true;\
                }\
            }\
        }\
        else 

DEF_ENUMTYPES(X_RECORD_ARRAY_READ,unused)
#undef X_RECORD_ARRAY_READ
#undef X_SCALAR_TYPES

#else 

void spew0();

#endif