Changeset 422

Timestamp:

07/16/15 19:22:35 (10 years ago)

Author:

epyon

Message:

• stream and file_system moved to stl
• hash literal operator
• string table rewrite

Location:

trunk

Files:

• nv/formats/assimp_loader.hh (modified) (1 diff)
• nv/io/c_file_system.hh (modified) (1 diff)
• nv/io/c_stream.hh (modified) (1 diff)
• nv/io/std_stream.hh (modified) (1 diff)
• nv/io/string_table.hh (modified) (1 diff)
• nv/stl/file_system.hh (moved) (moved from trunk/nv/interface/file_system.hh) (2 diffs)
• nv/stl/functional/hash.hh (modified) (1 diff)
• nv/stl/stream.hh (moved) (moved from trunk/nv/interface/stream.hh) (2 diffs)
• src/io/string_table.cc (modified) (1 diff)

trunk/nv/formats/assimp_loader.hh

@@ -37 +37 @@
                 void create_keys( mesh_node_data* data, const void* vnode );
 
-                string_table_creator m_strings;
+                string_table* m_strings;
                 std::string m_ext;
                 uint32 m_assimp_flags;

trunk/nv/io/c_file_system.hh

@@ -15 +15 @@
 
 #include <nv/common.hh>
-#include <nv/interface/file_system.hh>
+#include <nv/stl/file_system.hh>
 
 namespace nv

trunk/nv/io/c_stream.hh

@@ -15 +15 @@
 
 #include <nv/common.hh>
-#include <nv/interface/stream.hh>
+#include <nv/stl/stream.hh>
 
 namespace nv

trunk/nv/io/std_stream.hh

@@ -15 +15 @@
 
 #include <nv/common.hh>
-#include <nv/interface/stream.hh>
+#include <nv/stl/stream.hh>
 #include <streambuf>
 #include <istream>

trunk/nv/io/string_table.hh

@@ -17 +17 @@
 #include <nv/stl/string.hh>
 #include <nv/stl/unordered_map.hh>
-#include <nv/interface/stream.hh>
+#include <nv/stl/stream.hh>
 
 namespace nv
 {
+
         class string_table : noncopyable
         {
         public:
-                typedef uint16 index;
-                typedef uint32 offset;
+                typedef string_view value_type;
+                typedef uint64 key_type;
+                typedef uint64 hash_type;
+                typedef uint32 size_type;
+                typedef uint16 length_type;
+                typedef unordered_map< key_type, size_type > indexer_type;
+                typedef vector< char > storage_type;
+                typedef indexer_type::const_iterator const_iterator;
 
-                string_table( char* data, uint32 size, offset* offsets, index count )
-                        : m_data( data ), m_size( size ), m_offsets( offsets ), m_count( count )
+                string_table() {}
+                string_table( stream& in );
+                void insert( string_table* in );
+                key_type insert( const value_type& str )
                 {
-
-                }
-                explicit string_table( nv::stream* in )
-                {
-                        load( in );
+                        key_type hash_value = str.get_hash< uint64 >();
+                        insert( hash_value, str );
+                        return hash_value;
                 }
 
-                const char* get( index i ) const
+                bool exists( key_type i ) const 
                 {
-                        return i < m_count ? m_data + m_offsets[i] : nullptr;
+                        return m_map.find( i ) != m_map.end();
                 }
 
-                void dump( nv::stream* out ) const
+                value_type at( key_type i ) const
                 {
-                        out->write( &m_count,  sizeof( m_count ), 1 );
-                        out->write( &m_size,   sizeof( m_size ), 1 );
-                        out->write( m_offsets, sizeof( offset ), m_count );
-                        out->write( m_data,    sizeof( char ), m_size );
+                        const auto& it = m_map.find( i );
+                        NV_ASSERT_ALWAYS( it != m_map.end(), "Key not found in string_table!" );
+                        return extract_raw( it->second );
                 }
 
-                ~string_table()
+                value_type operator[]( key_type i ) const
                 {
-                        delete[] m_data;
-                        delete[] m_offsets;
-                }
-        private:
-                void load( stream* in )
-                {
-                        in->read( &m_count, sizeof( m_count ), 1 );
-                        in->read( &m_size, sizeof( m_size ), 1 );
-                        m_offsets = new offset[ m_count ];
-                        m_data    = new char[ m_size ];
-                        in->read( m_offsets, sizeof( offset ), m_count );
-                        in->read( m_data,    sizeof( char ), m_size );
+                        const auto& it = m_map.find( i );
+                        return it != m_map.end() ? extract_raw( it->second ) : value_type();
                 }
 
-                char*   m_data;
-                uint32  m_size;
-                offset* m_offsets;
-                index   m_count;
+                uint32 dump_size() const;
+                void dump( stream& out ) const;
+        protected:
+                void insert( hash_type h, const value_type& str )
+                {
+                        NV_ASSERT_ALWAYS( str.size() < 0xFFF0, "String table can only hold strings up to 64k!" );
+                        auto it = m_map.find( h );
+                        if ( it != m_map.end() )
+                        {
+                                // TODO : perform comparison check if in debug mode!
+                                return;
+                        }
+                        insert_raw( h, str.data(), static_cast<length_type>( str.size() ) );
+                }
+
+
+                size_type insert_raw( hash_type h, const char* str, length_type s );
+
+                value_type extract_raw( size_type index ) const
+                {
+                        uint16 length = *reinterpret_cast<const uint16*>( m_data.data() + index );
+                        return value_type( m_data.data() + index + 2, length );
+                }
+
+                indexer_type m_map;
+                storage_type m_data;
         };
-
-        class string_table_creator
-        {
-        public:
-                typedef string_table::index  index;
-                typedef string_table::offset offset;
-
-                string_table_creator();
-                index insert( const string_view& s );
-                string_table* create_table() const;
-                uint32 dump_size() const;
-                void dump( nv::stream* out ) const;
-                const char* get( index i ) const;
-                index get( const string_view& s ) const;
-                void clear();
-        private:
-                unordered_map< uint64, index > m_map;
-                vector< offset > m_offsets;
-                vector< char >   m_data;
-        };
-
 
 }

trunk/nv/stl/file_system.hh

@@ -10 +10 @@
  */
 
-#ifndef NV_INTERFACE_FILE_SYSTEM_HH
-#define NV_INTERFACE_FILE_SYSTEM_HH
+#ifndef NV_STL_FILE_SYSTEM_HH
+#define NV_STL_FILE_SYSTEM_HH
 
 #include <nv/common.hh>
-#include <nv/interface/stream.hh>
+#include <nv/stl/stream.hh>
 
 namespace nv
@@ -31 +31 @@
 } // namespace nv
 
-#endif // NV_FILE_SYSTEM_HH
+#endif // NV_STL_FILE_SYSTEM_HH

trunk/nv/stl/functional/hash.hh

@@ -203 +203 @@
         };
 
+        constexpr uint32 operator "" _h32( const char* str, size_t len )
+        {
+                return detail::fnv_hash< uint32 >::hash( str, len );
+        }
+
+        constexpr uint64 operator "" _h64( const char* str, size_t len )
+        {
+                return detail::fnv_hash< uint32 >::hash( str, len );
+        }
+
 }
 
+using nv::operator "" _h32;
+using nv::operator "" _h64;
+
 #endif // NV_STL_FUNCTIONAL_HASH_HH

trunk/nv/stl/stream.hh

@@ -11 +11 @@
  */
 
-#ifndef NV_INTERFACE_STREAM_HH
-#define NV_INTERFACE_STREAM_HH
+#ifndef NV_STL_STREAM_HH
+#define NV_STL_STREAM_HH
 
 #include <nv/common.hh>
@@ -42 +42 @@
 } // namespace nv
 
-#endif // NV_INTERFACE_STREAM_HH
+#endif // NV_STL_STREAM_HH

trunk/src/io/string_table.cc

@@ -7 +7 @@
 #include "nv/io/string_table.hh"
 
-nv::string_table_creator::string_table_creator()
+nv::string_table::string_table( stream& in )
 {
-        insert(""); // 0 always is empty string
+        uint32 entry_count = 0;
+        in.read( &entry_count, sizeof( entry_count ), 1 );
+        m_map.reserve( entry_count );
+        indexer_type::value_type entry;
+        for ( uint32 i = 0; i < entry_count; )
+        {
+                in.read( &entry, sizeof( entry ), 1 );
+                // TODO: this performs a existence check -
+                //   using a per-container stream implementation would avoid this!
+                m_map.insert( entry );
+        }
+
+        uint32 data_size = 0;
+        in.read( &data_size, sizeof( data_size ), 1 );
+        // TODO: either no-init resize, or per-container implementation?
+        m_data.resize( data_size );
+        in.read( m_data.data(), data_size, 1 );
 }
 
-nv::string_table_creator::index nv::string_table_creator::insert( const string_view& s )
+void nv::string_table::insert( string_table* in )
 {
-        uint64 hash_value = hash_string< uint64 >( s.data() );
-        auto i = m_map.find( hash_value );
-        if ( i != m_map.end() )
+        m_data.reserve( m_data.size() + in->m_data.size() );
+        m_map.reserve( m_map.size() + in->m_map.size() );
+        for ( const auto& hash_index : in->m_map )
         {
-                return i->second;
+                insert( hash_index.first, in->extract_raw( hash_index.second ) );
         }
-        const char* cs = s.data();
-        uint32 cs_size = s.size() + 1;
-        NV_ASSERT( m_offsets.size() < index(-1), "Too many strings!" );
-        index  result = index( m_offsets.size() );
-        size_t dsize = m_data.size();
-        m_offsets.push_back( dsize );
-        m_data.resize( dsize + cs_size );
-        raw_copy( cs, cs + cs_size, m_data.data() + dsize );
-        m_map[ hash_value ] = result;
-        return result;
 }
 
-nv::string_table* nv::string_table_creator::create_table() const
+nv::uint32 nv::string_table::dump_size() const
 {
-        offset* offsets = new offset[m_offsets.size()];
-        char*   data    = new char [m_data.size()];
-        raw_copy( m_offsets.begin(), m_offsets.end(), offsets );
-        raw_copy( m_data.begin(),    m_data.end(),    data );
-        return new string_table( data, m_data.size(), offsets, index( m_offsets.size() ) );
+        return sizeof( uint32 ) + sizeof( indexer_type::value_type ) * m_map.size() +
+                sizeof( uint32 ) + m_data.size();
 }
 
-void nv::string_table_creator::dump( nv::stream* out ) const
+void nv::string_table::dump( nv::stream& out ) const
 {
-        index  count = index( m_offsets.size() );
-        uint32 size  = m_data.size();
-        out->write( &count,  sizeof( count ), 1 );
-        out->write( &size,   sizeof( size ), 1 );
-        out->write( m_offsets.data(), sizeof( offset ), count );
-        out->write( m_data.data(),    sizeof( char ),  size );
+        // TODO these should be generic stream operations
+        uint32 entry_count = m_map.size();
+        out.write( &entry_count, sizeof( entry_count ), 1 );
+        for ( const auto& entry : m_map )
+        {
+                out.write( &entry, sizeof( entry ), 1 );
+        }
+
+        uint32 data_size = m_data.size();
+        out.write( &data_size, sizeof( data_size ), 1 );
+        // TODO: either no-init resize, or per-container implementation?
+        out.write( m_data.data(), data_size, 1 );
 }
 
-const char* nv::string_table_creator::get( index i ) const
+nv::string_table::size_type  nv::string_table::insert_raw( hash_type h, const char* str, length_type length )
 {
-        return i < m_offsets.size() ? m_data.data() + m_offsets[i] : nullptr;
+        size_type dsize  = static_cast<uint32>( m_data.size() );
+        // TODO : resize without init!
+        m_data.resize( dsize + 2 + length + 1 );
+        *reinterpret_cast<uint16*>( m_data.data() + dsize ) = length;
+        raw_copy( str, str + length + 1, m_data.data() + dsize + 2 );
+        m_map[h] = dsize;
+        return dsize;
 }
 
-nv::string_table_creator::index nv::string_table_creator::get( const string_view& s ) const
-{
-        uint64 hash_value = hash_string< uint64 >( s.data() );
-        auto i = m_map.find( hash_value );
-        if ( i != m_map.end() )
-        {
-                return i->second;
-        }
-        return 0;
-}
-
-void nv::string_table_creator::clear()
-{
-        m_map.clear();
-        m_offsets.clear();
-        m_data.clear();
-}
-
-nv::uint32 nv::string_table_creator::dump_size() const
-{
-        return sizeof( index ) + sizeof( uint32 ) +
-                sizeof( offset ) * m_offsets.size() +
-                sizeof( char ) * m_data.size();
-}
-