RmUtilTokenizer.h

//=================================================================================================//
// filename: RmUtilTokenizer.h                                                                     //
//                                                                                                 //
//           ATI Research, Inc.                                                                    //
//           3D Application Research Group                                                         //
//                                                                                                 //
// Description: declaration file for Tokenizer Utility.                                            //
//                                                                                                 //
//=================================================================================================//
//   (C) 2004 ATI Research, Inc.  All rights reserved.                                             //
//=================================================================================================//

#ifndef _RM_UTILITIES_TOKENIZER_H_
#define _RM_UTILITIES_TOKENIZER_H_

//...........................................................................
//...........................................................................
//...........................................................................
//  RmUtilTokenizer
//...........................................................................
//...........................................................................
//...........................................................................
class RMUTIL_API RmUtilTokenizer
{
public : // Sub Classes / Types
   enum CharType
   {
      CHARTYPE_NUMBER,
      CHARTYPE_CHAR,
      CHARTYPE_OPERATOR,
      CHARTYPE_WHITESPACE
   }; // End of CharType

   enum TokenType
   {
      TOKEN_UNKNOWN,
      TOKEN_KEYWORD,
      TOKEN_OPERATOR,
      TOKEN_IDENTIFIER,
      TOKEN_INTEGER,
      TOKEN_FLOAT,
      TOKEN_INVALID,
      TOKEN_EOF
   }; // End of TokenType

   //----------------------------------------------------------------------
   // Keyword
   //----------------------------------------------------------------------
   class RMUTIL_API Keyword
   {
   public :
      Keyword( const RM_TCHAR *szWord, int id ) : 
          m_word(szWord), 
          m_id(id)
      {
      }; // End of Constructor

      const RmStringT& GetWord() const { return m_word; };
      int   GetID() const { return m_id; };

   private :
      RmStringT m_word;
      int       m_id;
   }; // End of Keyword

   //----------------------------------------------------------------------
   // Comment Block
   //----------------------------------------------------------------------
   class RMUTIL_API CommentBlock
   {
   public :
      CommentBlock( const RM_TCHAR *szStart, const RM_TCHAR *szEnd ) :
         m_startOfComment(szStart),
         m_endOfComment(szEnd)
      {
      }; // Ebdof Constructor

      ~CommentBlock() {};

      const RmStringT& GetStartOfComment() const { return m_startOfComment; };
      const RmStringT& GetEndOfComment() const { return m_endOfComment; };

   private :
      RmStringT  m_startOfComment;
      RmStringT  m_endOfComment;
   }; // End of CommentBlock

   //----------------------------------------------------------------------
   // Token
   //----------------------------------------------------------------------
   class RMUTIL_API Token
   {
      friend RmUtilTokenizer;

   public :
      Token() {};
      ~Token() {};

      TokenType GetTokenType() const { return m_tokenType; };

      const RmStringT& GetText() const { return m_text; };
      int   GetID() const { return m_id; };

      double GetFloatNumber() const { return m_floatNumber; };
      int    GetIntegerNumber() const { return m_integerNumber; };

   private :
      TokenType m_tokenType;

      RmStringT m_text;

      int       m_id;   // Keyword or Operator 

      double    m_floatNumber;
      int       m_integerNumber;
   }; // End of Token


   //----------------------------------------------------------------------
   // ParseParam
   //----------------------------------------------------------------------
   class RMUTIL_API ParseParam
   {
   public :
      const RM_TCHAR *m_pBuffer;
      int             m_size;
      int             m_curPos;

      //-------------------------------------------------------------------
      // Returns current char and advances position
      //-------------------------------------------------------------------
      RM_TCHAR GetChar( bool bAdvance = true )
      {
         assert(IsEndReached()==false);
         RM_TCHAR ch = m_pBuffer[m_curPos];
         if (bAdvance)
            m_curPos++;

         return ch;
      }; // End of GetChar

      //-------------------------------------------------------------------
      bool IsEndReached() const 
      {
         if (m_curPos>=m_size)
            return true;

         return false;
      }; // End of IsEnedReached
   }; // End of ParseParam

public :
   RmUtilTokenizer();
   virtual ~RmUtilTokenizer();

   //------------------------------------------------------------------
   // Init/Uninit
   //------------------------------------------------------------------
   virtual void Initialize();
   virtual void Uninitialize();

   //------------------------------------------------------------------
   // Parsing
   //------------------------------------------------------------------
   void BeginParsing( const RM_TCHAR *pBuffer, int sizeOfBuffer );

      void ParseNext();
      bool IsEndReached() const;

   void EndParsing();

   //------------------------------------------------------------------
   // Keyword
   //------------------------------------------------------------------
   void AddKeyword( const RM_TCHAR *szKeyword, int id );
   void AddOperator( const RM_TCHAR *szOperator, int id );

   //------------------------------------------------------------------
   // Comment Blocks
   //------------------------------------------------------------------
   void AddComment( const RM_TCHAR *szStartOfComment, 
                    const RM_TCHAR *szEndOfComment );

   //------------------------------------------------------------------
   // Current Token
   //------------------------------------------------------------------
   const Token* GetCurrentToken() const { return &m_currentToken; };

protected :
   virtual void SetDefaultCharTypes();

private :
   //------------------------------------------------------------------
   // Character Types
   //------------------------------------------------------------------
   CharType  m_charTypes[65536];

   //------------------------------------------------------------------
   // Hash Map for Keywords
   //
   // - Use first character as index to list ( hash key )
   //------------------------------------------------------------------
   RmLinkedList<Keyword*> m_keywordMap[256];
   RmLinkedList<Keyword*> m_operatorMap[256];

   int GetHashKey( const RM_TCHAR *szWord );
   void RemoveAllKeywords();
   void RemoveAllOperators();

   const Keyword* FindKeyword( const RM_TCHAR *szWord );
   const Keyword* FindOperator( const RM_TCHAR *szWord );

   //------------------------------------------------------------------
   // Comment
   //------------------------------------------------------------------
   RmLinkedList<CommentBlock*> m_commentBlocks;

   void RemoveAllCommentBlocks();

   //------------------------------------------------------------------
   // Current Token
   //------------------------------------------------------------------
   Token      m_currentToken;

   //------------------------------------------------------------------
   // Parsing
   //------------------------------------------------------------------
   ParseParam m_parseParam;

   CharType GetCharType( RM_TCHAR ch ) const { return m_charTypes[ch]; };

   void SkipWhitespace( ParseParam &parseParam );   

   // Return true if comment was processed
   bool ProcessComment( ParseParam &parseParam );

   // Find starting comment block at current position
   CommentBlock* GetCommentBlock( ParseParam &parseParam );

   // Get to end of comment
   void ProcessToEndOfComment( ParseParam &parseParam, CommentBlock *pBlock );

   // Process Number
   void ProcessNumber( ParseParam &parseParam );

   // Process Text ( Keyword or Identifier )
   void ProcessText( ParseParam &parseParam );

   // Process Operator
   void ProcessOperator( ParseParam &parseParam );
}; // End of RmUtilTokenizer

#endif

---