ezEngine  Milestone 7
Preprocessor.h
1 #pragma once
2 
3 #include <CoreUtils/Basics.h>
4 #include <CoreUtils/CodeUtils/Tokenizer.h>
5 #include <Foundation/IO/Stream.h>
6 #include <Foundation/Logging/Log.h>
7 #include <Foundation/Containers/Map.h>
8 #include <Foundation/Containers/Set.h>
9 #include <Foundation/Time/Timestamp.h>
10 
13 class EZ_COREUTILS_DLL ezTokenizedFileCache
14 {
15 public:
16  struct FileData
17  {
18  ezTokenizer m_Tokens;
19  ezTimestamp m_Timestamp;
20  };
21 
23  ezMap<ezString, FileData>::ConstIterator Lookup(const ezString& sFileName) const;
24 
26  void Remove(const ezString& sFileName);
27 
29  void Clear();
30 
35  const ezTokenizer* Tokenize(const ezString& sFileName, const ezDynamicArray<ezUInt8>& FileContent, const ezTimestamp& FileTimeStamp, ezLogInterface* pLog);
36 
37 private:
38  void SkipWhitespace(ezDeque<ezToken>& Tokens, ezUInt32& uiCurToken);
39 
40  mutable ezMutex m_Mutex;
42 };
43 
59 class EZ_COREUTILS_DLL ezPreprocessor
60 {
61 public:
62 
65  {
68  GlobalInclude
69  };
70 
73 
80 
85 
88 
94  {
96  enum EventType
97  {
105  };
106 
108  {
109  m_pToken = nullptr;
110  m_szInfo = "";
111  }
112 
113  EventType m_Type;
114 
115  const ezToken* m_pToken;
116  const char* m_szInfo;
117  };
118 
122 
123  ezPreprocessor();
124 
132  void SetLogInterface(ezLogInterface* pLog);
133 
138  void SetCustomFileCache(ezTokenizedFileCache* pFileCache = nullptr);
139 
141  void SetPassThroughPragma(bool bPassThrough) { m_bPassThroughPragma = bPassThrough; }
142 
144  void SetPassThroughLine(bool bPassThrough) { m_bPassThroughLine = bPassThrough; }
145 
147  void SetPassThroughUnknownCmdsCB(PassThroughUnknownCmdCB callback) { m_PassThroughUnknownCmdCB = callback; }
148 
152  void SetFileOpenFunction(FileOpenCB OpenAbsFileCB);
153 
158  void SetFileLocatorFunction(FileLocatorCB LocateAbsFileCB);
159 
168  ezResult AddCustomDefine(const char* szDefinition);
169 
173  ezResult Process(const char* szMainFile, TokenStream& TokenOutput);
174 
179  ezResult Process(const char* szMainFile, ezStringBuilder& sOutput, bool bKeepComments = true, bool bRemoveRedundantWhitespace = false, bool bInsertLine = false);
180 
181 
182 private:
183  struct FileData
184  {
185  FileData()
186  {
187  m_iCurrentLine = 1;
188  m_iExpandDepth = 0;
189  }
190 
191  ezString m_sVirtualFileName;
192  ezString m_sFileName;
193  ezInt32 m_iCurrentLine;
194  ezInt32 m_iExpandDepth;
195  };
196 
197  enum IfDefActivity
198  {
199  IsActive,
200  IsInactive,
201  WasActive,
202  };
203 
205  {
206  ezDynamicArray<ezUInt8> m_Content;
207  ezTokenizer m_Tokenized;
208  };
209 
210  bool m_bPassThroughPragma;
211  bool m_bPassThroughLine;
212  PassThroughUnknownCmdCB m_PassThroughUnknownCmdCB;
213 
214  // this file cache is used as long as the user does not provide his own
215  ezTokenizedFileCache m_InternalFileCache;
216 
217  // pointer to the file cache that is in use
218  ezTokenizedFileCache* m_pUsedFileCache;
219 
220  ezDeque<FileData> m_sCurrentFileStack;
221 
222  ezLogInterface* m_pLog;
223 
224  ezDeque<CustomDefine> m_CustomDefines;
225 
226  struct IfDefState
227  {
228  IfDefState(IfDefActivity ActiveState = IfDefActivity::IsActive) : m_ActiveState(ActiveState), m_bIsInElseClause(false) {}
229 
230  IfDefActivity m_ActiveState;
231  bool m_bIsInElseClause;
232  };
233 
234  ezDeque<IfDefState> m_IfdefActiveStack;
235 
236  ezResult ProcessFile(const char* szFile, TokenStream& TokenOutput);
237  ezResult ProcessCmd(const TokenStream& Tokens, TokenStream& TokenOutput);
238 
239 private: // *** File Handling ***
240  ezResult OpenFile(const char* szFile, const ezTokenizer** pTokenizer);
241  static ezResult DefaultFileLocator(const char* szCurAbsoluteFile, const char* szIncludeFile, ezPreprocessor::IncludeType IncType, ezString& out_sAbsoluteFilePath);
242  static ezResult DefaultFileOpen(const char* szAbsoluteFile, ezDynamicArray<ezUInt8>& FileContent, ezTimestamp& out_FileModification);
243 
244  FileOpenCB m_FileOpenCallback;
245  FileLocatorCB m_FileLocatorCallback;
246  ezSet<ezString> m_PragmaOnce;
247 
248 private: // *** Macro Definition ***
249 
250  bool RemoveDefine(const char* szName);
251  ezResult HandleDefine(const TokenStream& Tokens, ezUInt32& uiCurToken);
252 
254  {
255  MacroDefinition();
256 
257  const ezToken* m_MacroIdentifier;
258  bool m_bIsFunction;
259  bool m_bCurrentlyExpanding;
260  bool m_bHasVarArgs;
261  ezInt32 m_iNumParameters;
262  TokenStream m_Replacement;
263  };
264 
265  ezResult StoreDefine(const ezToken* pMacroNameToken, const TokenStream* pReplacementTokens, ezUInt32 uiFirstReplacementToken, ezInt32 iNumParameters, bool bUsesVarArgs);
266  ezResult ExtractParameterName(const TokenStream& Tokens, ezUInt32& uiCurToken, ezString& sIdentifierName);
267 
269 
270  static const ezInt32 s_MacroParameter0 = ezTokenType::ENUM_COUNT + 2;
271  static ezString s_ParamNames[32];
272  ezToken m_ParameterTokens[32];
273 
274 private: // *** #if condition parsing ***
275 
276  ezResult EvaluateCondition(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
277  ezResult ParseCondition(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
278  ezResult ParseFactor(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
279  ezResult ParseExpressionMul(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
280  ezResult ParseExpressionOr(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
281  ezResult ParseExpressionAnd(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
282  ezResult ParseExpressionPlus(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
283  ezResult ParseExpressionShift(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
284  ezResult ParseExpressionBitOr(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
285  ezResult ParseExpressionBitAnd(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
286  ezResult ParseExpressionBitXor(const TokenStream& Tokens, ezUInt32& uiCurToken, ezInt64& iResult);
287 
288 
289 private: // *** Parsing ***
290  static void SkipWhitespace(const TokenStream& Tokens, ezUInt32& uiCurToken);
291  static void SkipWhitespaceAndNewline(const TokenStream& Tokens, ezUInt32& uiCurToken);
292  static bool IsEndOfLine(const TokenStream& Tokens, ezUInt32 uiCurToken, bool bIgnoreWhitespace);
293  static void CopyRelevantTokens(const TokenStream& Source, ezUInt32 uiFirstSourceToken, TokenStream& Destination, bool bPreserveNewLines);
294  ezResult CopyTokensAndEvaluateDefined(const TokenStream& Source, ezUInt32 uiFirstSourceToken, TokenStream& Destination);
295 
296  bool Accept(const TokenStream& Tokens, ezUInt32& uiCurToken, const char* szToken, ezUInt32* pAccepted = nullptr);
297  bool Accept(const TokenStream& Tokens, ezUInt32& uiCurToken, ezTokenType::Enum Type, ezUInt32* pAccepted = nullptr);
298  bool Accept(const TokenStream& Tokens, ezUInt32& uiCurToken, const char* szToken1, const char* szToken2, ezUInt32* pAccepted = nullptr);
299  bool AcceptUnless(const TokenStream& Tokens, ezUInt32& uiCurToken, const char* szToken1, const char* szToken2, ezUInt32* pAccepted = nullptr);
300 
301  ezResult Expect(const TokenStream& Tokens, ezUInt32& uiCurToken, const char* szToken, ezUInt32* pAccepted = nullptr);
302  ezResult Expect(const TokenStream& Tokens, ezUInt32& uiCurToken, ezTokenType::Enum Type, ezUInt32* pAccepted = nullptr);
303  ezResult Expect(const TokenStream& Tokens, ezUInt32& uiCurToken, const char* szToken1, const char* szToken2, ezUInt32* pAccepted = nullptr);
304  ezResult ExpectEndOfLine(const TokenStream& Tokens, ezUInt32& uiCurToken);
305 
306  void CopyTokensReplaceParams(const TokenStream& Source, ezUInt32 uiFirstSourceToken, TokenStream& Destination, const ezHybridArray<ezString, 16>& parameters);
307  void CombineTokensToString(const TokenStream& Tokens, ezUInt32 uiCurToken, ezStringBuilder& sResult, bool bKeepComments = true, bool bRemoveRedundantWhitespace = false, bool bInsertLine = false);
308  void CombineRelevantTokensToString(const TokenStream& Tokens, ezUInt32 uiCurToken, ezStringBuilder& sResult);
309  void CreateCleanTokenStream(const TokenStream& Tokens, ezUInt32 uiCurToken, TokenStream& Destination, bool bKeepComments);
310 
311 private: // *** Macro Expansion ***
312  ezResult Expand(const TokenStream& Tokens, TokenStream& Output);
313  ezResult ExpandOnce(const TokenStream& Tokens, TokenStream& Output);
314  ezResult ExpandObjectMacro(MacroDefinition& Macro, TokenStream& Output, const ezToken* pMacroToken);
315  ezResult ExpandFunctionMacro(MacroDefinition& Macro, const MacroParameters& Parameters, TokenStream& Output, const ezToken* pMacroToken);
316  ezResult ExpandMacroParam(const ezToken& MacroToken, ezUInt32 uiParam, TokenStream& Output, const MacroDefinition& Macro);
317  void PassThroughFunctionMacro(MacroDefinition& Macro, const MacroParameters& Parameters, TokenStream& Output);
318  ezToken* AddCustomToken(const ezToken* pPrevious, const char* szNewText);
319  void OutputNotExpandableMacro(MacroDefinition& Macro, TokenStream& Output);
320  ezResult ExtractAllMacroParameters(const TokenStream& Tokens, ezUInt32& uiCurToken, ezDeque< TokenStream >& AllParameters);
321  ezResult ExtractParameterValue(const TokenStream& Tokens, ezUInt32& uiCurToken, TokenStream& ParamTokens);
322 
323  ezResult InsertParameters(const TokenStream& Tokens, TokenStream& Output, const MacroDefinition& Macro);
324 
325  ezResult InsertStringifiedParameters(const TokenStream& Tokens, TokenStream& Output, const MacroDefinition& Macro);
326  ezResult ConcatenateParameters(const TokenStream& Tokens, TokenStream& Output, const MacroDefinition& Macro);
327  void MergeTokens(const ezToken* pFirst, const ezToken* pSecond, TokenStream& Output, const MacroDefinition& Macro);
328 
329  struct CustomToken
330  {
331  ezToken m_Token;
332  ezString m_sIdentifierString;
333  };
334 
335  enum TokenFlags : ezUInt32
336  {
337  NoFurtherExpansion = EZ_BIT(0),
338  };
339 
340  ezToken m_TokenFile;
341  ezToken m_TokenLine;
342  const ezToken* m_TokenOpenParenthesis;
343  const ezToken* m_TokenClosedParenthesis;
344  const ezToken* m_TokenComma;
345 
346  ezDeque<const MacroParameters*> m_MacroParamStack;
347  ezDeque<const MacroParameters*> m_MacroParamStackExpanded;
348  ezDeque<CustomToken> m_CustomTokens;
349 
350 private: // *** Other ***
351  static void StringifyTokens(const TokenStream& Tokens, ezStringBuilder& sResult, bool bSurroundWithQuotes);
352  ezToken* CreateStringifiedParameter(ezUInt32 uiParam, const ezToken* pParamToken, const MacroDefinition& Macro);
353 
354  ezResult HandleErrorDirective(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
355  ezResult HandleWarningDirective(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
356  ezResult HandleUndef(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
357 
358  ezResult HandleEndif(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
359  ezResult HandleElif(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
360  ezResult HandleIf(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
361  ezResult HandleElse(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken);
362  ezResult HandleIfdef(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken, bool bIsIfdef);
363  ezResult HandleInclude(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken, TokenStream& TokenOutput);
364  ezResult HandleLine(const TokenStream& Tokens, ezUInt32 uiCurToken, ezUInt32 uiDirectiveToken, TokenStream& TokenOutput);
365 };
366 
367 #define PP_LOG0(Type, FormatStr, ErrorToken) \
368  { \
369  ProcessingEvent pe; \
370  pe.m_Type = ProcessingEvent::Type; \
371  pe.m_pToken = ErrorToken; \
372  pe.m_szInfo = FormatStr; \
373  if (pe.m_pToken->m_uiLine == 0 && pe.m_pToken->m_uiColumn == 0) \
374  { \
375  const_cast<ezToken*>(pe.m_pToken)->m_uiLine = m_sCurrentFileStack.PeekBack().m_iCurrentLine; \
376  const_cast<ezToken*>(pe.m_pToken)->m_File.Assign(m_sCurrentFileStack.PeekBack().m_sVirtualFileName.GetData()); \
377  } \
378  m_ProcessingEvents.Broadcast(pe); \
379  ezLog::Type(m_pLog, "File '%s', Line %u (%u): " FormatStr, pe.m_pToken->m_File.GetString().GetData(), pe.m_pToken->m_uiLine, pe.m_pToken->m_uiColumn); \
380  }
381 
382 #define PP_LOG(Type, FormatStr, ErrorToken, ...) \
383  { \
384  ProcessingEvent pe; \
385  pe.m_Type = ProcessingEvent::Type; \
386  pe.m_pToken = ErrorToken; \
387  if (pe.m_pToken->m_uiLine == 0 && pe.m_pToken->m_uiColumn == 0) \
388  { \
389  const_cast<ezToken*>(pe.m_pToken)->m_uiLine = m_sCurrentFileStack.PeekBack().m_iCurrentLine; \
390  const_cast<ezToken*>(pe.m_pToken)->m_File.Assign(m_sCurrentFileStack.PeekBack().m_sVirtualFileName.GetData()); \
391  } \
392  ezStringBuilder sInfo; \
393  sInfo.Format(FormatStr, __VA_ARGS__); \
394  pe.m_szInfo = sInfo.GetData(); \
395  m_ProcessingEvents.Broadcast(pe); \
396  ezLog::Type(m_pLog, "File '%s', Line %u (%u): " FormatStr, pe.m_pToken->m_File.GetString().GetData(), pe.m_pToken->m_uiLine, pe.m_pToken->m_uiColumn, __VA_ARGS__); \
397  }