Tooling/Syntax/TokenBufferTokenManager.h

0001 //===- TokenBufferTokenManager.h  -----------------------------------------===//
0002 //
0003 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 // See https://llvm.org/LICENSE.txt for license information.
0005 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 //
0007 //===----------------------------------------------------------------------===//
0008
0009 #ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKEN_BUFFER_TOKEN_MANAGER_H
0010 #define LLVM_CLANG_TOOLING_SYNTAX_TOKEN_BUFFER_TOKEN_MANAGER_H
0011
0012 #include "clang/Tooling/Syntax/TokenManager.h"
0013 #include "clang/Tooling/Syntax/Tokens.h"
0014
0015 namespace clang {
0016 namespace syntax {
0017
0018 /// A TokenBuffer-powered token manager.
0019 /// It tracks the underlying token buffers, source manager, etc.
0020 class TokenBufferTokenManager : public TokenManager {
0021 public:
0022   TokenBufferTokenManager(const TokenBuffer &Tokens,
0023                           const LangOptions &LangOpts, SourceManager &SourceMgr)
0024       : Tokens(Tokens), LangOpts(LangOpts), SM(SourceMgr) {}
0025
0026   static bool classof(const TokenManager *N) { return N->kind() == Kind; }
0027   llvm::StringLiteral kind() const override { return Kind; }
0028
0029   llvm::StringRef getText(Key I) const override {
0030     const auto *Token = getToken(I);
0031     assert(Token);
0032     // Handle 'eof' separately, calling text() on it produces an empty string.
0033     // FIXME: this special logic is for syntax::Leaf dump, move it when we
0034     // have a direct way to retrive token kind in the syntax::Leaf.
0035     if (Token->kind() == tok::eof)
0036       return "<eof>";
0037     return Token->text(SM);
0038   }
0039
0040   const syntax::Token *getToken(Key I) const {
0041     return reinterpret_cast<const syntax::Token *>(I);
0042   }
0043   SourceManager &sourceManager() { return SM; }
0044   const SourceManager &sourceManager() const { return SM; }
0045   const TokenBuffer &tokenBuffer() const { return Tokens; }
0046
0047 private:
0048   // This manager is powered by the TokenBuffer.
0049   static constexpr llvm::StringLiteral Kind = "TokenBuffer";
0050
0051   /// Add \p Buffer to the underlying source manager, tokenize it and store the
0052   /// resulting tokens. Used exclusively in `FactoryImpl` to materialize tokens
0053   /// that were not written in user code.
0054   std::pair<FileID, ArrayRef<Token>>
0055   lexBuffer(std::unique_ptr<llvm::MemoryBuffer> Buffer);
0056   friend class FactoryImpl;
0057
0058   const TokenBuffer &Tokens;
0059   const LangOptions &LangOpts;
0060
0061   /// The underlying source manager for the ExtraTokens.
0062   SourceManager &SM;
0063   /// IDs and storage for additional tokenized files.
0064   llvm::DenseMap<FileID, std::vector<Token>> ExtraTokens;
0065 };
0066
0067 } // namespace syntax
0068 } // namespace clang
0069
0070 #endif // LLVM_CLANG_TOOLING_SYNTAX_TOKEN_BUFFER_TOKEN_MANAGER_H