[clangd] Remove support for pre-standard semanticHighlighting notification

This is obsoleted by the standard semanticTokens request family.
As well as the protocol details, this allows us to remove a bunch of plumbing
around pushing highlights to clients.

This should not land until the new protocol has feature parity, see D77702.

Differential Revision: https://reviews.llvm.org/D95576
This commit is contained in:
Sam McCall 2021-01-28 01:16:47 +01:00
parent 5e77ea04f2
commit 4dc8365f80
13 changed files with 12 additions and 740 deletions

View File

@ -117,18 +117,6 @@ CompletionItemKindBitset defaultCompletionItemKinds() {
return Defaults;
}
// Build a lookup table (HighlightingKind => {TextMate Scopes}), which is sent
// to the LSP client.
std::vector<std::vector<std::string>> buildHighlightScopeLookupTable() {
std::vector<std::vector<std::string>> LookupTable;
// HighlightingKind is using as the index.
for (int KindValue = 0; KindValue <= (int)HighlightingKind::LastKind;
++KindValue)
LookupTable.push_back(
{std::string(toTextMateScope((HighlightingKind)(KindValue)))});
return LookupTable;
}
// Makes sure edits in \p FE are applicable to latest file contents reported by
// editor. If not generates an error message containing information about files
// that needs to be saved.
@ -511,18 +499,10 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
}
}
Opts.TheiaSemanticHighlighting =
Params.capabilities.TheiaSemanticHighlighting;
if (Params.capabilities.TheiaSemanticHighlighting &&
Params.capabilities.SemanticTokens) {
log("Client supports legacy semanticHighlights notification and standard "
"semanticTokens request, choosing the latter (no notifications).");
Opts.TheiaSemanticHighlighting = false;
}
if (Opts.TheiaSemanticHighlighting) {
log("Using legacy semanticHighlights notification, which will be removed "
"in clangd 13. Clients should use the standard semanticTokens "
"request instead.");
!Params.capabilities.SemanticTokens) {
elog("Client requested legacy semanticHighlights notification, which is "
"no longer supported. Migrate to standard semanticTokens request");
}
if (Params.rootUri && *Params.rootUri)
@ -674,11 +654,6 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
}}}};
if (Opts.Encoding)
Result["offsetEncoding"] = *Opts.Encoding;
if (Opts.TheiaSemanticHighlighting)
Result.getObject("capabilities")
->insert(
{"semanticHighlighting",
llvm::json::Object{{"scopes", buildHighlightScopeLookupTable()}}});
if (Opts.FoldingRanges)
Result.getObject("capabilities")->insert({"foldingRangeProvider", true});
Reply(std::move(Result));
@ -898,10 +873,6 @@ void ClangdLSPServer::onDocumentDidClose(
std::lock_guard<std::mutex> Lock(FixItsMutex);
FixItsMap.erase(File);
}
{
std::lock_guard<std::mutex> HLock(HighlightingsMutex);
FileToHighlightings.erase(File);
}
{
std::lock_guard<std::mutex> HLock(SemanticTokensMutex);
LastSemanticTokens.erase(File);
@ -1313,11 +1284,6 @@ void ClangdLSPServer::applyConfiguration(
[&](llvm::StringRef File) { return ModifiedFiles.count(File) != 0; });
}
void ClangdLSPServer::publishTheiaSemanticHighlighting(
const TheiaSemanticHighlightingParams &Params) {
notify("textDocument/semanticHighlighting", Params);
}
void ClangdLSPServer::publishDiagnostics(
const PublishDiagnosticsParams &Params) {
notify("textDocument/publishDiagnostics", Params);
@ -1628,27 +1594,6 @@ bool ClangdLSPServer::shouldRunCompletion(
return allowImplicitCompletion(Code->Contents, *Offset);
}
void ClangdLSPServer::onHighlightingsReady(
PathRef File, llvm::StringRef Version,
std::vector<HighlightingToken> Highlightings) {
std::vector<HighlightingToken> Old;
std::vector<HighlightingToken> HighlightingsCopy = Highlightings;
{
std::lock_guard<std::mutex> Lock(HighlightingsMutex);
Old = std::move(FileToHighlightings[File]);
FileToHighlightings[File] = std::move(HighlightingsCopy);
}
// LSP allows us to send incremental edits of highlightings. Also need to diff
// to remove highlightings from tokens that should no longer have them.
std::vector<LineHighlightings> Diffed = diffHighlightings(Highlightings, Old);
TheiaSemanticHighlightingParams Notification;
Notification.TextDocument.uri =
URIForFile::canonicalize(File, /*TUPath=*/File);
Notification.TextDocument.version = decodeVersion(Version);
Notification.Lines = toTheiaSemanticHighlightingInformation(Diffed);
publishTheiaSemanticHighlighting(Notification);
}
void ClangdLSPServer::onDiagnosticsReady(PathRef File, llvm::StringRef Version,
std::vector<Diag> Diagnostics) {
PublishDiagnosticsParams Notification;

View File

@ -84,9 +84,6 @@ private:
void onDiagnosticsReady(PathRef File, llvm::StringRef Version,
std::vector<Diag> Diagnostics) override;
void onFileUpdated(PathRef File, const TUStatus &Status) override;
void
onHighlightingsReady(PathRef File, llvm::StringRef Version,
std::vector<HighlightingToken> Highlightings) override;
void onBackgroundIndexProgress(const BackgroundQueue::Stats &Stats) override;
// LSP methods. Notifications have signature void(const Params&).
@ -179,10 +176,6 @@ private:
llvm::function_ref<bool(llvm::StringRef File)> Filter);
void applyConfiguration(const ConfigurationSettings &Settings);
/// Sends a "publishSemanticHighlighting" notification to the LSP client.
void
publishTheiaSemanticHighlighting(const TheiaSemanticHighlightingParams &);
/// Sends a "publishDiagnostics" notification to the LSP client.
void publishDiagnostics(const PublishDiagnosticsParams &);
@ -214,8 +207,6 @@ private:
DiagnosticToReplacementMap;
/// Caches FixIts per file and diagnostics
llvm::StringMap<DiagnosticToReplacementMap> FixItsMap;
std::mutex HighlightingsMutex;
llvm::StringMap<std::vector<HighlightingToken>> FileToHighlightings;
// Last semantic-tokens response, for incremental requests.
std::mutex SemanticTokensMutex;
llvm::StringMap<SemanticTokens> LastSemanticTokens;

View File

@ -65,10 +65,8 @@ namespace {
// Update the FileIndex with new ASTs and plumb the diagnostics responses.
struct UpdateIndexCallbacks : public ParsingCallbacks {
UpdateIndexCallbacks(FileIndex *FIndex,
ClangdServer::Callbacks *ServerCallbacks,
bool TheiaSemanticHighlighting)
: FIndex(FIndex), ServerCallbacks(ServerCallbacks),
TheiaSemanticHighlighting(TheiaSemanticHighlighting) {}
ClangdServer::Callbacks *ServerCallbacks)
: FIndex(FIndex), ServerCallbacks(ServerCallbacks) {}
void onPreambleAST(PathRef Path, llvm::StringRef Version, ASTContext &Ctx,
std::shared_ptr<clang::Preprocessor> PP,
@ -82,17 +80,10 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
FIndex->updateMain(Path, AST);
std::vector<Diag> Diagnostics = AST.getDiagnostics();
std::vector<HighlightingToken> Highlightings;
if (TheiaSemanticHighlighting)
Highlightings = getSemanticHighlightings(AST);
if (ServerCallbacks)
Publish([&]() {
ServerCallbacks->onDiagnosticsReady(Path, AST.version(),
std::move(Diagnostics));
if (TheiaSemanticHighlighting)
ServerCallbacks->onHighlightingsReady(Path, AST.version(),
std::move(Highlightings));
});
}
@ -111,7 +102,6 @@ struct UpdateIndexCallbacks : public ParsingCallbacks {
private:
FileIndex *FIndex;
ClangdServer::Callbacks *ServerCallbacks;
bool TheiaSemanticHighlighting;
};
} // namespace
@ -121,7 +111,6 @@ ClangdServer::Options ClangdServer::optsForTest() {
Opts.UpdateDebounce = DebouncePolicy::fixed(/*zero*/ {});
Opts.StorePreamblesInMemory = true;
Opts.AsyncThreadsCount = 4; // Consistent!
Opts.TheiaSemanticHighlighting = true;
return Opts;
}
@ -149,8 +138,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
// critical paths.
WorkScheduler(
CDB, TUScheduler::Options(Opts),
std::make_unique<UpdateIndexCallbacks>(
DynamicIdx.get(), Callbacks, Opts.TheiaSemanticHighlighting)) {
std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), Callbacks)) {
// Adds an index to the stack, at higher priority than existing indexes.
auto AddIndex = [&](SymbolIndex *Idx) {
if (this->Index != nullptr) {

View File

@ -68,12 +68,6 @@ public:
/// May be called concurrently for separate files, not for a single file.
virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
/// Called by ClangdServer when some \p Highlightings for \p File are ready.
/// May be called concurrently for separate files, not for a single file.
virtual void
onHighlightingsReady(PathRef File, llvm::StringRef Version,
std::vector<HighlightingToken> Highlightings) {}
/// Called when background indexing tasks are enqueued/started/completed.
/// Not called concurrently.
virtual void
@ -145,9 +139,6 @@ public:
/// fetch system include path.
std::vector<std::string> QueryDriverGlobs;
/// Enable notification-based semantic highlighting.
bool TheiaSemanticHighlighting = false;
/// Enable preview of FoldingRanges feature.
bool FoldingRanges = false;

View File

@ -1314,25 +1314,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, OffsetEncoding Enc) {
return OS << toString(Enc);
}
bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
const TheiaSemanticHighlightingInformation &Rhs) {
return Lhs.Line == Rhs.Line && Lhs.Tokens == Rhs.Tokens;
}
llvm::json::Value
toJSON(const TheiaSemanticHighlightingInformation &Highlighting) {
return llvm::json::Object{{"line", Highlighting.Line},
{"tokens", Highlighting.Tokens},
{"isInactive", Highlighting.IsInactive}};
}
llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting) {
return llvm::json::Object{
{"textDocument", Highlighting.TextDocument},
{"lines", std::move(Highlighting.Lines)},
};
}
bool fromJSON(const llvm::json::Value &Params, SelectionRangeParams &S,
llvm::json::Path P) {
llvm::json::ObjectMapper O(Params, P);

View File

@ -449,9 +449,8 @@ struct ClientCapabilities {
bool SemanticTokens = false;
/// Client supports Theia semantic highlighting extension.
/// https://github.com/microsoft/vscode-languageserver-node/pull/367
/// This will be ignored if the client also supports semanticTokens.
/// clangd no longer supports this, we detect it just to log a warning.
/// textDocument.semanticHighlightingCapabilities.semanticHighlighting
/// FIXME: drop this support once clients support LSP 3.16 Semantic Tokens.
bool TheiaSemanticHighlighting = false;
/// Supported encodings for LSP character offsets. (clangd extension).
@ -1566,33 +1565,6 @@ struct SemanticTokensOrDelta {
};
llvm::json::Value toJSON(const SemanticTokensOrDelta &);
/// Represents a semantic highlighting information that has to be applied on a
/// specific line of the text document.
struct TheiaSemanticHighlightingInformation {
/// The line these highlightings belong to.
int Line = 0;
/// The base64 encoded string of highlighting tokens.
std::string Tokens;
/// Is the line in an inactive preprocessor branch?
/// This is a clangd extension.
/// An inactive line can still contain highlighting tokens as well;
/// clients should combine line style and token style if possible.
bool IsInactive = false;
};
bool operator==(const TheiaSemanticHighlightingInformation &Lhs,
const TheiaSemanticHighlightingInformation &Rhs);
llvm::json::Value
toJSON(const TheiaSemanticHighlightingInformation &Highlighting);
/// Parameters for the semantic highlighting (server-side) push notification.
struct TheiaSemanticHighlightingParams {
/// The textdocument these highlightings belong to.
VersionedTextDocumentIdentifier TextDocument;
/// The lines of highlightings that should be sent.
std::vector<TheiaSemanticHighlightingInformation> Lines;
};
llvm::json::Value toJSON(const TheiaSemanticHighlightingParams &Highlighting);
struct SelectionRangeParams {
/// The text document.
TextDocumentIdentifier textDocument;

View File

@ -526,29 +526,6 @@ public:
private:
HighlightingsBuilder &H;
};
void write32be(uint32_t I, llvm::raw_ostream &OS) {
std::array<char, 4> Buf;
llvm::support::endian::write32be(Buf.data(), I);
OS.write(Buf.data(), Buf.size());
}
void write16be(uint16_t I, llvm::raw_ostream &OS) {
std::array<char, 2> Buf;
llvm::support::endian::write16be(Buf.data(), I);
OS.write(Buf.data(), Buf.size());
}
// Get the highlightings on \c Line where the first entry of line is at \c
// StartLineIt. If it is not at \c StartLineIt an empty vector is returned.
ArrayRef<HighlightingToken>
takeLine(ArrayRef<HighlightingToken> AllTokens,
ArrayRef<HighlightingToken>::iterator StartLineIt, int Line) {
return ArrayRef<HighlightingToken>(StartLineIt, AllTokens.end())
.take_while([Line](const HighlightingToken &Token) {
return Token.R.start.line == Line;
});
}
} // namespace
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
@ -656,64 +633,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
}
}
std::vector<LineHighlightings>
diffHighlightings(ArrayRef<HighlightingToken> New,
ArrayRef<HighlightingToken> Old) {
assert(std::is_sorted(New.begin(), New.end()) &&
"New must be a sorted vector");
assert(std::is_sorted(Old.begin(), Old.end()) &&
"Old must be a sorted vector");
// FIXME: There's an edge case when tokens span multiple lines. If the first
// token on the line started on a line above the current one and the rest of
// the line is the equal to the previous one than we will remove all
// highlights but the ones for the token spanning multiple lines. This means
// that when we get into the LSP layer the only highlights that will be
// visible are the ones for the token spanning multiple lines.
// Example:
// EndOfMultilineToken Token Token Token
// If "Token Token Token" don't differ from previously the line is
// incorrectly removed. Suggestion to fix is to separate any multiline tokens
// into one token for every line it covers. This requires reading from the
// file buffer to figure out the length of each line though.
std::vector<LineHighlightings> DiffedLines;
// ArrayRefs to the current line in the highlightings.
ArrayRef<HighlightingToken> NewLine(New.begin(),
/*length*/ static_cast<size_t>(0));
ArrayRef<HighlightingToken> OldLine(Old.begin(),
/*length*/ static_cast<size_t>(0));
auto NewEnd = New.end();
auto OldEnd = Old.end();
auto NextLineNumber = [&]() {
int NextNew = NewLine.end() != NewEnd ? NewLine.end()->R.start.line
: std::numeric_limits<int>::max();
int NextOld = OldLine.end() != OldEnd ? OldLine.end()->R.start.line
: std::numeric_limits<int>::max();
return std::min(NextNew, NextOld);
};
for (int LineNumber = 0; NewLine.end() < NewEnd || OldLine.end() < OldEnd;
LineNumber = NextLineNumber()) {
NewLine = takeLine(New, NewLine.end(), LineNumber);
OldLine = takeLine(Old, OldLine.end(), LineNumber);
if (NewLine != OldLine) {
DiffedLines.push_back({LineNumber, NewLine, /*IsInactive=*/false});
// Turn a HighlightingKind::InactiveCode token into the IsInactive flag.
auto &AddedLine = DiffedLines.back();
llvm::erase_if(AddedLine.Tokens, [&](const HighlightingToken &T) {
if (T.Kind == HighlightingKind::InactiveCode) {
AddedLine.IsInactive = true;
return true;
}
return false;
});
}
}
return DiffedLines;
}
bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
return std::tie(L.R, L.Kind, L.Modifiers) ==
std::tie(R.R, R.Kind, R.Modifiers);
@ -722,9 +641,6 @@ bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
return std::tie(L.R, L.Kind, R.Modifiers) <
std::tie(R.R, R.Kind, R.Modifiers);
}
bool operator==(const LineHighlightings &L, const LineHighlightings &R) {
return std::tie(L.Line, L.Tokens) == std::tie(R.Line, R.Tokens);
}
std::vector<SemanticToken>
toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens) {
@ -829,87 +745,6 @@ llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
llvm_unreachable("unhandled HighlightingModifier");
}
std::vector<TheiaSemanticHighlightingInformation>
toTheiaSemanticHighlightingInformation(
llvm::ArrayRef<LineHighlightings> Tokens) {
if (Tokens.size() == 0)
return {};
// FIXME: Tokens might be multiple lines long (block comments) in this case
// this needs to add multiple lines for those tokens.
std::vector<TheiaSemanticHighlightingInformation> Lines;
Lines.reserve(Tokens.size());
for (const auto &Line : Tokens) {
llvm::SmallVector<char> LineByteTokens;
llvm::raw_svector_ostream OS(LineByteTokens);
for (const auto &Token : Line.Tokens) {
// Writes the token to LineByteTokens in the byte format specified by the
// LSP proposal. Described below.
// |<---- 4 bytes ---->|<-- 2 bytes -->|<--- 2 bytes -->|
// | character | length | index |
write32be(Token.R.start.character, OS);
write16be(Token.R.end.character - Token.R.start.character, OS);
write16be(static_cast<int>(Token.Kind), OS);
}
Lines.push_back({Line.Line, encodeBase64(LineByteTokens), Line.IsInactive});
}
return Lines;
}
llvm::StringRef toTextMateScope(HighlightingKind Kind) {
// FIXME: Add scopes for C and Objective C.
switch (Kind) {
case HighlightingKind::Function:
return "entity.name.function.cpp";
case HighlightingKind::Method:
return "entity.name.function.method.cpp";
case HighlightingKind::StaticMethod:
return "entity.name.function.method.static.cpp";
case HighlightingKind::Variable:
return "variable.other.cpp";
case HighlightingKind::LocalVariable:
return "variable.other.local.cpp";
case HighlightingKind::Parameter:
return "variable.parameter.cpp";
case HighlightingKind::Field:
return "variable.other.field.cpp";
case HighlightingKind::StaticField:
return "variable.other.field.static.cpp";
case HighlightingKind::Class:
return "entity.name.type.class.cpp";
case HighlightingKind::Enum:
return "entity.name.type.enum.cpp";
case HighlightingKind::EnumConstant:
return "variable.other.enummember.cpp";
case HighlightingKind::Typedef:
return "entity.name.type.typedef.cpp";
case HighlightingKind::Type:
// Fragile: all paths emitting `Type` are dependent names for now.
// But toTextMateScope is going away soon.
return "entity.name.type.dependent.cpp";
case HighlightingKind::Unknown:
// Fragile: all paths emitting `Unknown` are dependent names for now.
// But toTextMateScope is going away soon.
return "entity.name.other.dependent.cpp";
case HighlightingKind::Namespace:
return "entity.name.namespace.cpp";
case HighlightingKind::TemplateParameter:
return "entity.name.type.template.cpp";
case HighlightingKind::Concept:
return "entity.name.type.concept.cpp";
case HighlightingKind::Primitive:
return "storage.type.primitive.cpp";
case HighlightingKind::Macro:
return "entity.name.function.preprocessor.cpp";
case HighlightingKind::InactiveCode:
return "meta.disabled";
}
llvm_unreachable("unhandled HighlightingKind");
}
std::vector<SemanticTokensEdit>
diffTokens(llvm::ArrayRef<SemanticToken> Old,
llvm::ArrayRef<SemanticToken> New) {

View File

@ -8,19 +8,9 @@
//
// This file supports semantic highlighting: categorizing tokens in the file so
// that the editor can color/style them differently.
//
// This is particularly valuable for C++: its complex and context-dependent
// grammar is a challenge for simple syntax-highlighting techniques.
//
// We support two protocols for providing highlights to the client:
// - the `textDocument/semanticTokens` request from LSP 3.16
// https://github.com/microsoft/vscode-languageserver-node/blob/release/protocol/3.16.0-next.1/protocol/src/protocol.semanticTokens.proposed.ts
// - the earlier proposed `textDocument/semanticHighlighting` notification
// https://github.com/microsoft/vscode-languageserver-node/pull/367
// This is referred to as "Theia" semantic highlighting in the code.
// It was supported from clangd 9 but should be considered deprecated as of
// clangd 11 and eventually removed.
//
// Semantic highlightings are calculated for an AST by visiting every AST node
// and classifying nodes that are interesting to highlight (variables/function
// calls etc.).
@ -103,15 +93,6 @@ struct HighlightingToken {
bool operator==(const HighlightingToken &L, const HighlightingToken &R);
bool operator<(const HighlightingToken &L, const HighlightingToken &R);
/// Contains all information about highlightings on a single line.
struct LineHighlightings {
int Line;
std::vector<HighlightingToken> Tokens;
bool IsInactive;
};
bool operator==(const LineHighlightings &L, const LineHighlightings &R);
// Returns all HighlightingTokens from an AST. Only generates highlights for the
// main AST.
std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST);
@ -122,28 +103,6 @@ llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier);
std::vector<SemanticTokensEdit> diffTokens(llvm::ArrayRef<SemanticToken> Before,
llvm::ArrayRef<SemanticToken> After);
/// Converts a HighlightingKind to a corresponding TextMate scope
/// (https://manual.macromates.com/en/language_grammars).
llvm::StringRef toTextMateScope(HighlightingKind Kind);
/// Convert to LSP's semantic highlighting information.
std::vector<TheiaSemanticHighlightingInformation>
toTheiaSemanticHighlightingInformation(
llvm::ArrayRef<LineHighlightings> Tokens);
/// Return a line-by-line diff between two highlightings.
/// - if the tokens on a line are the same in both highlightings, this line is
/// omitted.
/// - if a line exists in New but not in Old, the tokens on this line are
/// emitted.
/// - if a line does not exist in New but exists in Old, an empty line is
/// emitted (to tell client to clear the previous highlightings on this line).
///
/// REQUIRED: Old and New are sorted.
std::vector<LineHighlightings>
diffHighlightings(ArrayRef<HighlightingToken> New,
ArrayRef<HighlightingToken> Old);
} // namespace clangd
} // namespace clang

View File

@ -513,7 +513,7 @@ private:
/// PreambleRequest.
mutable std::condition_variable PreambleCV;
/// Guards the callback that publishes results of AST-related computations
/// (diagnostics, highlightings) and file statuses.
/// (diagnostics) and file statuses.
std::mutex PublishMu;
// Used to prevent remove document + add document races that lead to
// out-of-order callbacks for publishing results of onMainAST callback.

View File

@ -151,11 +151,11 @@ public:
/// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
/// optimal performance.
///
/// When information about the file (diagnostics, syntax highlighting) is
/// When information about the file (e.g. diagnostics) is
/// published to clients, this should be wrapped in Publish, e.g.
/// void onMainAST(...) {
/// Highlights = computeHighlights();
/// Publish([&] { notifyHighlights(Path, Highlights); });
/// Diags = renderDiagnostics();
/// Publish([&] { notifyDiagnostics(Path, Diags); });
/// }
/// This guarantees that clients will see results in the correct sequence if
/// the file is concurrently closed and/or reopened. (The lambda passed to

View File

@ -1,145 +0,0 @@
# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"textDocument":{"semanticHighlightingCapabilities":{"semanticHighlighting":true}}},"trace":"off"}}
---
# CHECK: "id": 0,
# CHECK: "semanticHighlighting": {
# CHECK-NEXT: "scopes": [
# CHECK-NEXT: [
# CHECK-NEXT: "variable.other.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "variable.other.local.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "variable.parameter.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.function.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.function.method.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.function.method.static.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "variable.other.field.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "variable.other.field.static.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.class.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.enum.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "variable.other.enummember.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.typedef.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.dependent.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.other.dependent.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.namespace.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.template.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.type.concept.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "storage.type.primitive.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "entity.name.function.preprocessor.cpp"
# CHECK-NEXT: ],
# CHECK-NEXT: [
# CHECK-NEXT: "meta.disabled"
# CHECK-NEXT: ]
# CHECK-NEXT: ]
# CHECK-NEXT: },
---
{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.cpp","languageId":"cpp","text":"int x = 2;"}}}
# CHECK: "method": "textDocument/semanticHighlighting",
# CHECK-NEXT: "params": {
# CHECK-NEXT: "lines": [
# CHECK-NEXT: {
# CHECK-NEXT: "isInactive": false,
# CHECK-NEXT: "line": 0,
# CHECK-NEXT: "tokens": "AAAABAABAAA="
# CHECK-NEXT: }
# CHECK-NEXT: ],
# CHECK-NEXT: "textDocument": {
# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
# CHECK-NEXT: "version": 0
# CHECK-NEXT: }
# CHECK-NEXT: }
# CHECK-NEXT:}
---
{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo2.cpp","languageId":"cpp","text":"int x = 2;\nint y = 2;"}}}
# CHECK: "method": "textDocument/semanticHighlighting",
# CHECK-NEXT: "params": {
# CHECK-NEXT: "lines": [
# CHECK-NEXT: {
# CHECK-NEXT: "isInactive": false,
# CHECK-NEXT: "line": 0,
# CHECK-NEXT: "tokens": "AAAABAABAAA="
# CHECK-NEXT: }
# CHECK-NEXT: {
# CHECK-NEXT: "isInactive": false,
# CHECK-NEXT: "line": 1,
# CHECK-NEXT: "tokens": "AAAABAABAAA="
# CHECK-NEXT: }
# CHECK-NEXT: ],
# CHECK-NEXT: "textDocument": {
# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo2.cpp",
# CHECK-NEXT: "version": 0
# CHECK-NEXT: }
# CHECK-NEXT: }
# CHECK-NEXT:}
---
{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp"},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 0,"character": 10}},"rangeLength": 0,"text": "\nint y = 2;"}]}}
# CHECK: "method": "textDocument/semanticHighlighting",
# CHECK-NEXT: "params": {
# CHECK-NEXT: "lines": [
# CHECK-NEXT: {
# CHECK-NEXT: "isInactive": false,
# CHECK-NEXT: "line": 1,
# CHECK-NEXT: "tokens": "AAAABAABAAA="
# CHECK-NEXT: }
# CHECK-NEXT: ],
# CHECK-NEXT: "textDocument": {
# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
# CHECK-NEXT: "version": 1
# CHECK-NEXT: }
# CHECK-NEXT: }
# CHECK-NEXT:}
---
{"jsonrpc":"2.0","method":"textDocument/didChange","params":{"textDocument":{"uri":"test:///foo.cpp"},"contentChanges": [{"range":{"start": {"line": 0,"character": 10},"end": {"line": 1,"character": 10}},"rangeLength": 11,"text": ""}]}}
# CHECK: "method": "textDocument/semanticHighlighting",
# CHECK-NEXT: "params": {
# CHECK-NEXT: "lines": [
# CHECK-NEXT: {
# CHECK-NEXT: "isInactive": false,
# CHECK-NEXT: "line": 1,
# CHECK-NEXT: "tokens": ""
# CHECK-NEXT: }
# CHECK-NEXT: ],
# CHECK-NEXT: "textDocument": {
# CHECK-NEXT: "uri": "file://{{.*}}/clangd-test/foo.cpp",
# CHECK-NEXT: "version": 2
# CHECK-NEXT: }
# CHECK-NEXT: }
# CHECK-NEXT:}
---
{"jsonrpc":"2.0","id":3,"method":"shutdown"}
---
{"jsonrpc":"2.0","method":"exit"}

View File

@ -1,6 +1,6 @@
# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s -implicit-check-not=semanticHighlight
# Send capabilities for both Theia semanticHighlight & standard semanticTokens.
# clangd should not use/acknowledge the Theia protocol in this case.
# clangd should not use/acknowledge the Theia protocol.
{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"capabilities":{"textDocument":{
"semanticHighlightingCapabilities":{"semanticHighlighting":true},
"semanticTokens":{"dynamicRegistration":true}

View File

@ -29,51 +29,6 @@ namespace {
using testing::IsEmpty;
using testing::SizeIs;
MATCHER_P(LineNumber, L, "") { return arg.Line == L; }
MATCHER(EmptyHighlightings, "") { return arg.Tokens.empty(); }
std::vector<HighlightingToken>
makeHighlightingTokens(llvm::ArrayRef<Range> Ranges, HighlightingKind Kind) {
std::vector<HighlightingToken> Tokens(Ranges.size());
for (int I = 0, End = Ranges.size(); I < End; ++I) {
Tokens[I].R = Ranges[I];
Tokens[I].Kind = Kind;
}
return Tokens;
}
std::vector<HighlightingToken> getExpectedTokens(Annotations &Test) {
static const std::map<HighlightingKind, std::string> KindToString{
{HighlightingKind::Variable, "Variable"},
{HighlightingKind::LocalVariable, "LocalVariable"},
{HighlightingKind::Parameter, "Parameter"},
{HighlightingKind::Function, "Function"},
{HighlightingKind::Class, "Class"},
{HighlightingKind::Enum, "Enum"},
{HighlightingKind::Namespace, "Namespace"},
{HighlightingKind::EnumConstant, "EnumConstant"},
{HighlightingKind::Field, "Field"},
{HighlightingKind::StaticField, "StaticField"},
{HighlightingKind::Method, "Method"},
{HighlightingKind::StaticMethod, "StaticMethod"},
{HighlightingKind::Typedef, "Typedef"},
{HighlightingKind::Type, "Type"},
{HighlightingKind::Unknown, "Unknown"},
{HighlightingKind::TemplateParameter, "TemplateParameter"},
{HighlightingKind::Concept, "Concept"},
{HighlightingKind::Primitive, "Primitive"},
{HighlightingKind::Macro, "Macro"}};
std::vector<HighlightingToken> ExpectedTokens;
for (const auto &KindString : KindToString) {
std::vector<HighlightingToken> Toks = makeHighlightingTokens(
Test.ranges(KindString.second), KindString.first);
ExpectedTokens.insert(ExpectedTokens.end(), Toks.begin(), Toks.end());
}
llvm::sort(ExpectedTokens);
return ExpectedTokens;
}
/// Annotates the input code with provided semantic highlightings. Results look
/// something like:
/// class $Class[[X]] {
@ -134,39 +89,6 @@ void checkHighlightings(llvm::StringRef Code,
EXPECT_EQ(Code, annotate(Test.code(), Actual));
}
// Any annotations in OldCode and NewCode are converted into their corresponding
// HighlightingToken. The tokens are diffed against each other. Any lines where
// the tokens should diff must be marked with a ^ somewhere on that line in
// NewCode. If there are diffs that aren't marked with ^ the test fails. The
// test also fails if there are lines marked with ^ that don't differ.
void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
Annotations OldTest(OldCode);
Annotations NewTest(NewCode);
std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
llvm::DenseMap<int, std::vector<HighlightingToken>> ExpectedLines;
for (const Position &Point : NewTest.points()) {
ExpectedLines[Point.line]; // Default initialize to an empty line. Tokens
// are inserted on these lines later.
}
std::vector<LineHighlightings> ExpectedLinePairHighlighting;
for (const HighlightingToken &Token : NewTokens) {
auto It = ExpectedLines.find(Token.R.start.line);
if (It != ExpectedLines.end())
It->second.push_back(Token);
}
for (auto &LineTokens : ExpectedLines)
ExpectedLinePairHighlighting.push_back(
{LineTokens.first, LineTokens.second, /*IsInactive = */ false});
std::vector<LineHighlightings> ActualDiffed =
diffHighlightings(NewTokens, OldTokens);
EXPECT_THAT(ActualDiffed,
testing::UnorderedElementsAreArray(ExpectedLinePairHighlighting))
<< OldCode;
}
constexpr static uint32_t ScopeModifierMask =
1 << unsigned(HighlightingModifier::FunctionScope) |
1 << unsigned(HighlightingModifier::ClassScope) |
@ -809,30 +731,6 @@ TEST(SemanticHighlighting, ScopeModifiers) {
checkHighlightings(Test, {}, ScopeModifierMask);
}
TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {
class HighlightingsCounter : public ClangdServer::Callbacks {
public:
std::atomic<int> Count = {0};
void onHighlightingsReady(
PathRef File, llvm::StringRef Version,
std::vector<HighlightingToken> Highlightings) override {
++Count;
}
};
auto FooCpp = testPath("foo.cpp");
MockFS FS;
FS.Files[FooCpp] = "";
MockCompilationDatabase MCD;
HighlightingsCounter Counter;
ClangdServer Server(MCD, FS, ClangdServer::optsForTest(), &Counter);
Server.addDocument(FooCpp, "int a;");
ASSERT_TRUE(Server.blockUntilIdleForTest()) << "Waiting for server";
ASSERT_EQ(Counter.Count, 1);
}
// Ranges are highlighted as variables, unless highlighted as $Function etc.
std::vector<HighlightingToken> tokens(llvm::StringRef MarkedText) {
Annotations A(MarkedText);
@ -912,149 +810,6 @@ TEST(SemanticHighlighting, diffSemanticTokens) {
EXPECT_EQ(3u, Diff.front().tokens[2].length);
}
TEST(SemanticHighlighting, toTheiaSemanticHighlightingInformation) {
auto CreatePosition = [](int Line, int Character) -> Position {
Position Pos;
Pos.line = Line;
Pos.character = Character;
return Pos;
};
std::vector<LineHighlightings> Tokens{
{3,
{{HighlightingKind::Variable, 0,
Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
{HighlightingKind::Function, 0,
Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
/* IsInactive = */ false},
{1,
{{HighlightingKind::Variable, 0,
Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
/* IsInactive = */ true}};
std::vector<TheiaSemanticHighlightingInformation> ActualResults =
toTheiaSemanticHighlightingInformation(Tokens);
std::vector<TheiaSemanticHighlightingInformation> ExpectedResults = {
{3, "AAAACAAEAAAAAAAEAAMAAw=="}, {1, "AAAAAQAEAAA="}};
EXPECT_EQ(ActualResults, ExpectedResults);
}
TEST(SemanticHighlighting, HighlightingDiffer) {
struct {
llvm::StringRef OldCode;
llvm::StringRef NewCode;
} TestCases[]{{
R"(
$Variable[[A]]
$Class[[B]]
$Function[[C]]
)",
R"(
$Variable[[A]]
$Class[[D]]
$Function[[C]]
)"},
{
R"(
$Class[[C]]
$Field[[F]]
$Variable[[V]]
$Class[[C]] $Variable[[V]] $Field[[F]]
)",
R"(
$Class[[C]]
$Field[[F]]
^$Function[[F]]
$Class[[C]] $Variable[[V]] $Field[[F]]
)"},
{
R"(
$Class[[A]]
$Variable[[A]]
)",
R"(
^
^$Class[[A]]
^$Variable[[A]]
)"},
{
R"(
$Class[[C]]
$Field[[F]]
$Variable[[V]]
$Class[[C]] $Variable[[V]] $Field[[F]]
)",
R"(
$Class[[C]]
^
^
$Class[[C]] $Variable[[V]] $Field[[F]]
)"},
{
R"(
$Class[[A]]
$Variable[[A]]
$Variable[[A]]
)",
R"(
$Class[[A]]
^$Variable[[AA]]
$Variable[[A]]
)"},
{
R"(
$Class[[A]]
$Variable[[A]]
)",
R"(
$Class[[A]]
$Variable[[A]]
^$Class[[A]]
^$Variable[[A]]
)"},
{
R"(
$Variable[[A]]
$Variable[[A]]
$Variable[[A]]
)",
R"(
^$Class[[A]]
^$Class[[A]]
^$Class[[A]]
)"}};
for (const auto &Test : TestCases)
checkDiffedHighlights(Test.OldCode, Test.NewCode);
}
TEST(SemanticHighlighting, DiffBeyondTheEndOfFile) {
llvm::StringRef OldCode =
R"(
$Class[[A]]
$Variable[[A]]
$Class[[A]]
$Variable[[A]]
)";
llvm::StringRef NewCode =
R"(
$Class[[A]] // line 1
$Variable[[A]] // line 2
)";
Annotations OldTest(OldCode);
Annotations NewTest(NewCode);
std::vector<HighlightingToken> OldTokens = getExpectedTokens(OldTest);
std::vector<HighlightingToken> NewTokens = getExpectedTokens(NewTest);
auto ActualDiff = diffHighlightings(NewTokens, OldTokens);
EXPECT_THAT(ActualDiff,
testing::UnorderedElementsAre(
testing::AllOf(LineNumber(3), EmptyHighlightings()),
testing::AllOf(LineNumber(4), EmptyHighlightings())));
}
} // namespace
} // namespace clangd
} // namespace clang