1 module workspaced.com.snippets; 2 3 import dparse.lexer; 4 import dparse.parser; 5 import dparse.rollback_allocator; 6 7 import workspaced.api; 8 import workspaced.com.dfmt : DfmtComponent; 9 import workspaced.com.snippets.generator; 10 import workspaced.dparseext; 11 12 import std.algorithm; 13 import std.array; 14 import std.ascii; 15 import std.conv; 16 import std.json; 17 import std.string; 18 import std.typecons; 19 20 public import workspaced.com.snippets.plain; 21 public import workspaced.com.snippets.smart; 22 public import workspaced.com.snippets.dependencies; 23 24 /// Component for auto completing snippets with context information and formatting these snippets with dfmt. 25 @component("snippets") 26 class SnippetsComponent : ComponentWrapper 27 { 28 mixin DefaultComponentWrapper; 29 30 static PlainSnippetProvider plainSnippets; 31 static SmartSnippetProvider smartSnippets; 32 static DependencyBasedSnippetProvider dependencySnippets; 33 34 protected SnippetProvider[] providers; 35 36 protected void load() 37 { 38 if (!plainSnippets) 39 plainSnippets = new PlainSnippetProvider(); 40 if (!smartSnippets) 41 smartSnippets = new SmartSnippetProvider(); 42 if (!dependencySnippets) 43 dependencySnippets = new DependencyBasedSnippetProvider(); 44 45 config.stringBehavior = StringBehavior.source; 46 providers.reserve(16); 47 providers ~= plainSnippets; 48 providers ~= smartSnippets; 49 providers ~= dependencySnippets; 50 } 51 52 /** 53 * Params: 54 * file = Filename to resolve dependencies relatively from. 55 * code = Code to complete snippet in. 56 * position = Byte offset of where to find scope in. 57 * 58 * Returns: a `SnippetInfo` object for all snippet information. 59 * 60 * `.loopScope` is set if a loop can be inserted at this position, Optionally 61 * with information about close ranges. Contains `SnippetLoopScope.init` if 62 * this is not a location where a loop can be inserted. 63 */ 64 SnippetInfo determineSnippetInfo(scope const(char)[] file, scope const(char)[] code, int position) 65 { 66 // each variable is 1 67 // maybe more expensive lookups with DCD in the future 68 enum LoopVariableAnalyzeMaxCost = 90; 69 70 scope tokens = getTokensForParser(cast(ubyte[]) code, config, &workspaced.stringCache); 71 auto loc = tokens.tokenIndexAtByteIndex(position); 72 73 // first check if at end of identifier, move current location to that 74 // identifier. 75 if (loc > 0 76 && loc < tokens.length 77 && tokens[loc - 1].isLikeIdentifier 78 && tokens[loc - 1].index <= position 79 && tokens[loc - 1].index + tokens[loc - 1].textLength >= position) 80 loc--; 81 // also determine info from before start of identifier (so you can start 82 // typing something and it still finds a snippet scope) 83 // > double decrement when at end of identifier, start of other token! 84 if (loc > 0 85 && loc < tokens.length 86 && tokens[loc].isLikeIdentifier 87 && tokens[loc].index <= position 88 && tokens[loc].index + tokens[loc].textLength >= position) 89 loc--; 90 91 // nudge in next token if position is after this token 92 if (loc < tokens.length && tokens[loc].isLikeIdentifier 93 && position > tokens[loc].index + tokens[loc].textLength) 94 { 95 // cursor must not be glued to the end of identifiers 96 loc++; 97 } 98 else if (loc < tokens.length && !tokens[loc].isLikeIdentifier 99 && position >= tokens[loc].index + tokens[loc].textLength) 100 { 101 // but next token if end of non-identifiers (eg `""`, `;`, `.`, `(`) 102 loc++; 103 } 104 105 int contextIndex; 106 int checkLocation = position; 107 if (loc >= 0 && loc < tokens.length) 108 { 109 contextIndex = cast(int) tokens[loc].index; 110 if (tokens[loc].index < position) 111 checkLocation = contextIndex; 112 } 113 114 if (loc == 0 || loc == tokens.length) 115 return SnippetInfo(contextIndex, [SnippetLevel.global]); 116 117 auto leading = tokens[0 .. loc]; 118 119 if (leading.length) 120 { 121 auto last = leading[$ - 1]; 122 switch (last.type) 123 { 124 case tok!".": 125 case tok!")": 126 case tok!"characterLiteral": 127 case tok!"dstringLiteral": 128 case tok!"wstringLiteral": 129 case tok!"stringLiteral": 130 // no snippets immediately after these tokens (needs some other 131 // token inbetween) 132 return SnippetInfo(contextIndex, [SnippetLevel.other]); 133 case tok!"(": 134 // current token is something like `)`, check for previous 135 // tokens like `__traits` `(` 136 if (leading.length >= 2) 137 { 138 switch (leading[$ - 2].type) 139 { 140 case tok!"__traits": 141 case tok!"version": 142 case tok!"debug": 143 return SnippetInfo(contextIndex, [SnippetLevel.other]); 144 default: break; 145 } 146 } 147 break; 148 case tok!"__traits": 149 case tok!"version": 150 case tok!"debug": 151 return SnippetInfo(contextIndex, [SnippetLevel.other]); 152 case tok!"typeof": 153 case tok!"if": 154 case tok!"while": 155 case tok!"for": 156 case tok!"foreach": 157 case tok!"foreach_reverse": 158 case tok!"switch": 159 case tok!"with": 160 case tok!"catch": 161 // immediately after these tokens, missing opening parentheses 162 if (tokens[loc].type != tok!"(") 163 return SnippetInfo(contextIndex, [SnippetLevel.other]); 164 break; 165 default: 166 break; 167 } 168 } 169 170 auto current = tokens[loc]; 171 switch (current.type) 172 { 173 case tok!"comment": 174 size_t len = max(0, cast(ptrdiff_t)position 175 - cast(ptrdiff_t)current.index); 176 // TODO: currently never called because we would either need to 177 // use the DLexer struct as parser immediately or wait until 178 // libdparse >=0.15.0 which contains trivia, where this switch 179 // needs to be modified to check the exact trivia token instead 180 // of the associated token with it. 181 if (current.text[0 .. len].startsWith("///", "/++", "/**")) 182 return SnippetInfo(contextIndex, [SnippetLevel.docComment]); 183 else if (len >= 2) 184 return SnippetInfo(contextIndex, [SnippetLevel.comment]); 185 else 186 break; 187 case tok!"characterLiteral": 188 case tok!"dstringLiteral": 189 case tok!"wstringLiteral": 190 case tok!"stringLiteral": 191 if (position <= current.index) 192 break; 193 194 auto textSoFar = current.text[1 .. position - current.index]; 195 // no string complete if we are immediately after escape or 196 // quote character 197 // TODO: properly check if this is an unescaped escape 198 if (textSoFar.endsWith('\\', current.text[0])) 199 return SnippetInfo(contextIndex, [SnippetLevel.strings, SnippetLevel.other]); 200 else 201 return SnippetInfo(contextIndex, [SnippetLevel.strings]); 202 default: 203 break; 204 } 205 206 foreach_reverse (t; leading) 207 { 208 if (t.type == tok!";") 209 break; 210 211 // test for tokens semicolon closed statements where we should abort to avoid incomplete syntax 212 if (t.type.among!(tok!"import", tok!"module")) 213 { 214 return SnippetInfo(contextIndex, [SnippetLevel.global, SnippetLevel.other]); 215 } 216 else if (t.type.among!(tok!"=", tok!"+", tok!"-", tok!"*", tok!"/", 217 tok!"%", tok!"^^", tok!"&", tok!"|", tok!"^", tok!"<<", 218 tok!">>", tok!">>>", tok!"~", tok!"in")) 219 { 220 return SnippetInfo(contextIndex, [SnippetLevel.global, SnippetLevel.value]); 221 } 222 } 223 224 RollbackAllocator rba; 225 scope parsed = parseModule(tokens, cast(string) file, &rba); 226 227 //trace("determineSnippetInfo at ", contextIndex); 228 229 scope gen = new SnippetInfoGenerator(checkLocation); 230 gen.value.contextTokenIndex = contextIndex; 231 gen.variableStack.reserve(64); 232 gen.visit(parsed); 233 234 gen.value.loopScope.supported = gen.value.level == SnippetLevel.method; 235 if (gen.value.loopScope.supported) 236 { 237 int cost = 0; 238 foreach_reverse (v; gen.variableStack) 239 { 240 if (fillLoopScopeInfo(gen.value.loopScope, v)) 241 break; 242 if (++cost > LoopVariableAnalyzeMaxCost) 243 break; 244 } 245 } 246 247 if (gen.lastStatement) 248 { 249 import dparse.ast; 250 251 LastStatementInfo info; 252 auto nodeType = gen.lastStatement.findDeepestNonBlockNode; 253 if (gen.lastStatement.tokens.length) 254 info.location = cast(int) nodeType.tokens[0].index; 255 info.type = typeid(nodeType).name; 256 auto lastDot = info.type.lastIndexOf('.'); 257 if (lastDot != -1) 258 info.type = info.type[lastDot + 1 .. $]; 259 if (auto ifStmt = cast(IfStatement)nodeType) 260 { 261 auto elseStmt = getIfElse(ifStmt); 262 if (cast(IfStatement)elseStmt) 263 info.ifHasElse = false; 264 else 265 info.ifHasElse = elseStmt !is null; 266 } 267 else if (auto ifStmt = cast(ConditionalDeclaration)nodeType) 268 info.ifHasElse = ifStmt.hasElse; 269 // if (auto ifStmt = cast(ConditionalStatement)nodeType) 270 // info.ifHasElse = !!getIfElse(ifStmt); 271 272 gen.value.lastStatement = info; 273 } 274 275 return gen.value; 276 } 277 278 Future!SnippetList getSnippets(scope const(char)[] file, scope const(char)[] code, int position) 279 { 280 mixin(gthreadsAsyncProxy!`getSnippetsBlocking(file, code, position)`); 281 } 282 283 SnippetList getSnippetsBlocking(scope const(char)[] file, scope const(char)[] code, int position) 284 { 285 auto futures = collectSnippets(file, code, position); 286 287 auto ret = appender!(Snippet[]); 288 foreach (fut; futures[1]) 289 ret.put(fut.getBlocking()); 290 return SnippetList(futures[0], ret.data); 291 } 292 293 SnippetList getSnippetsYield(scope const(char)[] file, scope const(char)[] code, int position) 294 { 295 auto futures = collectSnippets(file, code, position); 296 297 auto ret = appender!(Snippet[]); 298 foreach (fut; futures[1]) 299 ret.put(fut.getYield()); 300 return SnippetList(futures[0], ret.data); 301 } 302 303 Future!Snippet resolveSnippet(scope const(char)[] file, scope const(char)[] code, 304 int position, Snippet snippet) 305 { 306 foreach (provider; providers) 307 { 308 if (typeid(provider).name == snippet.providerId) 309 { 310 const info = determineSnippetInfo(file, code, position); 311 return provider.resolveSnippet(instance, file, code, position, info, snippet); 312 } 313 } 314 315 return typeof(return).fromResult(snippet); 316 } 317 318 Future!string format(scope const(char)[] snippet, string[] arguments = [], 319 SnippetLevel level = SnippetLevel.global) 320 { 321 mixin(gthreadsAsyncProxy!`formatSync(snippet, arguments, level)`); 322 } 323 324 /// Will format the code passed in synchronously using dfmt. Might take a short moment on larger documents. 325 /// Returns: the formatted code as string or unchanged if dfmt is not active 326 string formatSync(scope const(char)[] snippet, string[] arguments = [], 327 SnippetLevel level = SnippetLevel.global) 328 { 329 if (!has!DfmtComponent) 330 return snippet.idup; 331 332 auto dfmt = get!DfmtComponent; 333 334 auto tmp = appender!string; 335 336 final switch (level) 337 { 338 case SnippetLevel.global: 339 case SnippetLevel.other: 340 case SnippetLevel.comment: 341 case SnippetLevel.docComment: 342 case SnippetLevel.strings: 343 case SnippetLevel.mixinTemplate: 344 break; 345 case SnippetLevel.type: 346 tmp.put("struct FORMAT_HELPER {\n"); 347 break; 348 case SnippetLevel.method: 349 tmp.put("void FORMAT_HELPER() {\n"); 350 break; 351 case SnippetLevel.value: 352 tmp.put("int FORMAT_HELPER() = "); 353 break; 354 } 355 356 scope const(char)[][string] tokens; 357 358 ptrdiff_t dollar, last; 359 while (true) 360 { 361 dollar = snippet.indexOfAny(`$\`, last); 362 if (dollar == -1) 363 { 364 tmp ~= snippet[last .. $]; 365 break; 366 } 367 368 tmp ~= snippet[last .. dollar]; 369 last = dollar + 1; 370 if (last >= snippet.length) 371 break; 372 if (snippet[dollar] == '\\') 373 { 374 tmp ~= snippet[dollar + 1]; 375 last = dollar + 2; 376 } 377 else 378 { 379 string key = "__WspD_Snp_" ~ dollar.to!string ~ "_"; 380 const(char)[] str; 381 382 bool startOfBlock = snippet[0 .. dollar].stripRight.endsWith("{"); 383 bool endOfBlock; 384 385 bool makeWrappingIfMayBeDelegate() 386 { 387 endOfBlock = snippet[last .. $].stripLeft.startsWith("}"); 388 if (startOfBlock && endOfBlock) 389 { 390 // make extra long to make dfmt definitely wrap this (in case this is a delegate, otherwise this doesn't hurt either) 391 key.reserve(key.length + 200); 392 foreach (i; 0 .. 200) 393 key ~= "_"; 394 return true; 395 } 396 else 397 return false; 398 } 399 400 if (snippet[dollar + 1] == '{') 401 { 402 ptrdiff_t i = dollar + 2; 403 int depth = 1; 404 while (true) 405 { 406 auto next = snippet.indexOfAny(`\{}`, i); 407 if (next == -1) 408 { 409 i = snippet.length; 410 break; 411 } 412 413 if (snippet[next] == '\\') 414 i = next + 2; 415 else 416 { 417 if (snippet[next] == '{') 418 depth++; 419 else if (snippet[next] == '}') 420 depth--; 421 else 422 assert(false); 423 424 i = next + 1; 425 } 426 427 if (depth == 0) 428 break; 429 } 430 str = snippet[dollar .. i]; 431 last = i; 432 433 const wrapped = makeWrappingIfMayBeDelegate(); 434 435 const placeholderMightBeIdentifier = str.length > 5 436 || snippet[last .. $].stripLeft.startsWith(";", ".", "{", "(", "["); 437 438 if (wrapped || placeholderMightBeIdentifier) 439 { 440 // let's insert some token in here instead of a comment because there is probably some default content 441 // if there is a semicolon at the end we probably need to insert a semicolon here too 442 // if this is a comment placeholder let's insert a semicolon to make dfmt wrap 443 if (str[0 .. $ - 1].endsWith(';') || str[0 .. $ - 1].canFind("//")) 444 key ~= ';'; 445 } 446 else if (level != SnippetLevel.value) 447 { 448 // empty default, put in comment 449 key = "/+++" ~ key ~ "+++/"; 450 } 451 } 452 else 453 { 454 size_t end = dollar + 1; 455 456 if (snippet[dollar + 1].isDigit) 457 { 458 while (end < snippet.length && snippet[end].isDigit) 459 end++; 460 } 461 else 462 { 463 while (end < snippet.length && (snippet[end].isAlphaNum || snippet[end] == '_')) 464 end++; 465 } 466 467 str = snippet[dollar .. end]; 468 last = end; 469 470 makeWrappingIfMayBeDelegate(); 471 472 const placeholderMightBeIdentifier = snippet[last .. $].stripLeft.startsWith(";", 473 ".", "{", "(", "["); 474 475 if (placeholderMightBeIdentifier) 476 { 477 // keep value thing as simple identifier as we don't have any placeholder text 478 } 479 else if (level != SnippetLevel.value) 480 { 481 // primitive placeholder as comment 482 key = "/+++" ~ key ~ "+++/"; 483 } 484 } 485 486 tokens[key] = str; 487 tmp ~= key; 488 } 489 } 490 491 final switch (level) 492 { 493 case SnippetLevel.global: 494 case SnippetLevel.other: 495 case SnippetLevel.comment: 496 case SnippetLevel.docComment: 497 case SnippetLevel.strings: 498 case SnippetLevel.mixinTemplate: 499 break; 500 case SnippetLevel.type: 501 case SnippetLevel.method: 502 tmp.put("}"); 503 break; 504 case SnippetLevel.value: 505 tmp.put(";"); 506 break; 507 } 508 509 auto res = dfmt.formatSync(tmp.data, arguments); 510 511 string chompStr; 512 char del; 513 final switch (level) 514 { 515 case SnippetLevel.global: 516 case SnippetLevel.other: 517 case SnippetLevel.comment: 518 case SnippetLevel.docComment: 519 case SnippetLevel.strings: 520 case SnippetLevel.mixinTemplate: 521 break; 522 case SnippetLevel.type: 523 case SnippetLevel.method: 524 chompStr = "}"; 525 del = '{'; 526 break; 527 case SnippetLevel.value: 528 chompStr = ";"; 529 del = '='; 530 break; 531 } 532 533 if (chompStr.length) 534 res = res.stripRight.chomp(chompStr); 535 536 if (del != char.init) 537 { 538 auto start = res.indexOf(del); 539 if (start != -1) 540 { 541 res = res[start + 1 .. $]; 542 543 while (true) 544 { 545 // delete empty lines before first line 546 auto nl = res.indexOf('\n'); 547 if (nl != -1 && res[0 .. nl].all!isWhite) 548 res = res[nl + 1 .. $]; 549 else 550 break; 551 } 552 553 auto indent = res[0 .. res.length - res.stripLeft.length]; 554 if (indent.length) 555 { 556 // remove indentation of whole block 557 assert(indent.all!isWhite); 558 res = res.splitLines.map!(a => a.startsWith(indent) 559 ? a[indent.length .. $] : a.stripRight).join("\n"); 560 } 561 } 562 } 563 564 foreach (key, value; tokens) 565 { 566 // TODO: replacing using aho-corasick would be far more efficient but there is nothing like that in phobos 567 res = res.replace(key, value); 568 } 569 570 if (res.endsWith("\r\n") && !snippet.endsWith('\n')) 571 res.length -= 2; 572 else if (res.endsWith('\n') && !snippet.endsWith('\n')) 573 res.length--; 574 575 if (res.endsWith(";\n\n$0")) 576 res = res[0 .. $ - "\n$0".length] ~ "$0"; 577 else if (res.endsWith(";\r\n\r\n$0")) 578 res = res[0 .. $ - "\r\n$0".length] ~ "$0"; 579 580 return res; 581 } 582 583 /// Adds snippets which complete conditionally based on dub dependencies being present. 584 /// This function affects the global configuration of all instances. 585 /// Params: 586 /// requiredDependencies = The dependencies which must be present in order for this snippet to show up. 587 /// snippet = The snippet to suggest when the required dependencies are matched. 588 void addDependencySnippet(string[] requiredDependencies, PlainSnippet snippet) 589 { 590 // maybe application global change isn't such a good idea? Current config system seems too inefficient for this. 591 dependencySnippets.addSnippet(requiredDependencies, snippet); 592 } 593 594 private: 595 Tuple!(SnippetInfo, Future!(Snippet[])[]) collectSnippets(scope const(char)[] file, 596 scope const(char)[] code, int position) 597 { 598 const inst = instance; 599 auto info = determineSnippetInfo(file, code, position); 600 auto futures = appender!(Future!(Snippet[])[]); 601 foreach (provider; providers) 602 futures.put(provider.provideSnippets(inst, file, code, position, info)); 603 return tuple(info, futures.data); 604 } 605 606 LexerConfig config; 607 } 608 609 /// 610 enum SnippetLevel 611 { 612 /// Outside of functions or types, possibly inside templates 613 global, 614 /// Inside interfaces, classes, structs or unions 615 type, 616 /// Inside method body 617 method, 618 /// inside a variable value, argument call, default value or similar 619 value, 620 /// Other scope types (for example outside of braces but after a function definition or some other invalid syntax place) 621 other, 622 /// Inside a string literal. 623 strings, 624 /// Inside a normal comment 625 comment, 626 /// Inside a documentation comment 627 docComment, 628 /// Inside explicitly declared mixin templates 629 mixinTemplate, 630 } 631 632 /// 633 struct SnippetLoopScope 634 { 635 /// true if an loop expression can be inserted at this point 636 bool supported; 637 /// true if we know we are iterating over a string (possibly needing unicode decoding) or false otherwise 638 bool stringIterator; 639 /// Explicit type to use when iterating or null if none is known 640 string type; 641 /// Best variable to iterate over or null if none was found 642 string iterator; 643 /// Number of keys to iterate over 644 int numItems = 1; 645 } 646 647 /// 648 struct SnippetInfo 649 { 650 /// Index in code which token was used to determine this snippet info. 651 int contextTokenIndex; 652 /// Levels this snippet location has gone through, latest one being the last 653 SnippetLevel[] stack = [SnippetLevel.global]; 654 /// Information about snippets using loop context 655 SnippetLoopScope loopScope; 656 /// Information about the last parsable statement before the cursor. May be 657 /// `LastStatementInfo.init` at start of function or block. 658 LastStatementInfo lastStatement; 659 660 /// Current snippet scope level of the location 661 SnippetLevel level() const @property 662 { 663 return stack.length ? stack[$ - 1] : SnippetLevel.other; 664 } 665 } 666 667 struct LastStatementInfo 668 { 669 /// The libdparse class name (typeid) of the last parsable statement before 670 /// the cursor, stripped of module name. 671 string type; 672 /// If type is set, this is the start location in bytes where 673 /// the first token was. 674 int location; 675 /// True if the type is (`IfStatement`, `ConditionalDeclaration` or 676 /// `ConditionalStatement`) and has a final `else` block defined. 677 bool ifHasElse; 678 } 679 680 /// A list of snippets resolved at a given position. 681 struct SnippetList 682 { 683 /// The info where this snippet is completing at. 684 SnippetInfo info; 685 /// The list of snippets that got returned. 686 Snippet[] snippets; 687 } 688 689 /// 690 interface SnippetProvider 691 { 692 Future!(Snippet[]) provideSnippets(scope const WorkspaceD.Instance instance, 693 scope const(char)[] file, scope const(char)[] code, int position, const SnippetInfo info); 694 695 Future!Snippet resolveSnippet(scope const WorkspaceD.Instance instance, 696 scope const(char)[] file, scope const(char)[] code, int position, 697 const SnippetInfo info, Snippet snippet); 698 } 699 700 /// Snippet to insert 701 struct Snippet 702 { 703 /// Internal ID for resolving this snippet 704 string id, providerId; 705 /// User-defined data for helping resolving this snippet 706 JSONValue data; 707 /// Label for this snippet 708 string title; 709 /// Shortcut to type for this snippet 710 string shortcut; 711 /// Markdown documentation for this snippet 712 string documentation; 713 /// Plain text to insert assuming global level indentation. 714 string plain; 715 /// Text with interactive snippet locations to insert assuming global indentation. 716 string snippet; 717 /// true if this snippet can be used as-is 718 bool resolved; 719 /// true if this snippet shouldn't be formatted. 720 bool unformatted; 721 }