1 module workspaced.com.snippets; 2 3 import dparse.lexer; 4 import dparse.parser; 5 import dparse.rollback_allocator; 6 7 import workspaced.api; 8 import workspaced.com.dfmt : DfmtComponent; 9 import workspaced.com.snippets.generator; 10 import workspaced.dparseext; 11 12 import std.algorithm; 13 import std.array; 14 import std.ascii; 15 import std.conv; 16 import std.json; 17 import std..string; 18 import std.typecons; 19 20 public import workspaced.com.snippets.plain; 21 public import workspaced.com.snippets.smart; 22 public import workspaced.com.snippets.dependencies; 23 24 /// Component for auto completing snippets with context information and formatting these snippets with dfmt. 25 @component("snippets") 26 class SnippetsComponent : ComponentWrapper 27 { 28 mixin DefaultComponentWrapper; 29 30 static PlainSnippetProvider plainSnippets; 31 static SmartSnippetProvider smartSnippets; 32 static DependencyBasedSnippetProvider dependencySnippets; 33 34 protected SnippetProvider[] providers; 35 36 protected void load() 37 { 38 if (!plainSnippets) 39 plainSnippets = new PlainSnippetProvider(); 40 if (!smartSnippets) 41 smartSnippets = new SmartSnippetProvider(); 42 if (!dependencySnippets) 43 dependencySnippets = new DependencyBasedSnippetProvider(); 44 45 config.stringBehavior = StringBehavior.source; 46 providers.reserve(16); 47 providers ~= plainSnippets; 48 providers ~= smartSnippets; 49 providers ~= dependencySnippets; 50 } 51 52 /** 53 * Params: 54 * file = Filename to resolve dependencies relatively from. 55 * code = Code to complete snippet in. 56 * position = Byte offset of where to find scope in. 57 * 58 * Returns: a `SnippetInfo` object for all snippet information. 59 * 60 * `.loopScope` is set if a loop can be inserted at this position, Optionally 61 * with information about close ranges. Contains `SnippetLoopScope.init` if 62 * this is not a location where a loop can be inserted. 63 */ 64 SnippetInfo determineSnippetInfo(scope const(char)[] file, scope const(char)[] code, int position) 65 { 66 // each variable is 1 67 // maybe more expensive lookups with DCD in the future 68 enum LoopVariableAnalyzeMaxCost = 90; 69 70 scope tokens = getTokensForParser(cast(ubyte[]) code, config, &workspaced.stringCache); 71 auto loc = tokens.tokenIndexAtByteIndex(position); 72 73 // nudge in next token if position is not exactly on the start of it 74 if (loc < tokens.length && tokens[loc].index < position) 75 loc++; 76 77 if (loc == 0 || loc == tokens.length) 78 return SnippetInfo([SnippetLevel.global]); 79 80 auto leading = tokens[0 .. loc]; 81 82 if (leading.length) 83 { 84 auto last = leading[$ - 1]; 85 switch (last.type) 86 { 87 case tok!"comment": 88 size_t len = max(0, cast(ptrdiff_t)position 89 - cast(ptrdiff_t)last.index); 90 // TODO: currently never called because we would either need to 91 // use the DLexer struct as parser immediately or wait until 92 // libdparse >=0.15.0 which contains trivia, where this switch 93 // needs to be modified to check the exact trivia token instead 94 // of the associated token with it. 95 if (last.text[0 .. len].startsWith("///", "/++", "/**")) 96 return SnippetInfo([SnippetLevel.docComment]); 97 else if (len >= 2) 98 return SnippetInfo([SnippetLevel.comment]); 99 else 100 break; 101 case tok!"dstringLiteral": 102 case tok!"wstringLiteral": 103 case tok!"stringLiteral": 104 if (position <= last.index) 105 break; 106 107 auto textSoFar = last.text[1 .. position - last.index]; 108 // no string complete if we are immediately after escape or 109 // quote character 110 // TODO: properly check if this is an unescaped escape 111 if (textSoFar.endsWith('\\', last.text[0])) 112 return SnippetInfo([SnippetLevel.strings, SnippetLevel.other]); 113 else 114 return SnippetInfo([SnippetLevel.strings]); 115 case tok!"(": 116 if (leading.length >= 2) 117 { 118 auto beforeLast = leading[$ - 2]; 119 if (beforeLast.type.among(tok!"__traits", tok!"version", tok!"debug")) 120 return SnippetInfo([SnippetLevel.other]); 121 } 122 break; 123 default: 124 break; 125 } 126 } 127 128 foreach_reverse (t; leading) 129 { 130 if (t.type == tok!";") 131 break; 132 133 // test for tokens semicolon closed statements where we should abort to avoid incomplete syntax 134 if (t.type.among!(tok!"import", tok!"module")) 135 { 136 return SnippetInfo([SnippetLevel.global, SnippetLevel.other]); 137 } 138 else if (t.type.among!(tok!"=", tok!"+", tok!"-", tok!"*", tok!"/", 139 tok!"%", tok!"^^", tok!"&", tok!"|", tok!"^", tok!"<<", 140 tok!">>", tok!">>>", tok!"~", tok!"in")) 141 { 142 return SnippetInfo([SnippetLevel.global, SnippetLevel.value]); 143 } 144 } 145 146 RollbackAllocator rba; 147 scope parsed = parseModule(tokens, cast(string) file, &rba); 148 149 //trace("determineSnippetInfo at ", position); 150 151 scope gen = new SnippetInfoGenerator(position); 152 gen.variableStack.reserve(64); 153 gen.visit(parsed); 154 155 gen.value.loopScope.supported = gen.value.level == SnippetLevel.method; 156 if (gen.value.loopScope.supported) 157 { 158 int cost = 0; 159 foreach_reverse (v; gen.variableStack) 160 { 161 if (fillLoopScopeInfo(gen.value.loopScope, v)) 162 break; 163 if (++cost > LoopVariableAnalyzeMaxCost) 164 break; 165 } 166 } 167 168 return gen.value; 169 } 170 171 Future!SnippetList getSnippets(scope const(char)[] file, scope const(char)[] code, int position) 172 { 173 mixin(gthreadsAsyncProxy!`getSnippetsBlocking(file, code, position)`); 174 } 175 176 SnippetList getSnippetsBlocking(scope const(char)[] file, scope const(char)[] code, int position) 177 { 178 auto futures = collectSnippets(file, code, position); 179 180 auto ret = appender!(Snippet[]); 181 foreach (fut; futures[1]) 182 ret.put(fut.getBlocking()); 183 return SnippetList(futures[0], ret.data); 184 } 185 186 SnippetList getSnippetsYield(scope const(char)[] file, scope const(char)[] code, int position) 187 { 188 auto futures = collectSnippets(file, code, position); 189 190 auto ret = appender!(Snippet[]); 191 foreach (fut; futures[1]) 192 ret.put(fut.getYield()); 193 return SnippetList(futures[0], ret.data); 194 } 195 196 Future!Snippet resolveSnippet(scope const(char)[] file, scope const(char)[] code, 197 int position, Snippet snippet) 198 { 199 foreach (provider; providers) 200 { 201 if (typeid(provider).name == snippet.providerId) 202 { 203 const info = determineSnippetInfo(file, code, position); 204 return provider.resolveSnippet(instance, file, code, position, info, snippet); 205 } 206 } 207 208 return typeof(return).fromResult(snippet); 209 } 210 211 Future!string format(scope const(char)[] snippet, string[] arguments = [], 212 SnippetLevel level = SnippetLevel.global) 213 { 214 mixin(gthreadsAsyncProxy!`formatSync(snippet, arguments, level)`); 215 } 216 217 /// Will format the code passed in synchronously using dfmt. Might take a short moment on larger documents. 218 /// Returns: the formatted code as string or unchanged if dfmt is not active 219 string formatSync(scope const(char)[] snippet, string[] arguments = [], 220 SnippetLevel level = SnippetLevel.global) 221 { 222 if (!has!DfmtComponent) 223 return snippet.idup; 224 225 auto dfmt = get!DfmtComponent; 226 227 auto tmp = appender!string; 228 229 final switch (level) 230 { 231 case SnippetLevel.global: 232 case SnippetLevel.other: 233 case SnippetLevel.comment: 234 case SnippetLevel.docComment: 235 case SnippetLevel.strings: 236 case SnippetLevel.mixinTemplate: 237 break; 238 case SnippetLevel.type: 239 tmp.put("struct FORMAT_HELPER {\n"); 240 break; 241 case SnippetLevel.method: 242 tmp.put("void FORMAT_HELPER() {\n"); 243 break; 244 case SnippetLevel.value: 245 tmp.put("int FORMAT_HELPER() = "); 246 break; 247 } 248 249 scope const(char)[][string] tokens; 250 251 ptrdiff_t dollar, last; 252 while (true) 253 { 254 dollar = snippet.indexOfAny(`$\`, last); 255 if (dollar == -1) 256 { 257 tmp ~= snippet[last .. $]; 258 break; 259 } 260 261 tmp ~= snippet[last .. dollar]; 262 last = dollar + 1; 263 if (last >= snippet.length) 264 break; 265 if (snippet[dollar] == '\\') 266 { 267 tmp ~= snippet[dollar + 1]; 268 last = dollar + 2; 269 } 270 else 271 { 272 string key = "__WspD_Snp_" ~ dollar.to!string ~ "_"; 273 const(char)[] str; 274 275 bool startOfBlock = snippet[0 .. dollar].stripRight.endsWith("{"); 276 bool endOfBlock; 277 278 bool makeWrappingIfMayBeDelegate() 279 { 280 endOfBlock = snippet[last .. $].stripLeft.startsWith("}"); 281 if (startOfBlock && endOfBlock) 282 { 283 // make extra long to make dfmt definitely wrap this (in case this is a delegate, otherwise this doesn't hurt either) 284 key.reserve(key.length + 200); 285 foreach (i; 0 .. 200) 286 key ~= "_"; 287 return true; 288 } 289 else 290 return false; 291 } 292 293 if (snippet[dollar + 1] == '{') 294 { 295 ptrdiff_t i = dollar + 2; 296 int depth = 1; 297 while (true) 298 { 299 auto next = snippet.indexOfAny(`\{}`, i); 300 if (next == -1) 301 { 302 i = snippet.length; 303 break; 304 } 305 306 if (snippet[next] == '\\') 307 i = next + 2; 308 else 309 { 310 if (snippet[next] == '{') 311 depth++; 312 else if (snippet[next] == '}') 313 depth--; 314 else 315 assert(false); 316 317 i = next + 1; 318 } 319 320 if (depth == 0) 321 break; 322 } 323 str = snippet[dollar .. i]; 324 last = i; 325 326 const wrapped = makeWrappingIfMayBeDelegate(); 327 328 const placeholderMightBeIdentifier = str.length > 5 329 || snippet[last .. $].stripLeft.startsWith(";", ".", "{", "(", "["); 330 331 if (wrapped || placeholderMightBeIdentifier) 332 { 333 // let's insert some token in here instead of a comment because there is probably some default content 334 // if there is a semicolon at the end we probably need to insert a semicolon here too 335 // if this is a comment placeholder let's insert a semicolon to make dfmt wrap 336 if (str[0 .. $ - 1].endsWith(';') || str[0 .. $ - 1].canFind("//")) 337 key ~= ';'; 338 } 339 else if (level != SnippetLevel.value) 340 { 341 // empty default, put in comment 342 key = "/+++" ~ key ~ "+++/"; 343 } 344 } 345 else 346 { 347 size_t end = dollar + 1; 348 349 if (snippet[dollar + 1].isDigit) 350 { 351 while (end < snippet.length && snippet[end].isDigit) 352 end++; 353 } 354 else 355 { 356 while (end < snippet.length && (snippet[end].isAlphaNum || snippet[end] == '_')) 357 end++; 358 } 359 360 str = snippet[dollar .. end]; 361 last = end; 362 363 makeWrappingIfMayBeDelegate(); 364 365 const placeholderMightBeIdentifier = snippet[last .. $].stripLeft.startsWith(";", 366 ".", "{", "(", "["); 367 368 if (placeholderMightBeIdentifier) 369 { 370 // keep value thing as simple identifier as we don't have any placeholder text 371 } 372 else if (level != SnippetLevel.value) 373 { 374 // primitive placeholder as comment 375 key = "/+++" ~ key ~ "+++/"; 376 } 377 } 378 379 tokens[key] = str; 380 tmp ~= key; 381 } 382 } 383 384 final switch (level) 385 { 386 case SnippetLevel.global: 387 case SnippetLevel.other: 388 case SnippetLevel.comment: 389 case SnippetLevel.docComment: 390 case SnippetLevel.strings: 391 case SnippetLevel.mixinTemplate: 392 break; 393 case SnippetLevel.type: 394 case SnippetLevel.method: 395 tmp.put("}"); 396 break; 397 case SnippetLevel.value: 398 tmp.put(";"); 399 break; 400 } 401 402 auto res = dfmt.formatSync(tmp.data, arguments); 403 404 string chompStr; 405 char del; 406 final switch (level) 407 { 408 case SnippetLevel.global: 409 case SnippetLevel.other: 410 case SnippetLevel.comment: 411 case SnippetLevel.docComment: 412 case SnippetLevel.strings: 413 case SnippetLevel.mixinTemplate: 414 break; 415 case SnippetLevel.type: 416 case SnippetLevel.method: 417 chompStr = "}"; 418 del = '{'; 419 break; 420 case SnippetLevel.value: 421 chompStr = ";"; 422 del = '='; 423 break; 424 } 425 426 if (chompStr.length) 427 res = res.stripRight.chomp(chompStr); 428 429 if (del != char.init) 430 { 431 auto start = res.indexOf(del); 432 if (start != -1) 433 { 434 res = res[start + 1 .. $]; 435 436 while (true) 437 { 438 // delete empty lines before first line 439 auto nl = res.indexOf('\n'); 440 if (nl != -1 && res[0 .. nl].all!isWhite) 441 res = res[nl + 1 .. $]; 442 else 443 break; 444 } 445 446 auto indent = res[0 .. res.length - res.stripLeft.length]; 447 if (indent.length) 448 { 449 // remove indentation of whole block 450 assert(indent.all!isWhite); 451 res = res.splitLines.map!(a => a.startsWith(indent) 452 ? a[indent.length .. $] : a.stripRight).join("\n"); 453 } 454 } 455 } 456 457 foreach (key, value; tokens) 458 { 459 // TODO: replacing using aho-corasick would be far more efficient but there is nothing like that in phobos 460 res = res.replace(key, value); 461 } 462 463 if (res.endsWith("\r\n") && !snippet.endsWith('\n')) 464 res.length -= 2; 465 else if (res.endsWith('\n') && !snippet.endsWith('\n')) 466 res.length--; 467 468 if (res.endsWith(";\n\n$0")) 469 res = res[0 .. $ - "\n$0".length] ~ "$0"; 470 else if (res.endsWith(";\r\n\r\n$0")) 471 res = res[0 .. $ - "\r\n$0".length] ~ "$0"; 472 473 return res; 474 } 475 476 /// Adds snippets which complete conditionally based on dub dependencies being present. 477 /// This function affects the global configuration of all instances. 478 /// Params: 479 /// requiredDependencies = The dependencies which must be present in order for this snippet to show up. 480 /// snippet = The snippet to suggest when the required dependencies are matched. 481 void addDependencySnippet(string[] requiredDependencies, PlainSnippet snippet) 482 { 483 // maybe application global change isn't such a good idea? Current config system seems too inefficient for this. 484 dependencySnippets.addSnippet(requiredDependencies, snippet); 485 } 486 487 private: 488 Tuple!(SnippetInfo, Future!(Snippet[])[]) collectSnippets(scope const(char)[] file, 489 scope const(char)[] code, int position) 490 { 491 const inst = instance; 492 auto info = determineSnippetInfo(file, code, position); 493 auto futures = appender!(Future!(Snippet[])[]); 494 foreach (provider; providers) 495 futures.put(provider.provideSnippets(inst, file, code, position, info)); 496 return tuple(info, futures.data); 497 } 498 499 LexerConfig config; 500 } 501 502 /// 503 enum SnippetLevel 504 { 505 /// Outside of functions or types, possibly inside templates 506 global, 507 /// Inside interfaces, classes, structs or unions 508 type, 509 /// Inside method body 510 method, 511 /// inside a variable value, argument call, default value or similar 512 value, 513 /// Other scope types (for example outside of braces but after a function definition or some other invalid syntax place) 514 other, 515 /// Inside a string literal. 516 strings, 517 /// Inside a normal comment 518 comment, 519 /// Inside a documentation comment 520 docComment, 521 /// Inside explicitly declared mixin templates 522 mixinTemplate, 523 } 524 525 /// 526 struct SnippetLoopScope 527 { 528 /// true if an loop expression can be inserted at this point 529 bool supported; 530 /// true if we know we are iterating over a string (possibly needing unicode decoding) or false otherwise 531 bool stringIterator; 532 /// Explicit type to use when iterating or null if none is known 533 string type; 534 /// Best variable to iterate over or null if none was found 535 string iterator; 536 /// Number of keys to iterate over 537 int numItems = 1; 538 } 539 540 /// 541 struct SnippetInfo 542 { 543 /// Levels this snippet location has gone through, latest one being the last 544 SnippetLevel[] stack = [SnippetLevel.global]; 545 /// Information about snippets using loop context 546 SnippetLoopScope loopScope; 547 548 /// Current snippet scope level of the location 549 SnippetLevel level() const @property 550 { 551 return stack.length ? stack[$ - 1] : SnippetLevel.other; 552 } 553 } 554 555 /// A list of snippets resolved at a given position. 556 struct SnippetList 557 { 558 /// The info where this snippet is completing at. 559 SnippetInfo info; 560 /// The list of snippets that got returned. 561 Snippet[] snippets; 562 } 563 564 /// 565 interface SnippetProvider 566 { 567 Future!(Snippet[]) provideSnippets(scope const WorkspaceD.Instance instance, 568 scope const(char)[] file, scope const(char)[] code, int position, const SnippetInfo info); 569 570 Future!Snippet resolveSnippet(scope const WorkspaceD.Instance instance, 571 scope const(char)[] file, scope const(char)[] code, int position, 572 const SnippetInfo info, Snippet snippet); 573 } 574 575 /// Snippet to insert 576 struct Snippet 577 { 578 /// Internal ID for resolving this snippet 579 string id, providerId; 580 /// User-defined data for helping resolving this snippet 581 JSONValue data; 582 /// Label for this snippet 583 string title; 584 /// Shortcut to type for this snippet 585 string shortcut; 586 /// Markdown documentation for this snippet 587 string documentation; 588 /// Plain text to insert assuming global level indentation. 589 string plain; 590 /// Text with interactive snippet locations to insert assuming global indentation. 591 string snippet; 592 /// true if this snippet can be used as-is 593 bool resolved; 594 /// true if this snippet shouldn't be formatted. 595 bool unformatted; 596 } 597 598 unittest 599 { 600 scope backend = new WorkspaceD(); 601 auto workspace = makeTemporaryTestingWorkspace; 602 auto instance = backend.addInstance(workspace.directory); 603 backend.register!SnippetsComponent; 604 backend.register!DfmtComponent; 605 SnippetsComponent snippets = backend.get!SnippetsComponent(workspace.directory); 606 607 auto args = ["--indent_style", "tab"]; 608 609 auto res = snippets.formatSync("void main(${1:string[] args}) {\n\t$0\n}", args); 610 assert(res == "void main(${1:string[] args})\n{\n\t$0\n}"); 611 612 res = snippets.formatSync("class ${1:MyClass} {\n\t$0\n}", args); 613 assert(res == "class ${1:MyClass}\n{\n\t$0\n}"); 614 615 res = snippets.formatSync("enum ${1:MyEnum} = $2;\n$0", args); 616 assert(res == "enum ${1:MyEnum} = $2;\n$0"); 617 618 res = snippets.formatSync("import ${1:std};\n$0", args); 619 assert(res == "import ${1:std};\n$0"); 620 621 res = snippets.formatSync("import ${1:std};\n$0", args, SnippetLevel.method); 622 assert(res == "import ${1:std};\n$0"); 623 624 res = snippets.formatSync("foo(delegate() {\n${1:// foo}\n});", args, SnippetLevel.method); 625 assert(res == "foo(delegate() {\n\t${1:// foo}\n});"); 626 627 res = snippets.formatSync(`auto ${1:window} = new SimpleWindow(Size(${2:800, 600}), "$3");`, args, SnippetLevel.method); 628 assert(res == `auto ${1:window} = new SimpleWindow(Size(${2:800, 600}), "$3");`); 629 } 630 631 unittest 632 { 633 scope backend = new WorkspaceD(); 634 auto workspace = makeTemporaryTestingWorkspace; 635 auto instance = backend.addInstance(workspace.directory); 636 backend.register!SnippetsComponent; 637 SnippetsComponent snippets = backend.get!SnippetsComponent(workspace.directory); 638 639 static immutable testCode = `/// this is a cool module 640 module something; 641 642 // todo stuff 643 644 /// a lot of functionality 645 void foo() 646 { 647 writeln("hello world"); 648 } 649 650 int main(string[] args) 651 { 652 foo(); 653 } 654 `; 655 656 // TODO: comment snippets not yet implemented 657 658 auto i = snippets.determineSnippetInfo(null, testCode, 0); 659 assert(i.level == SnippetLevel.global, i.stack.to!string); 660 i = snippets.determineSnippetInfo(null, testCode, 1); 661 assert(i.level == SnippetLevel.global, i.stack.to!string); 662 i = snippets.determineSnippetInfo(null, testCode, 2); 663 // assert(i.level == SnippetLevel.comment, i.stack.to!string); 664 i = snippets.determineSnippetInfo(null, testCode, 3); 665 // assert(i.level == SnippetLevel.docComment, i.stack.to!string); 666 i = snippets.determineSnippetInfo(null, testCode, 10); 667 // assert(i.level == SnippetLevel.docComment, i.stack.to!string); 668 i = snippets.determineSnippetInfo(null, testCode, 25); 669 // assert(i.level == SnippetLevel.docComment, i.stack.to!string); 670 671 i = snippets.determineSnippetInfo(null, testCode, 26); 672 assert(i.level == SnippetLevel.global, i.stack.to!string); 673 674 i = snippets.determineSnippetInfo(null, testCode, 47); 675 // assert(i.level == SnippetLevel.comment, i.stack.to!string); 676 i = snippets.determineSnippetInfo(null, testCode, 50); 677 // assert(i.level == SnippetLevel.comment, i.stack.to!string); 678 i = snippets.determineSnippetInfo(null, testCode, 58); 679 // assert(i.level == SnippetLevel.comment, i.stack.to!string); 680 681 i = snippets.determineSnippetInfo(null, testCode, 59); 682 assert(i.level == SnippetLevel.global, i.stack.to!string); 683 i = snippets.determineSnippetInfo(null, testCode, 60); 684 assert(i.level == SnippetLevel.global, i.stack.to!string); 685 686 i = snippets.determineSnippetInfo(null, testCode, 63); 687 // assert(i.level == SnippetLevel.docComment, i.stack.to!string); 688 689 i = snippets.determineSnippetInfo(null, testCode, 109); 690 assert(i.level == SnippetLevel.value, i.stack.to!string); 691 i = snippets.determineSnippetInfo(null, testCode, 110); 692 assert(i.level == SnippetLevel.strings, i.stack.to!string); 693 i = snippets.determineSnippetInfo(null, testCode, 111); 694 assert(i.level == SnippetLevel.strings, i.stack.to!string); 695 i = snippets.determineSnippetInfo(null, testCode, 121); 696 assert(i.level == SnippetLevel.strings, i.stack.to!string); 697 i = snippets.determineSnippetInfo(null, testCode, 122); 698 assert(i.level == SnippetLevel.other, i.stack.to!string); 699 }