1 module workspaced.com.dub; 2 3 import core.sync.mutex; 4 import core.exception; 5 import core.thread; 6 7 import std.algorithm; 8 import std.conv; 9 import std.exception; 10 import std.json : JSONValue, JSON_TYPE; 11 import std.parallelism; 12 import std.regex; 13 import std.stdio; 14 import std.string; 15 16 import painlessjson : toJSON, fromJSON; 17 18 import workspaced.api; 19 20 import dub.description; 21 import dub.dub; 22 import dub.package_; 23 import dub.project; 24 25 import dub.compilers.compiler; 26 import dub.generators.build; 27 import dub.generators.generator; 28 29 import dub.compilers.buildsettings; 30 31 import dub.internal.vibecompat.core.log; 32 import dub.internal.vibecompat.inet.url; 33 34 @component("dub") 35 class DubComponent : ComponentWrapper 36 { 37 mixin DefaultComponentWrapper; 38 39 static void registered() 40 { 41 setLogLevel(LogLevel.none); 42 } 43 44 protected void load() 45 { 46 if (!refInstance) 47 throw new Exception("dub requires to be instanced"); 48 49 if (config.get!bool("dub", "registerImportProvider", true)) 50 importPathProvider = &imports; 51 if (config.get!bool("dub", "registerStringImportProvider", true)) 52 stringImportPathProvider = &stringImports; 53 if (config.get!bool("dub", "registerImportFilesProvider", false)) 54 importFilesProvider = &fileImports; 55 56 try 57 { 58 start(); 59 60 _configuration = _dub.project.getDefaultConfiguration(_platform); 61 if (!_dub.project.configurations.canFind(_configuration)) 62 { 63 stderr.writeln("Dub Error: No configuration available"); 64 workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("warning"), 65 "component" : JSONValue("dub"), "detail" : JSONValue("invalid-default-config")])); 66 } 67 else 68 updateImportPaths(false); 69 } 70 catch (Exception e) 71 { 72 if (!_dub || !_dub.project) 73 throw e; 74 stderr.writeln("Dub Error (ignored): ", e); 75 } 76 /*catch (AssertError e) 77 { 78 if (!_dub || !_dub.project) 79 throw e; 80 stderr.writeln("Dub Error (ignored): ", e); 81 }*/ 82 } 83 84 private void start() 85 { 86 _dubRunning = false; 87 _dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none); 88 _dub.packageManager.getOrLoadPackage(NativePath(instance.cwd)); 89 _dub.loadPackage(); 90 _dub.project.validate(); 91 92 // mark all packages as optional so we don't crash 93 int missingPackages; 94 auto optionalified = optionalifyPackages; 95 foreach (ref pkg; _dub.project.getTopologicalPackageList()) 96 { 97 optionalifyRecipe(pkg); 98 foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name))) 99 { 100 auto d = _dub.project.getDependency(dep.name, true); 101 if (!d) 102 missingPackages++; 103 else 104 optionalifyRecipe(d); 105 } 106 } 107 108 if (!_compilerBinaryName.length) 109 _compilerBinaryName = _dub.defaultCompiler; 110 setCompiler(_compilerBinaryName); 111 if (missingPackages > 0) 112 { 113 upgrade(false); 114 optionalifyPackages(); 115 } 116 117 _dubRunning = true; 118 } 119 120 private string[] optionalifyPackages() 121 { 122 bool[Package] visited; 123 string[] optionalified; 124 foreach (pkg; _dub.project.dependencies) 125 optionalified ~= optionalifyRecipe(cast() pkg); 126 return optionalified; 127 } 128 129 private string[] optionalifyRecipe(Package pkg) 130 { 131 string[] optionalified; 132 foreach (key, ref value; pkg.recipe.buildSettings.dependencies) 133 { 134 if (!value.optional) 135 { 136 value.optional = true; 137 value.default_ = true; 138 optionalified ~= key; 139 } 140 } 141 foreach (ref config; pkg.recipe.configurations) 142 foreach (key, ref value; config.buildSettings.dependencies) 143 { 144 if (!value.optional) 145 { 146 value.optional = true; 147 value.default_ = true; 148 optionalified ~= key; 149 } 150 } 151 return optionalified; 152 } 153 154 private void restart() 155 { 156 _dub.destroy(); 157 _dubRunning = false; 158 start(); 159 } 160 161 bool isRunning() 162 { 163 return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null 164 && _dubRunning; 165 } 166 167 /// Reloads the dub.json or dub.sdl file from the cwd 168 /// Returns: `false` if there are no import paths available 169 Future!bool update() 170 { 171 restart(); 172 auto ret = new Future!bool; 173 threads.create({ /**/ 174 try 175 { 176 auto result = updateImportPaths(false); 177 ret.finish(result); 178 } 179 catch (Throwable t) 180 { 181 ret.error(t); 182 } 183 }); 184 return ret; 185 } 186 187 bool updateImportPaths(bool restartDub = true) 188 { 189 validateConfiguration(); 190 191 if (restartDub) 192 restart(); 193 194 GeneratorSettings settings; 195 settings.platform = _platform; 196 settings.config = _configuration; 197 settings.buildType = _buildType; 198 settings.compiler = _compiler; 199 settings.buildSettings = _settings; 200 settings.buildSettings.addOptions(BuildOption.syntaxOnly); 201 settings.combined = true; 202 settings.run = false; 203 204 try 205 { 206 auto paths = _dub.project.listBuildSettings(settings, ["import-paths", 207 "string-import-paths", "source-files"], ListBuildSettingsFormat.listNul); 208 _importPaths = paths[0].split('\0'); 209 _stringImportPaths = paths[1].split('\0'); 210 _importFiles = paths[2].split('\0'); 211 return _importPaths.length > 0 || _importFiles.length > 0; 212 } 213 catch (Exception e) 214 { 215 workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("error"), "component" 216 : JSONValue("dub"), "detail" 217 : JSONValue("Error while listing import paths: " ~ e.toString)])); 218 _importPaths = []; 219 _stringImportPaths = []; 220 return false; 221 } 222 } 223 224 /// Calls `dub upgrade` 225 void upgrade(bool save = true) 226 { 227 if (save) 228 _dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade); 229 else 230 _dub.upgrade(UpgradeOptions.noSaveSelections); 231 } 232 233 /// Throws if configuration is invalid, otherwise does nothing. 234 void validateConfiguration() 235 { 236 if (!_dub.project.configurations.canFind(_configuration)) 237 throw new Exception("Cannot use dub with invalid configuration"); 238 } 239 240 /// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing. 241 void validateBuildConfiguration() 242 { 243 if (!_dub.project.configurations.canFind(_configuration)) 244 throw new Exception("Cannot use dub with invalid configuration"); 245 if (_settings.targetType == TargetType.none) 246 throw new Exception("Cannot build with dub with targetType == none"); 247 if (_settings.targetType == TargetType.sourceLibrary) 248 throw new Exception("Cannot build with dub with targetType == sourceLibrary"); 249 } 250 251 /// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself. 252 /// Returns: `[{dependencies: [string], ver: string, name: string}]` 253 auto dependencies() @property 254 { 255 validateConfiguration(); 256 257 return _dub.project.listDependencies(); 258 } 259 260 /// Lists dependencies of the root package. This can be used as a base to create a tree structure. 261 string[] rootDependencies() @property 262 { 263 validateConfiguration(); 264 265 return _dub.project.rootPackage.listDependencies(); 266 } 267 268 /// Returns the path to the root package recipe (dub.json/dub.sdl) 269 /// 270 /// Note that this can be empty if the package is not in the local file system. 271 string recipePath() @property 272 { 273 return _dub.project.rootPackage.recipePath.toString; 274 } 275 276 /// Lists all import paths 277 string[] imports() @property 278 { 279 return _importPaths; 280 } 281 282 /// Lists all string import paths 283 string[] stringImports() @property 284 { 285 return _stringImportPaths; 286 } 287 288 /// Lists all import paths to files 289 string[] fileImports() @property 290 { 291 return _importFiles; 292 } 293 294 /// Lists all configurations defined in the package description 295 string[] configurations() @property 296 { 297 return _dub.project.configurations; 298 } 299 300 /// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov") 301 string[] buildTypes() @property 302 { 303 string[] types = [ 304 "plain", "debug", "release", "release-debug", "release-nobounds", 305 "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov" 306 ]; 307 foreach (type, info; _dub.project.rootPackage.recipe.buildTypes) 308 types ~= type; 309 return types; 310 } 311 312 /// Gets the current selected configuration 313 string configuration() @property 314 { 315 return _configuration; 316 } 317 318 /// Selects a new configuration and updates the import paths accordingly 319 /// Returns: `false` if there are no import paths in the new configuration 320 bool setConfiguration(string configuration) 321 { 322 if (!_dub.project.configurations.canFind(configuration)) 323 return false; 324 _configuration = configuration; 325 return updateImportPaths(false); 326 } 327 328 /// List all possible arch types for current set compiler 329 string[] archTypes() @property 330 { 331 string[] types = ["x86_64", "x86"]; 332 333 string compilerName = _compiler.name; 334 335 version (Windows) 336 { 337 if (compilerName == "dmd") 338 { 339 types ~= "x86_mscoff"; 340 } 341 } 342 if (compilerName == "gdc") 343 { 344 types ~= ["arm", "arm_thumb"]; 345 } 346 347 return types; 348 } 349 350 /// Returns the current selected arch type 351 string archType() @property 352 { 353 return _archType; 354 } 355 356 /// Selects a new arch type and updates the import paths accordingly 357 /// Returns: `false` if there are no import paths in the new arch type 358 bool setArchType(JSONValue request) 359 { 360 enforce(request.type == JSON_TYPE.OBJECT && "arch-type" in request, "arch-type not in request"); 361 auto type = request["arch-type"].fromJSON!string; 362 if (archTypes.canFind(type)) 363 { 364 _archType = type; 365 return updateImportPaths(false); 366 } 367 else 368 { 369 return false; 370 } 371 } 372 373 /// Returns the current selected build type 374 string buildType() @property 375 { 376 return _buildType; 377 } 378 379 /// Selects a new build type and updates the import paths accordingly 380 /// Returns: `false` if there are no import paths in the new build type 381 bool setBuildType(JSONValue request) 382 { 383 enforce(request.type == JSON_TYPE.OBJECT && "build-type" in request, 384 "build-type not in request"); 385 auto type = request["build-type"].fromJSON!string; 386 if (buildTypes.canFind(type)) 387 { 388 _buildType = type; 389 return updateImportPaths(false); 390 } 391 else 392 { 393 return false; 394 } 395 } 396 397 /// Returns the current selected compiler 398 string compiler() @property 399 { 400 return _compilerBinaryName; 401 } 402 403 /// Selects a new compiler for building 404 /// Returns: `false` if the compiler does not exist 405 bool setCompiler(string compiler) 406 { 407 try 408 { 409 _compilerBinaryName = compiler; 410 _compiler = getCompiler(compiler); // make sure it gets a valid compiler 411 } 412 catch (Exception e) 413 { 414 return false; 415 } 416 _platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType); 417 return true; 418 } 419 420 /// Returns the project name 421 string name() @property 422 { 423 return _dub.projectName; 424 } 425 426 /// Returns the project path 427 auto path() @property 428 { 429 return _dub.projectPath; 430 } 431 432 /// Returns whether there is a target set to build. If this is false then build will throw an exception. 433 bool canBuild() @property 434 { 435 if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary 436 || !_dub.project.configurations.canFind(_configuration)) 437 return false; 438 return true; 439 } 440 441 /// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE. 442 Future!(BuildIssue[]) build() 443 { 444 import std.process : thisProcessID; 445 import std.file : tempDir; 446 import std.random : uniform; 447 448 validateBuildConfiguration(); 449 450 // copy to this thread 451 auto compiler = _compiler; 452 auto buildPlatform = _platform; 453 454 GeneratorSettings settings; 455 settings.platform = buildPlatform; 456 settings.config = _configuration; 457 settings.buildType = _buildType; 458 settings.compiler = compiler; 459 settings.tempBuild = true; 460 settings.buildSettings = _settings; 461 settings.buildSettings.targetPath = tempDir; 462 settings.buildSettings.targetName = "workspace-d-build-" 463 ~ thisProcessID.to!string ~ "-" ~ uniform!uint.to!string(36); 464 settings.buildSettings.addOptions(BuildOption.syntaxOnly); 465 settings.buildSettings.addDFlags("-o-"); 466 467 auto ret = new Future!(BuildIssue[]); 468 new Thread({ 469 try 470 { 471 BuildIssue[] issues; 472 473 settings.compileCallback = (status, output) { 474 string[] lines = output.splitLines; 475 foreach (line; lines) 476 { 477 auto match = line.matchFirst(errorFormat); 478 if (match) 479 { 480 issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0), 481 match[1], match[4].to!ErrorType, match[5]); 482 } 483 else 484 { 485 if (line.canFind("from")) 486 { 487 auto contMatch = line.matchFirst(errorFormatCont); 488 if (contMatch) 489 { 490 issues ~= BuildIssue(contMatch[2].to!int, 491 contMatch[3].toOr!int(1), contMatch[1], ErrorType.Error, contMatch[4]); 492 } 493 } 494 if (line.canFind("is deprecated")) 495 { 496 auto deprMatch = line.matchFirst(deprecationFormat); 497 if (deprMatch) 498 { 499 issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1), 500 deprMatch[1], ErrorType.Deprecation, 501 deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead."); 502 // TODO: maybe add special type or output 503 } 504 } 505 } 506 } 507 }; 508 try 509 { 510 _dub.generateProject("build", settings); 511 } 512 catch (Exception e) 513 { 514 if (!e.msg.matchFirst(harmlessExceptionFormat)) 515 throw e; 516 } 517 ret.finish(issues); 518 } 519 catch (Throwable t) 520 { 521 ret.error(t); 522 } 523 }).start(); 524 return ret; 525 } 526 527 /// Converts the root package recipe to another format. 528 /// Params: 529 /// format = either "json" or "sdl". 530 string convertRecipe(string format) 531 { 532 import dub.recipe.io : serializePackageRecipe; 533 import std.array : appender; 534 535 auto dst = appender!string; 536 serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format); 537 return dst.data; 538 } 539 540 private: 541 Dub _dub; 542 bool _dubRunning = false; 543 string _configuration; 544 string _archType = "x86_64"; 545 string _buildType = "debug"; 546 string _compilerBinaryName; 547 Compiler _compiler; 548 BuildSettings _settings; 549 BuildPlatform _platform; 550 string[] _importPaths, _stringImportPaths, _importFiles; 551 } 552 553 /// 554 enum ErrorType : ubyte 555 { 556 /// 557 Error = 0, 558 /// 559 Warning = 1, 560 /// 561 Deprecation = 2 562 } 563 564 /// Returned by build 565 struct BuildIssue 566 { 567 /// 568 int line, column; 569 /// 570 string file; 571 /// 572 ErrorType type; 573 /// 574 string text; 575 } 576 577 private: 578 579 T toOr(T)(string s, T defaultValue) 580 { 581 if (!s || !s.length) 582 return defaultValue; 583 return s.to!T; 584 } 585 586 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g"); 587 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi"); 588 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (.*)`, "g"); 589 enum deprecationFormat = ctRegex!( 590 `(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g"); 591 592 struct DubPackageInfo 593 { 594 string[string] dependencies; 595 string ver; 596 string name; 597 string path; 598 string description; 599 string homepage; 600 const(string)[] authors; 601 string copyright; 602 string license; 603 DubPackageInfo[] subPackages; 604 605 void fill(in PackageRecipe recipe) 606 { 607 description = recipe.description; 608 homepage = recipe.homepage; 609 authors = recipe.authors; 610 copyright = recipe.copyright; 611 license = recipe.license; 612 613 foreach (subpackage; recipe.subPackages) 614 { 615 DubPackageInfo info; 616 info.ver = subpackage.recipe.version_; 617 info.name = subpackage.recipe.name; 618 info.path = subpackage.path; 619 info.fill(subpackage.recipe); 620 } 621 } 622 } 623 624 DubPackageInfo getInfo(in Package dep) 625 { 626 DubPackageInfo info; 627 info.name = dep.name; 628 info.ver = dep.version_.toString; 629 info.path = dep.path.toString; 630 info.fill(dep.recipe); 631 foreach (subDep; dep.getAllDependencies()) 632 { 633 info.dependencies[subDep.name] = subDep.spec.toString; 634 } 635 return info; 636 } 637 638 auto listDependencies(Project project) 639 { 640 auto deps = project.dependencies; 641 DubPackageInfo[] dependencies; 642 if (deps is null) 643 return dependencies; 644 foreach (dep; deps) 645 { 646 dependencies ~= getInfo(dep); 647 } 648 return dependencies; 649 } 650 651 string[] listDependencies(Package pkg) 652 { 653 auto deps = pkg.getAllDependencies(); 654 string[] dependencies; 655 if (deps is null) 656 return dependencies; 657 foreach (dep; deps) 658 dependencies ~= dep.name; 659 return dependencies; 660 }