1 module workspaced.com.dub;
2 
3 import core.exception;
4 import core.sync.mutex;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.array : appender;
9 import std.conv;
10 import std.exception;
11 import std.json : JSONType, JSONValue;
12 import std.parallelism;
13 import std.regex;
14 import std.stdio;
15 import std.string;
16 
17 import painlessjson : toJSON, fromJSON;
18 
19 import workspaced.api;
20 
21 import dub.description;
22 import dub.dub;
23 import dub.package_;
24 import dub.project;
25 
26 import dub.compilers.buildsettings;
27 import dub.compilers.compiler;
28 import dub.generators.build;
29 import dub.generators.generator;
30 
31 import dub.internal.vibecompat.core.log;
32 import dub.internal.vibecompat.inet.url;
33 
34 import dub.recipe.io;
35 
36 @component("dub")
37 class DubComponent : ComponentWrapper
38 {
39 	mixin DefaultComponentWrapper;
40 
41 	static void registered()
42 	{
43 		setLogLevel(LogLevel.none);
44 	}
45 
46 	protected void load()
47 	{
48 		if (!refInstance)
49 			throw new Exception("dub requires to be instanced");
50 
51 		if (config.get!bool("dub", "registerImportProvider", true))
52 			importPathProvider = &imports;
53 		if (config.get!bool("dub", "registerStringImportProvider", true))
54 			stringImportPathProvider = &stringImports;
55 		if (config.get!bool("dub", "registerImportFilesProvider", false))
56 			importFilesProvider = &fileImports;
57 
58 		try
59 		{
60 			start();
61 
62 			_configuration = _dub.project.getDefaultConfiguration(_platform);
63 			if (!_dub.project.configurations.canFind(_configuration))
64 			{
65 				stderr.writeln("Dub Error: No configuration available");
66 				workspaced.broadcast(refInstance, JSONValue([
67 							"type": JSONValue("warning"),
68 							"component": JSONValue("dub"),
69 							"detail": JSONValue("invalid-default-config")
70 						]));
71 			}
72 			else
73 				updateImportPaths(false);
74 		}
75 		catch (Exception e)
76 		{
77 			if (!_dub || !_dub.project)
78 				throw e;
79 			stderr.writeln("Dub Error (ignored): ", e);
80 		}
81 		/*catch (AssertError e)
82 		{
83 			if (!_dub || !_dub.project)
84 				throw e;
85 			stderr.writeln("Dub Error (ignored): ", e);
86 		}*/
87 	}
88 
89 	private void start()
90 	{
91 		_dubRunning = false;
92 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
93 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
94 		_dub.loadPackage();
95 		_dub.project.validate();
96 
97 		// mark all packages as optional so we don't crash
98 		int missingPackages;
99 		auto optionalified = optionalifyPackages;
100 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
101 		{
102 			optionalifyRecipe(pkg);
103 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
104 			{
105 				auto d = _dub.project.getDependency(dep.name, true);
106 				if (!d)
107 					missingPackages++;
108 				else
109 					optionalifyRecipe(d);
110 			}
111 		}
112 
113 		if (!_compilerBinaryName.length)
114 			_compilerBinaryName = _dub.defaultCompiler;
115 		setCompiler(_compilerBinaryName);
116 
117 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings();
118 
119 		if (missingPackages > 0)
120 		{
121 			upgrade(false);
122 			optionalifyPackages();
123 		}
124 
125 		_dubRunning = true;
126 	}
127 
128 	private string[] optionalifyPackages()
129 	{
130 		bool[Package] visited;
131 		string[] optionalified;
132 		foreach (pkg; _dub.project.dependencies)
133 			optionalified ~= optionalifyRecipe(cast() pkg);
134 		return optionalified;
135 	}
136 
137 	private string[] optionalifyRecipe(Package pkg)
138 	{
139 		string[] optionalified;
140 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
141 		{
142 			if (!value.optional)
143 			{
144 				value.optional = true;
145 				value.default_ = true;
146 				optionalified ~= key;
147 			}
148 		}
149 		foreach (ref config; pkg.recipe.configurations)
150 			foreach (key, ref value; config.buildSettings.dependencies)
151 			{
152 				if (!value.optional)
153 				{
154 					value.optional = true;
155 					value.default_ = true;
156 					optionalified ~= key;
157 				}
158 			}
159 		return optionalified;
160 	}
161 
162 	private void restart()
163 	{
164 		_dub.destroy();
165 		_dubRunning = false;
166 		start();
167 	}
168 
169 	bool isRunning()
170 	{
171 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
172 			&& _dubRunning;
173 	}
174 
175 	/// Reloads the dub.json or dub.sdl file from the cwd
176 	/// Returns: `false` if there are no import paths available
177 	Future!bool update()
178 	{
179 		restart();
180 		mixin(gthreadsAsyncProxy!`updateImportPaths(false)`);
181 	}
182 
183 	bool updateImportPaths(bool restartDub = true)
184 	{
185 		validateConfiguration();
186 
187 		if (restartDub)
188 			restart();
189 
190 		GeneratorSettings settings;
191 		settings.platform = _platform;
192 		settings.config = _configuration;
193 		settings.buildType = _buildType;
194 		settings.compiler = _compiler;
195 		settings.buildSettings = _settings;
196 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
197 		settings.combined = true;
198 		settings.run = false;
199 
200 		try
201 		{
202 			auto paths = _dub.project.listBuildSettings(settings, [
203 					"import-paths", "string-import-paths", "source-files"
204 					], ListBuildSettingsFormat.listNul);
205 			_importPaths = paths[0].split('\0');
206 			_stringImportPaths = paths[1].split('\0');
207 			_importFiles = paths[2].split('\0');
208 			return _importPaths.length > 0 || _importFiles.length > 0;
209 		}
210 		catch (Exception e)
211 		{
212 			workspaced.broadcast(refInstance, JSONValue([
213 						"type": JSONValue("error"),
214 						"component": JSONValue("dub"),
215 						"detail": JSONValue("Error while listing import paths: " ~ e.toString)
216 					]));
217 			_importPaths = [];
218 			_stringImportPaths = [];
219 			return false;
220 		}
221 	}
222 
223 	/// Calls `dub upgrade`
224 	void upgrade(bool save = true)
225 	{
226 		if (save)
227 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
228 		else
229 			_dub.upgrade(UpgradeOptions.noSaveSelections);
230 	}
231 
232 	/// Throws if configuration is invalid, otherwise does nothing.
233 	void validateConfiguration() const
234 	{
235 		if (!_dub.project.configurations.canFind(_configuration))
236 			throw new Exception("Cannot use dub with invalid configuration");
237 	}
238 
239 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
240 	void validateBuildConfiguration()
241 	{
242 		if (!_dub.project.configurations.canFind(_configuration))
243 			throw new Exception("Cannot use dub with invalid configuration");
244 		if (_settings.targetType == TargetType.none)
245 			throw new Exception("Cannot build with dub with targetType == none");
246 		if (_settings.targetType == TargetType.sourceLibrary)
247 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
248 	}
249 
250 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
251 	/// Returns: `[{dependencies: string[string], ver: string, name: string}]`
252 	auto dependencies() @property const
253 	{
254 		validateConfiguration();
255 
256 		return listDependencies(_dub.project);
257 	}
258 
259 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
260 	string[] rootDependencies() @property const
261 	{
262 		validateConfiguration();
263 
264 		return listDependencies(_dub.project.rootPackage);
265 	}
266 
267 	/// Returns the path to the root package recipe (dub.json/dub.sdl)
268 	///
269 	/// Note that this can be empty if the package is not in the local file system.
270 	string recipePath() @property
271 	{
272 		return _dub.project.rootPackage.recipePath.toString;
273 	}
274 
275 	/// Re-parses the package recipe on the file system and returns if the syntax is valid.
276 	/// Returns: empty string/null if no error occured, error message if an error occured.
277 	string validateRecipeSyntaxOnFileSystem()
278 	{
279 		auto p = recipePath;
280 		if (!p.length)
281 			return "Package is not in local file system";
282 
283 		try
284 		{
285 			readPackageRecipe(p);
286 			return null;
287 		}
288 		catch (Exception e)
289 		{
290 			return e.msg;
291 		}
292 	}
293 
294 	/// Lists all import paths
295 	string[] imports() @property nothrow
296 	{
297 		return _importPaths;
298 	}
299 
300 	/// Lists all string import paths
301 	string[] stringImports() @property nothrow
302 	{
303 		return _stringImportPaths;
304 	}
305 
306 	/// Lists all import paths to files
307 	string[] fileImports() @property nothrow
308 	{
309 		return _importFiles;
310 	}
311 
312 	/// Lists all configurations defined in the package description
313 	string[] configurations() @property
314 	{
315 		return _dub.project.configurations;
316 	}
317 
318 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
319 	string[] buildTypes() @property
320 	{
321 		string[] types = [
322 			"plain", "debug", "release", "release-debug", "release-nobounds",
323 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
324 		];
325 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
326 			types ~= type;
327 		return types;
328 	}
329 
330 	/// Gets the current selected configuration
331 	string configuration() @property
332 	{
333 		return _configuration;
334 	}
335 
336 	/// Selects a new configuration and updates the import paths accordingly
337 	/// Returns: `false` if there are no import paths in the new configuration
338 	bool setConfiguration(string configuration)
339 	{
340 		if (!_dub.project.configurations.canFind(configuration))
341 			return false;
342 		_configuration = configuration;
343 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings(configuration);
344 		return updateImportPaths(false);
345 	}
346 
347 	/// List all possible arch types for current set compiler
348 	string[] archTypes() @property
349 	{
350 		string[] types = ["x86_64", "x86"];
351 
352 		string compilerName = _compiler.name;
353 
354 		version (Windows)
355 		{
356 			if (compilerName == "dmd")
357 			{
358 				types ~= "x86_mscoff";
359 			}
360 		}
361 		if (compilerName == "gdc")
362 		{
363 			types ~= ["arm", "arm_thumb"];
364 		}
365 
366 		return types;
367 	}
368 
369 	/// Returns the current selected arch type
370 	string archType() @property
371 	{
372 		return _archType;
373 	}
374 
375 	/// Selects a new arch type and updates the import paths accordingly
376 	/// Returns: `false` if there are no import paths in the new arch type
377 	bool setArchType(JSONValue request)
378 	{
379 		enforce(request.type == JSONType.object && "arch-type" in request, "arch-type not in request");
380 		auto type = request["arch-type"].fromJSON!string;
381 		if (archTypes.canFind(type))
382 		{
383 			_archType = type;
384 			return updateImportPaths(false);
385 		}
386 		else
387 		{
388 			return false;
389 		}
390 	}
391 
392 	/// Returns the current selected build type
393 	string buildType() @property
394 	{
395 		return _buildType;
396 	}
397 
398 	/// Selects a new build type and updates the import paths accordingly
399 	/// Returns: `false` if there are no import paths in the new build type
400 	bool setBuildType(JSONValue request)
401 	{
402 		enforce(request.type == JSONType.object && "build-type" in request, "build-type not in request");
403 		auto type = request["build-type"].fromJSON!string;
404 		if (buildTypes.canFind(type))
405 		{
406 			_buildType = type;
407 			return updateImportPaths(false);
408 		}
409 		else
410 		{
411 			return false;
412 		}
413 	}
414 
415 	/// Returns the current selected compiler
416 	string compiler() @property
417 	{
418 		return _compilerBinaryName;
419 	}
420 
421 	/// Selects a new compiler for building
422 	/// Returns: `false` if the compiler does not exist
423 	bool setCompiler(string compiler)
424 	{
425 		try
426 		{
427 			_compilerBinaryName = compiler;
428 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
429 		}
430 		catch (Exception e)
431 		{
432 			return false;
433 		}
434 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
435 		_settingsTemplate.getPlatformSettings(_settings, _platform, _dub.project.rootPackage.path);
436 		return _compiler !is null;
437 	}
438 
439 	/// Returns the project name
440 	string name() @property
441 	{
442 		return _dub.projectName;
443 	}
444 
445 	/// Returns the project path
446 	auto path() @property
447 	{
448 		return _dub.projectPath;
449 	}
450 
451 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
452 	bool canBuild() @property
453 	{
454 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
455 				|| !_dub.project.configurations.canFind(_configuration))
456 			return false;
457 		return true;
458 	}
459 
460 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
461 	Future!(BuildIssue[]) build()
462 	{
463 		import std.process : thisProcessID;
464 		import std.file : tempDir;
465 		import std.random : uniform;
466 
467 		validateBuildConfiguration();
468 
469 		// copy to this thread
470 		auto compiler = _compiler;
471 		auto buildPlatform = _platform;
472 
473 		GeneratorSettings settings;
474 		settings.platform = buildPlatform;
475 		settings.config = _configuration;
476 		settings.buildType = _buildType;
477 		settings.compiler = compiler;
478 		settings.buildSettings = _settings;
479 
480 		auto ret = new Future!(BuildIssue[]);
481 		new Thread({
482 			try
483 			{
484 				auto issues = appender!(BuildIssue[]);
485 
486 				settings.compileCallback = (status, output) {
487 					string[] lines = output.splitLines;
488 					foreach (line; lines)
489 					{
490 						auto match = line.matchFirst(errorFormat);
491 						if (match)
492 						{
493 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
494 								match[1], match[4].to!ErrorType, match[5]);
495 						}
496 						else
497 						{
498 							auto contMatch = line.matchFirst(errorFormatCont);
499 							if (issues.data.length && contMatch)
500 							{
501 								issues ~= BuildIssue(contMatch[2].to!int,
502 									contMatch[3].toOr!int(1), contMatch[1],
503 									issues.data[$ - 1].type, contMatch[4], true);
504 							}
505 							else if (line.canFind("is deprecated"))
506 							{
507 								auto deprMatch = line.matchFirst(deprecationFormat);
508 								if (deprMatch)
509 								{
510 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
511 										deprMatch[1], ErrorType.Deprecation,
512 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
513 									// TODO: maybe add special type or output
514 								}
515 							}
516 						}
517 					}
518 				};
519 				try
520 				{
521 					import workspaced.dub.lintgenerator : DubLintGenerator;
522 
523 					new DubLintGenerator(_dub.project).generate(settings);
524 				}
525 				catch (Exception e)
526 				{
527 					if (!e.msg.matchFirst(harmlessExceptionFormat))
528 						throw e;
529 				}
530 				ret.finish(issues.data);
531 			}
532 			catch (Throwable t)
533 			{
534 				ret.error(t);
535 			}
536 		}).start();
537 		return ret;
538 	}
539 
540 	/// Converts the root package recipe to another format.
541 	/// Params:
542 	///     format = either "json" or "sdl".
543 	string convertRecipe(string format)
544 	{
545 		import dub.recipe.io : serializePackageRecipe;
546 		import std.array : appender;
547 
548 		auto dst = appender!string;
549 		serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format);
550 		return dst.data;
551 	}
552 
553 private:
554 	Dub _dub;
555 	bool _dubRunning = false;
556 	string _configuration;
557 	string _archType = "x86_64";
558 	string _buildType = "debug";
559 	string _compilerBinaryName;
560 	Compiler _compiler;
561 	BuildSettingsTemplate _settingsTemplate;
562 	BuildSettings _settings;
563 	BuildPlatform _platform;
564 	string[] _importPaths, _stringImportPaths, _importFiles;
565 }
566 
567 ///
568 enum ErrorType : ubyte
569 {
570 	///
571 	Error = 0,
572 	///
573 	Warning = 1,
574 	///
575 	Deprecation = 2
576 }
577 
578 /// Returned by build
579 struct BuildIssue
580 {
581 	///
582 	int line, column;
583 	///
584 	string file;
585 	/// The error type (Error/Warning/Deprecation) outputted by dmd or inherited from the last error if this is additional information of the last issue. (indicated by cont)
586 	ErrorType type;
587 	///
588 	string text;
589 	/// true if this is additional error information for the last error.
590 	bool cont;
591 }
592 
593 private:
594 
595 T toOr(T)(string s, T defaultValue)
596 {
597 	if (!s || !s.length)
598 		return defaultValue;
599 	return s.to!T;
600 }
601 
602 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
603 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
604 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\):[ ]{6,}(.*)`, "g");
605 enum deprecationFormat = ctRegex!(
606 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
607 
608 struct DubPackageInfo
609 {
610 	string[string] dependencies;
611 	string ver;
612 	string name;
613 	string path;
614 	string description;
615 	string homepage;
616 	const(string)[] authors;
617 	string copyright;
618 	string license;
619 	DubPackageInfo[] subPackages;
620 
621 	void fill(in PackageRecipe recipe)
622 	{
623 		description = recipe.description;
624 		homepage = recipe.homepage;
625 		authors = recipe.authors;
626 		copyright = recipe.copyright;
627 		license = recipe.license;
628 
629 		foreach (subpackage; recipe.subPackages)
630 		{
631 			DubPackageInfo info;
632 			info.ver = subpackage.recipe.version_;
633 			info.name = subpackage.recipe.name;
634 			info.path = subpackage.path;
635 			info.fill(subpackage.recipe);
636 		}
637 	}
638 }
639 
640 DubPackageInfo getInfo(in Package dep)
641 {
642 	DubPackageInfo info;
643 	info.name = dep.name;
644 	info.ver = dep.version_.toString;
645 	info.path = dep.path.toString;
646 	info.fill(dep.recipe);
647 	foreach (subDep; dep.getAllDependencies())
648 	{
649 		info.dependencies[subDep.name] = subDep.spec.toString;
650 	}
651 	return info;
652 }
653 
654 auto listDependencies(scope const Project project)
655 {
656 	auto deps = project.dependencies;
657 	DubPackageInfo[] dependencies;
658 	if (deps is null)
659 		return dependencies;
660 	foreach (dep; deps)
661 	{
662 		dependencies ~= getInfo(dep);
663 	}
664 	return dependencies;
665 }
666 
667 string[] listDependencies(scope const Package pkg)
668 {
669 	auto deps = pkg.getAllDependencies();
670 	string[] dependencies;
671 	if (deps is null)
672 		return dependencies;
673 	foreach (dep; deps)
674 		dependencies ~= dep.name;
675 	return dependencies;
676 }