1 module workspaced.com.dub;
2 
3 import core.sync.mutex;
4 import core.exception;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.conv;
9 import std.exception;
10 import std.json : JSONValue, JSON_TYPE;
11 import std.parallelism;
12 import std.regex;
13 import std.stdio;
14 import std.string;
15 
16 import painlessjson : toJSON, fromJSON;
17 
18 import workspaced.api;
19 
20 import dub.description;
21 import dub.dub;
22 import dub.package_;
23 import dub.project;
24 
25 import dub.compilers.compiler;
26 import dub.generators.build;
27 import dub.generators.generator;
28 
29 import dub.compilers.buildsettings;
30 
31 import dub.internal.vibecompat.core.log;
32 import dub.internal.vibecompat.inet.url;
33 
34 @component("dub")
35 class DubComponent : ComponentWrapper
36 {
37 	mixin DefaultComponentWrapper;
38 
39 	static void registered()
40 	{
41 		setLogLevel(LogLevel.none);
42 	}
43 
44 	protected void load()
45 	{
46 		if (!refInstance)
47 			throw new Exception("dub requires to be instanced");
48 
49 		if (config.get!bool("dub", "registerImportProvider", true))
50 			importPathProvider = &imports;
51 		if (config.get!bool("dub", "registerStringImportProvider", true))
52 			stringImportPathProvider = &stringImports;
53 		if (config.get!bool("dub", "registerImportFilesProvider", false))
54 			importFilesProvider = &fileImports;
55 
56 		try
57 		{
58 			start();
59 
60 			_configuration = _dub.project.getDefaultConfiguration(_platform);
61 			if (!_dub.project.configurations.canFind(_configuration))
62 			{
63 				stderr.writeln("Dub Error: No configuration available");
64 				workspaced.broadcast(refInstance, JSONValue([
65 							"type": JSONValue("warning"),
66 							"component": JSONValue("dub"),
67 							"detail": JSONValue("invalid-default-config")
68 						]));
69 			}
70 			else
71 				updateImportPaths(false);
72 		}
73 		catch (Exception e)
74 		{
75 			if (!_dub || !_dub.project)
76 				throw e;
77 			stderr.writeln("Dub Error (ignored): ", e);
78 		}
79 		/*catch (AssertError e)
80 		{
81 			if (!_dub || !_dub.project)
82 				throw e;
83 			stderr.writeln("Dub Error (ignored): ", e);
84 		}*/
85 	}
86 
87 	private void start()
88 	{
89 		_dubRunning = false;
90 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
91 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
92 		_dub.loadPackage();
93 		_dub.project.validate();
94 
95 		// mark all packages as optional so we don't crash
96 		int missingPackages;
97 		auto optionalified = optionalifyPackages;
98 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
99 		{
100 			optionalifyRecipe(pkg);
101 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
102 			{
103 				auto d = _dub.project.getDependency(dep.name, true);
104 				if (!d)
105 					missingPackages++;
106 				else
107 					optionalifyRecipe(d);
108 			}
109 		}
110 
111 		if (!_compilerBinaryName.length)
112 			_compilerBinaryName = _dub.defaultCompiler;
113 		setCompiler(_compilerBinaryName);
114 
115 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings();
116 
117 		if (missingPackages > 0)
118 		{
119 			upgrade(false);
120 			optionalifyPackages();
121 		}
122 
123 		_dubRunning = true;
124 	}
125 
126 	private string[] optionalifyPackages()
127 	{
128 		bool[Package] visited;
129 		string[] optionalified;
130 		foreach (pkg; _dub.project.dependencies)
131 			optionalified ~= optionalifyRecipe(cast() pkg);
132 		return optionalified;
133 	}
134 
135 	private string[] optionalifyRecipe(Package pkg)
136 	{
137 		string[] optionalified;
138 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
139 		{
140 			if (!value.optional)
141 			{
142 				value.optional = true;
143 				value.default_ = true;
144 				optionalified ~= key;
145 			}
146 		}
147 		foreach (ref config; pkg.recipe.configurations)
148 			foreach (key, ref value; config.buildSettings.dependencies)
149 			{
150 				if (!value.optional)
151 				{
152 					value.optional = true;
153 					value.default_ = true;
154 					optionalified ~= key;
155 				}
156 			}
157 		return optionalified;
158 	}
159 
160 	private void restart()
161 	{
162 		_dub.destroy();
163 		_dubRunning = false;
164 		start();
165 	}
166 
167 	bool isRunning()
168 	{
169 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
170 			&& _dubRunning;
171 	}
172 
173 	/// Reloads the dub.json or dub.sdl file from the cwd
174 	/// Returns: `false` if there are no import paths available
175 	Future!bool update()
176 	{
177 		restart();
178 		auto ret = new Future!bool;
179 		gthreads.create({
180 			mixin(traceTask);
181 			try
182 			{
183 				auto result = updateImportPaths(false);
184 				ret.finish(result);
185 			}
186 			catch (Throwable t)
187 			{
188 				ret.error(t);
189 			}
190 		});
191 		return ret;
192 	}
193 
194 	bool updateImportPaths(bool restartDub = true)
195 	{
196 		validateConfiguration();
197 
198 		if (restartDub)
199 			restart();
200 
201 		GeneratorSettings settings;
202 		settings.platform = _platform;
203 		settings.config = _configuration;
204 		settings.buildType = _buildType;
205 		settings.compiler = _compiler;
206 		settings.buildSettings = _settings;
207 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
208 		settings.combined = true;
209 		settings.run = false;
210 
211 		try
212 		{
213 			auto paths = _dub.project.listBuildSettings(settings, ["import-paths",
214 					"string-import-paths", "source-files"], ListBuildSettingsFormat.listNul);
215 			_importPaths = paths[0].split('\0');
216 			_stringImportPaths = paths[1].split('\0');
217 			_importFiles = paths[2].split('\0');
218 			return _importPaths.length > 0 || _importFiles.length > 0;
219 		}
220 		catch (Exception e)
221 		{
222 			workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("error"), "component"
223 					: JSONValue("dub"), "detail"
224 					: JSONValue("Error while listing import paths: " ~ e.toString)]));
225 			_importPaths = [];
226 			_stringImportPaths = [];
227 			return false;
228 		}
229 	}
230 
231 	/// Calls `dub upgrade`
232 	void upgrade(bool save = true)
233 	{
234 		if (save)
235 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
236 		else
237 			_dub.upgrade(UpgradeOptions.noSaveSelections);
238 	}
239 
240 	/// Throws if configuration is invalid, otherwise does nothing.
241 	void validateConfiguration()
242 	{
243 		if (!_dub.project.configurations.canFind(_configuration))
244 			throw new Exception("Cannot use dub with invalid configuration");
245 	}
246 
247 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
248 	void validateBuildConfiguration()
249 	{
250 		if (!_dub.project.configurations.canFind(_configuration))
251 			throw new Exception("Cannot use dub with invalid configuration");
252 		if (_settings.targetType == TargetType.none)
253 			throw new Exception("Cannot build with dub with targetType == none");
254 		if (_settings.targetType == TargetType.sourceLibrary)
255 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
256 	}
257 
258 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
259 	/// Returns: `[{dependencies: [string], ver: string, name: string}]`
260 	auto dependencies() @property
261 	{
262 		validateConfiguration();
263 
264 		return _dub.project.listDependencies();
265 	}
266 
267 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
268 	string[] rootDependencies() @property
269 	{
270 		validateConfiguration();
271 
272 		return _dub.project.rootPackage.listDependencies();
273 	}
274 
275 	/// Returns the path to the root package recipe (dub.json/dub.sdl)
276 	///
277 	/// Note that this can be empty if the package is not in the local file system.
278 	string recipePath() @property
279 	{
280 		return _dub.project.rootPackage.recipePath.toString;
281 	}
282 
283 	/// Lists all import paths
284 	string[] imports() @property nothrow
285 	{
286 		return _importPaths;
287 	}
288 
289 	/// Lists all string import paths
290 	string[] stringImports() @property nothrow
291 	{
292 		return _stringImportPaths;
293 	}
294 
295 	/// Lists all import paths to files
296 	string[] fileImports() @property nothrow
297 	{
298 		return _importFiles;
299 	}
300 
301 	/// Lists all configurations defined in the package description
302 	string[] configurations() @property
303 	{
304 		return _dub.project.configurations;
305 	}
306 
307 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
308 	string[] buildTypes() @property
309 	{
310 		string[] types = [
311 			"plain", "debug", "release", "release-debug", "release-nobounds",
312 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
313 		];
314 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
315 			types ~= type;
316 		return types;
317 	}
318 
319 	/// Gets the current selected configuration
320 	string configuration() @property
321 	{
322 		return _configuration;
323 	}
324 
325 	/// Selects a new configuration and updates the import paths accordingly
326 	/// Returns: `false` if there are no import paths in the new configuration
327 	bool setConfiguration(string configuration)
328 	{
329 		if (!_dub.project.configurations.canFind(configuration))
330 			return false;
331 		_configuration = configuration;
332 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings(configuration);
333 		return updateImportPaths(false);
334 	}
335 
336 	/// List all possible arch types for current set compiler
337 	string[] archTypes() @property
338 	{
339 		string[] types = ["x86_64", "x86"];
340 
341 		string compilerName = _compiler.name;
342 
343 		version (Windows)
344 		{
345 			if (compilerName == "dmd")
346 			{
347 				types ~= "x86_mscoff";
348 			}
349 		}
350 		if (compilerName == "gdc")
351 		{
352 			types ~= ["arm", "arm_thumb"];
353 		}
354 
355 		return types;
356 	}
357 
358 	/// Returns the current selected arch type
359 	string archType() @property
360 	{
361 		return _archType;
362 	}
363 
364 	/// Selects a new arch type and updates the import paths accordingly
365 	/// Returns: `false` if there are no import paths in the new arch type
366 	bool setArchType(JSONValue request)
367 	{
368 		enforce(request.type == JSON_TYPE.OBJECT && "arch-type" in request, "arch-type not in request");
369 		auto type = request["arch-type"].fromJSON!string;
370 		if (archTypes.canFind(type))
371 		{
372 			_archType = type;
373 			return updateImportPaths(false);
374 		}
375 		else
376 		{
377 			return false;
378 		}
379 	}
380 
381 	/// Returns the current selected build type
382 	string buildType() @property
383 	{
384 		return _buildType;
385 	}
386 
387 	/// Selects a new build type and updates the import paths accordingly
388 	/// Returns: `false` if there are no import paths in the new build type
389 	bool setBuildType(JSONValue request)
390 	{
391 		enforce(request.type == JSON_TYPE.OBJECT && "build-type" in request,
392 				"build-type not in request");
393 		auto type = request["build-type"].fromJSON!string;
394 		if (buildTypes.canFind(type))
395 		{
396 			_buildType = type;
397 			return updateImportPaths(false);
398 		}
399 		else
400 		{
401 			return false;
402 		}
403 	}
404 
405 	/// Returns the current selected compiler
406 	string compiler() @property
407 	{
408 		return _compilerBinaryName;
409 	}
410 
411 	/// Selects a new compiler for building
412 	/// Returns: `false` if the compiler does not exist
413 	bool setCompiler(string compiler)
414 	{
415 		try
416 		{
417 			_compilerBinaryName = compiler;
418 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
419 		}
420 		catch (Exception e)
421 		{
422 			return false;
423 		}
424 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
425 		_settingsTemplate.getPlatformSettings(_settings, _platform, _dub.project.rootPackage.path);
426 		return _compiler !is null;
427 	}
428 
429 	/// Returns the project name
430 	string name() @property
431 	{
432 		return _dub.projectName;
433 	}
434 
435 	/// Returns the project path
436 	auto path() @property
437 	{
438 		return _dub.projectPath;
439 	}
440 
441 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
442 	bool canBuild() @property
443 	{
444 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
445 				|| !_dub.project.configurations.canFind(_configuration))
446 			return false;
447 		return true;
448 	}
449 
450 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
451 	Future!(BuildIssue[]) build()
452 	{
453 		import std.process : thisProcessID;
454 		import std.file : tempDir;
455 		import std.random : uniform;
456 
457 		validateBuildConfiguration();
458 
459 		// copy to this thread
460 		auto compiler = _compiler;
461 		auto buildPlatform = _platform;
462 
463 		GeneratorSettings settings;
464 		settings.platform = buildPlatform;
465 		settings.config = _configuration;
466 		settings.buildType = _buildType;
467 		settings.compiler = compiler;
468 		settings.tempBuild = true;
469 		settings.buildSettings = _settings;
470 		settings.buildSettings.targetPath = tempDir;
471 		settings.buildSettings.targetName = "workspace-d-build-"
472 			~ thisProcessID.to!string ~ "-" ~ uniform!uint.to!string(36);
473 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
474 		settings.buildSettings.addDFlags("-o-");
475 
476 		auto ret = new Future!(BuildIssue[]);
477 		new Thread({
478 			try
479 			{
480 				BuildIssue[] issues;
481 
482 				settings.compileCallback = (status, output) {
483 					string[] lines = output.splitLines;
484 					foreach (line; lines)
485 					{
486 						auto match = line.matchFirst(errorFormat);
487 						if (match)
488 						{
489 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
490 								match[1], match[4].to!ErrorType, match[5]);
491 						}
492 						else
493 						{
494 							if (line.canFind("from"))
495 							{
496 								auto contMatch = line.matchFirst(errorFormatCont);
497 								if (contMatch)
498 								{
499 									issues ~= BuildIssue(contMatch[2].to!int,
500 										contMatch[3].toOr!int(1), contMatch[1], ErrorType.Error, contMatch[4]);
501 								}
502 							}
503 							if (line.canFind("is deprecated"))
504 							{
505 								auto deprMatch = line.matchFirst(deprecationFormat);
506 								if (deprMatch)
507 								{
508 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
509 										deprMatch[1], ErrorType.Deprecation,
510 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
511 									// TODO: maybe add special type or output
512 								}
513 							}
514 						}
515 					}
516 				};
517 				try
518 				{
519 					_dub.generateProject("build", settings);
520 				}
521 				catch (Exception e)
522 				{
523 					if (!e.msg.matchFirst(harmlessExceptionFormat))
524 						throw e;
525 				}
526 				ret.finish(issues);
527 			}
528 			catch (Throwable t)
529 			{
530 				ret.error(t);
531 			}
532 		}).start();
533 		return ret;
534 	}
535 
536 	/// Converts the root package recipe to another format.
537 	/// Params:
538 	///     format = either "json" or "sdl".
539 	string convertRecipe(string format)
540 	{
541 		import dub.recipe.io : serializePackageRecipe;
542 		import std.array : appender;
543 
544 		auto dst = appender!string;
545 		serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format);
546 		return dst.data;
547 	}
548 
549 private:
550 	Dub _dub;
551 	bool _dubRunning = false;
552 	string _configuration;
553 	string _archType = "x86_64";
554 	string _buildType = "debug";
555 	string _compilerBinaryName;
556 	Compiler _compiler;
557 	BuildSettingsTemplate _settingsTemplate;
558 	BuildSettings _settings;
559 	BuildPlatform _platform;
560 	string[] _importPaths, _stringImportPaths, _importFiles;
561 }
562 
563 ///
564 enum ErrorType : ubyte
565 {
566 	///
567 	Error = 0,
568 	///
569 	Warning = 1,
570 	///
571 	Deprecation = 2
572 }
573 
574 /// Returned by build
575 struct BuildIssue
576 {
577 	///
578 	int line, column;
579 	///
580 	string file;
581 	///
582 	ErrorType type;
583 	///
584 	string text;
585 }
586 
587 private:
588 
589 T toOr(T)(string s, T defaultValue)
590 {
591 	if (!s || !s.length)
592 		return defaultValue;
593 	return s.to!T;
594 }
595 
596 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
597 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
598 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (.*)`, "g");
599 enum deprecationFormat = ctRegex!(
600 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
601 
602 struct DubPackageInfo
603 {
604 	string[string] dependencies;
605 	string ver;
606 	string name;
607 	string path;
608 	string description;
609 	string homepage;
610 	const(string)[] authors;
611 	string copyright;
612 	string license;
613 	DubPackageInfo[] subPackages;
614 
615 	void fill(in PackageRecipe recipe)
616 	{
617 		description = recipe.description;
618 		homepage = recipe.homepage;
619 		authors = recipe.authors;
620 		copyright = recipe.copyright;
621 		license = recipe.license;
622 
623 		foreach (subpackage; recipe.subPackages)
624 		{
625 			DubPackageInfo info;
626 			info.ver = subpackage.recipe.version_;
627 			info.name = subpackage.recipe.name;
628 			info.path = subpackage.path;
629 			info.fill(subpackage.recipe);
630 		}
631 	}
632 }
633 
634 DubPackageInfo getInfo(in Package dep)
635 {
636 	DubPackageInfo info;
637 	info.name = dep.name;
638 	info.ver = dep.version_.toString;
639 	info.path = dep.path.toString;
640 	info.fill(dep.recipe);
641 	foreach (subDep; dep.getAllDependencies())
642 	{
643 		info.dependencies[subDep.name] = subDep.spec.toString;
644 	}
645 	return info;
646 }
647 
648 auto listDependencies(Project project)
649 {
650 	auto deps = project.dependencies;
651 	DubPackageInfo[] dependencies;
652 	if (deps is null)
653 		return dependencies;
654 	foreach (dep; deps)
655 	{
656 		dependencies ~= getInfo(dep);
657 	}
658 	return dependencies;
659 }
660 
661 string[] listDependencies(Package pkg)
662 {
663 	auto deps = pkg.getAllDependencies();
664 	string[] dependencies;
665 	if (deps is null)
666 		return dependencies;
667 	foreach (dep; deps)
668 		dependencies ~= dep.name;
669 	return dependencies;
670 }