1 module workspaced.com.dub;
2 
3 import core.exception;
4 import core.sync.mutex;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.conv;
9 import std.exception;
10 import std.json : JSONType, JSONValue;
11 import std.parallelism;
12 import std.regex;
13 import std.stdio;
14 import std.string;
15 
16 import painlessjson : toJSON, fromJSON;
17 
18 import workspaced.api;
19 
20 import dub.description;
21 import dub.dub;
22 import dub.package_;
23 import dub.project;
24 
25 import dub.compilers.compiler;
26 import dub.generators.build;
27 import dub.generators.generator;
28 
29 import dub.compilers.buildsettings;
30 
31 import dub.internal.vibecompat.core.log;
32 import dub.internal.vibecompat.inet.url;
33 
34 @component("dub")
35 class DubComponent : ComponentWrapper
36 {
37 	mixin DefaultComponentWrapper;
38 
39 	static void registered()
40 	{
41 		setLogLevel(LogLevel.none);
42 	}
43 
44 	protected void load()
45 	{
46 		if (!refInstance)
47 			throw new Exception("dub requires to be instanced");
48 
49 		if (config.get!bool("dub", "registerImportProvider", true))
50 			importPathProvider = &imports;
51 		if (config.get!bool("dub", "registerStringImportProvider", true))
52 			stringImportPathProvider = &stringImports;
53 		if (config.get!bool("dub", "registerImportFilesProvider", false))
54 			importFilesProvider = &fileImports;
55 
56 		try
57 		{
58 			start();
59 
60 			_configuration = _dub.project.getDefaultConfiguration(_platform);
61 			if (!_dub.project.configurations.canFind(_configuration))
62 			{
63 				stderr.writeln("Dub Error: No configuration available");
64 				workspaced.broadcast(refInstance, JSONValue([
65 							"type": JSONValue("warning"),
66 							"component": JSONValue("dub"),
67 							"detail": JSONValue("invalid-default-config")
68 						]));
69 			}
70 			else
71 				updateImportPaths(false);
72 		}
73 		catch (Exception e)
74 		{
75 			if (!_dub || !_dub.project)
76 				throw e;
77 			stderr.writeln("Dub Error (ignored): ", e);
78 		}
79 		/*catch (AssertError e)
80 		{
81 			if (!_dub || !_dub.project)
82 				throw e;
83 			stderr.writeln("Dub Error (ignored): ", e);
84 		}*/
85 	}
86 
87 	private void start()
88 	{
89 		_dubRunning = false;
90 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
91 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
92 		_dub.loadPackage();
93 		_dub.project.validate();
94 
95 		// mark all packages as optional so we don't crash
96 		int missingPackages;
97 		auto optionalified = optionalifyPackages;
98 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
99 		{
100 			optionalifyRecipe(pkg);
101 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
102 			{
103 				auto d = _dub.project.getDependency(dep.name, true);
104 				if (!d)
105 					missingPackages++;
106 				else
107 					optionalifyRecipe(d);
108 			}
109 		}
110 
111 		if (!_compilerBinaryName.length)
112 			_compilerBinaryName = _dub.defaultCompiler;
113 		setCompiler(_compilerBinaryName);
114 
115 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings();
116 
117 		if (missingPackages > 0)
118 		{
119 			upgrade(false);
120 			optionalifyPackages();
121 		}
122 
123 		_dubRunning = true;
124 	}
125 
126 	private string[] optionalifyPackages()
127 	{
128 		bool[Package] visited;
129 		string[] optionalified;
130 		foreach (pkg; _dub.project.dependencies)
131 			optionalified ~= optionalifyRecipe(cast() pkg);
132 		return optionalified;
133 	}
134 
135 	private string[] optionalifyRecipe(Package pkg)
136 	{
137 		string[] optionalified;
138 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
139 		{
140 			if (!value.optional)
141 			{
142 				value.optional = true;
143 				value.default_ = true;
144 				optionalified ~= key;
145 			}
146 		}
147 		foreach (ref config; pkg.recipe.configurations)
148 			foreach (key, ref value; config.buildSettings.dependencies)
149 			{
150 				if (!value.optional)
151 				{
152 					value.optional = true;
153 					value.default_ = true;
154 					optionalified ~= key;
155 				}
156 			}
157 		return optionalified;
158 	}
159 
160 	private void restart()
161 	{
162 		_dub.destroy();
163 		_dubRunning = false;
164 		start();
165 	}
166 
167 	bool isRunning()
168 	{
169 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
170 			&& _dubRunning;
171 	}
172 
173 	/// Reloads the dub.json or dub.sdl file from the cwd
174 	/// Returns: `false` if there are no import paths available
175 	Future!bool update()
176 	{
177 		restart();
178 		mixin(gthreadsAsyncProxy!`updateImportPaths(false)`);
179 	}
180 
181 	bool updateImportPaths(bool restartDub = true)
182 	{
183 		validateConfiguration();
184 
185 		if (restartDub)
186 			restart();
187 
188 		GeneratorSettings settings;
189 		settings.platform = _platform;
190 		settings.config = _configuration;
191 		settings.buildType = _buildType;
192 		settings.compiler = _compiler;
193 		settings.buildSettings = _settings;
194 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
195 		settings.combined = true;
196 		settings.run = false;
197 
198 		try
199 		{
200 			auto paths = _dub.project.listBuildSettings(settings, [
201 					"import-paths", "string-import-paths", "source-files"
202 					], ListBuildSettingsFormat.listNul);
203 			_importPaths = paths[0].split('\0');
204 			_stringImportPaths = paths[1].split('\0');
205 			_importFiles = paths[2].split('\0');
206 			return _importPaths.length > 0 || _importFiles.length > 0;
207 		}
208 		catch (Exception e)
209 		{
210 			workspaced.broadcast(refInstance, JSONValue([
211 						"type": JSONValue("error"),
212 						"component": JSONValue("dub"),
213 						"detail": JSONValue("Error while listing import paths: " ~ e.toString)
214 					]));
215 			_importPaths = [];
216 			_stringImportPaths = [];
217 			return false;
218 		}
219 	}
220 
221 	/// Calls `dub upgrade`
222 	void upgrade(bool save = true)
223 	{
224 		if (save)
225 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
226 		else
227 			_dub.upgrade(UpgradeOptions.noSaveSelections);
228 	}
229 
230 	/// Throws if configuration is invalid, otherwise does nothing.
231 	void validateConfiguration()
232 	{
233 		if (!_dub.project.configurations.canFind(_configuration))
234 			throw new Exception("Cannot use dub with invalid configuration");
235 	}
236 
237 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
238 	void validateBuildConfiguration()
239 	{
240 		if (!_dub.project.configurations.canFind(_configuration))
241 			throw new Exception("Cannot use dub with invalid configuration");
242 		if (_settings.targetType == TargetType.none)
243 			throw new Exception("Cannot build with dub with targetType == none");
244 		if (_settings.targetType == TargetType.sourceLibrary)
245 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
246 	}
247 
248 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
249 	/// Returns: `[{dependencies: [string], ver: string, name: string}]`
250 	auto dependencies() @property
251 	{
252 		validateConfiguration();
253 
254 		return _dub.project.listDependencies();
255 	}
256 
257 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
258 	string[] rootDependencies() @property
259 	{
260 		validateConfiguration();
261 
262 		return _dub.project.rootPackage.listDependencies();
263 	}
264 
265 	/// Returns the path to the root package recipe (dub.json/dub.sdl)
266 	///
267 	/// Note that this can be empty if the package is not in the local file system.
268 	string recipePath() @property
269 	{
270 		return _dub.project.rootPackage.recipePath.toString;
271 	}
272 
273 	/// Lists all import paths
274 	string[] imports() @property nothrow
275 	{
276 		return _importPaths;
277 	}
278 
279 	/// Lists all string import paths
280 	string[] stringImports() @property nothrow
281 	{
282 		return _stringImportPaths;
283 	}
284 
285 	/// Lists all import paths to files
286 	string[] fileImports() @property nothrow
287 	{
288 		return _importFiles;
289 	}
290 
291 	/// Lists all configurations defined in the package description
292 	string[] configurations() @property
293 	{
294 		return _dub.project.configurations;
295 	}
296 
297 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
298 	string[] buildTypes() @property
299 	{
300 		string[] types = [
301 			"plain", "debug", "release", "release-debug", "release-nobounds",
302 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
303 		];
304 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
305 			types ~= type;
306 		return types;
307 	}
308 
309 	/// Gets the current selected configuration
310 	string configuration() @property
311 	{
312 		return _configuration;
313 	}
314 
315 	/// Selects a new configuration and updates the import paths accordingly
316 	/// Returns: `false` if there are no import paths in the new configuration
317 	bool setConfiguration(string configuration)
318 	{
319 		if (!_dub.project.configurations.canFind(configuration))
320 			return false;
321 		_configuration = configuration;
322 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings(configuration);
323 		return updateImportPaths(false);
324 	}
325 
326 	/// List all possible arch types for current set compiler
327 	string[] archTypes() @property
328 	{
329 		string[] types = ["x86_64", "x86"];
330 
331 		string compilerName = _compiler.name;
332 
333 		version (Windows)
334 		{
335 			if (compilerName == "dmd")
336 			{
337 				types ~= "x86_mscoff";
338 			}
339 		}
340 		if (compilerName == "gdc")
341 		{
342 			types ~= ["arm", "arm_thumb"];
343 		}
344 
345 		return types;
346 	}
347 
348 	/// Returns the current selected arch type
349 	string archType() @property
350 	{
351 		return _archType;
352 	}
353 
354 	/// Selects a new arch type and updates the import paths accordingly
355 	/// Returns: `false` if there are no import paths in the new arch type
356 	bool setArchType(JSONValue request)
357 	{
358 		enforce(request.type == JSONType.object && "arch-type" in request, "arch-type not in request");
359 		auto type = request["arch-type"].fromJSON!string;
360 		if (archTypes.canFind(type))
361 		{
362 			_archType = type;
363 			return updateImportPaths(false);
364 		}
365 		else
366 		{
367 			return false;
368 		}
369 	}
370 
371 	/// Returns the current selected build type
372 	string buildType() @property
373 	{
374 		return _buildType;
375 	}
376 
377 	/// Selects a new build type and updates the import paths accordingly
378 	/// Returns: `false` if there are no import paths in the new build type
379 	bool setBuildType(JSONValue request)
380 	{
381 		enforce(request.type == JSONType.object && "build-type" in request, "build-type not in request");
382 		auto type = request["build-type"].fromJSON!string;
383 		if (buildTypes.canFind(type))
384 		{
385 			_buildType = type;
386 			return updateImportPaths(false);
387 		}
388 		else
389 		{
390 			return false;
391 		}
392 	}
393 
394 	/// Returns the current selected compiler
395 	string compiler() @property
396 	{
397 		return _compilerBinaryName;
398 	}
399 
400 	/// Selects a new compiler for building
401 	/// Returns: `false` if the compiler does not exist
402 	bool setCompiler(string compiler)
403 	{
404 		try
405 		{
406 			_compilerBinaryName = compiler;
407 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
408 		}
409 		catch (Exception e)
410 		{
411 			return false;
412 		}
413 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
414 		_settingsTemplate.getPlatformSettings(_settings, _platform, _dub.project.rootPackage.path);
415 		return _compiler !is null;
416 	}
417 
418 	/// Returns the project name
419 	string name() @property
420 	{
421 		return _dub.projectName;
422 	}
423 
424 	/// Returns the project path
425 	auto path() @property
426 	{
427 		return _dub.projectPath;
428 	}
429 
430 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
431 	bool canBuild() @property
432 	{
433 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
434 				|| !_dub.project.configurations.canFind(_configuration))
435 			return false;
436 		return true;
437 	}
438 
439 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
440 	Future!(BuildIssue[]) build()
441 	{
442 		import std.process : thisProcessID;
443 		import std.file : tempDir;
444 		import std.random : uniform;
445 
446 		validateBuildConfiguration();
447 
448 		// copy to this thread
449 		auto compiler = _compiler;
450 		auto buildPlatform = _platform;
451 
452 		GeneratorSettings settings;
453 		settings.platform = buildPlatform;
454 		settings.config = _configuration;
455 		settings.buildType = _buildType;
456 		settings.compiler = compiler;
457 		settings.tempBuild = true;
458 		settings.buildSettings = _settings;
459 		settings.buildSettings.targetPath = tempDir;
460 		settings.buildSettings.targetName = "workspace-d-build-"
461 			~ thisProcessID.to!string ~ "-" ~ uniform!uint.to!string(36);
462 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
463 		settings.buildSettings.addDFlags("-o-");
464 
465 		auto ret = new Future!(BuildIssue[]);
466 		new Thread({
467 			try
468 			{
469 				BuildIssue[] issues;
470 
471 				settings.compileCallback = (status, output) {
472 					string[] lines = output.splitLines;
473 					foreach (line; lines)
474 					{
475 						auto match = line.matchFirst(errorFormat);
476 						if (match)
477 						{
478 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
479 								match[1], match[4].to!ErrorType, match[5]);
480 						}
481 						else
482 						{
483 							if (line.canFind("from"))
484 							{
485 								auto contMatch = line.matchFirst(errorFormatCont);
486 								if (contMatch)
487 								{
488 									issues ~= BuildIssue(contMatch[2].to!int,
489 										contMatch[3].toOr!int(1), contMatch[1], ErrorType.Error, contMatch[4]);
490 								}
491 							}
492 							if (line.canFind("is deprecated"))
493 							{
494 								auto deprMatch = line.matchFirst(deprecationFormat);
495 								if (deprMatch)
496 								{
497 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
498 										deprMatch[1], ErrorType.Deprecation,
499 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
500 									// TODO: maybe add special type or output
501 								}
502 							}
503 						}
504 					}
505 				};
506 				try
507 				{
508 					_dub.generateProject("build", settings);
509 				}
510 				catch (Exception e)
511 				{
512 					if (!e.msg.matchFirst(harmlessExceptionFormat))
513 						throw e;
514 				}
515 				ret.finish(issues);
516 			}
517 			catch (Throwable t)
518 			{
519 				ret.error(t);
520 			}
521 		}).start();
522 		return ret;
523 	}
524 
525 	/// Converts the root package recipe to another format.
526 	/// Params:
527 	///     format = either "json" or "sdl".
528 	string convertRecipe(string format)
529 	{
530 		import dub.recipe.io : serializePackageRecipe;
531 		import std.array : appender;
532 
533 		auto dst = appender!string;
534 		serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format);
535 		return dst.data;
536 	}
537 
538 private:
539 	Dub _dub;
540 	bool _dubRunning = false;
541 	string _configuration;
542 	string _archType = "x86_64";
543 	string _buildType = "debug";
544 	string _compilerBinaryName;
545 	Compiler _compiler;
546 	BuildSettingsTemplate _settingsTemplate;
547 	BuildSettings _settings;
548 	BuildPlatform _platform;
549 	string[] _importPaths, _stringImportPaths, _importFiles;
550 }
551 
552 ///
553 enum ErrorType : ubyte
554 {
555 	///
556 	Error = 0,
557 	///
558 	Warning = 1,
559 	///
560 	Deprecation = 2
561 }
562 
563 /// Returned by build
564 struct BuildIssue
565 {
566 	///
567 	int line, column;
568 	///
569 	string file;
570 	///
571 	ErrorType type;
572 	///
573 	string text;
574 }
575 
576 private:
577 
578 T toOr(T)(string s, T defaultValue)
579 {
580 	if (!s || !s.length)
581 		return defaultValue;
582 	return s.to!T;
583 }
584 
585 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
586 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
587 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (.*)`, "g");
588 enum deprecationFormat = ctRegex!(
589 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
590 
591 struct DubPackageInfo
592 {
593 	string[string] dependencies;
594 	string ver;
595 	string name;
596 	string path;
597 	string description;
598 	string homepage;
599 	const(string)[] authors;
600 	string copyright;
601 	string license;
602 	DubPackageInfo[] subPackages;
603 
604 	void fill(in PackageRecipe recipe)
605 	{
606 		description = recipe.description;
607 		homepage = recipe.homepage;
608 		authors = recipe.authors;
609 		copyright = recipe.copyright;
610 		license = recipe.license;
611 
612 		foreach (subpackage; recipe.subPackages)
613 		{
614 			DubPackageInfo info;
615 			info.ver = subpackage.recipe.version_;
616 			info.name = subpackage.recipe.name;
617 			info.path = subpackage.path;
618 			info.fill(subpackage.recipe);
619 		}
620 	}
621 }
622 
623 DubPackageInfo getInfo(in Package dep)
624 {
625 	DubPackageInfo info;
626 	info.name = dep.name;
627 	info.ver = dep.version_.toString;
628 	info.path = dep.path.toString;
629 	info.fill(dep.recipe);
630 	foreach (subDep; dep.getAllDependencies())
631 	{
632 		info.dependencies[subDep.name] = subDep.spec.toString;
633 	}
634 	return info;
635 }
636 
637 auto listDependencies(Project project)
638 {
639 	auto deps = project.dependencies;
640 	DubPackageInfo[] dependencies;
641 	if (deps is null)
642 		return dependencies;
643 	foreach (dep; deps)
644 	{
645 		dependencies ~= getInfo(dep);
646 	}
647 	return dependencies;
648 }
649 
650 string[] listDependencies(Package pkg)
651 {
652 	auto deps = pkg.getAllDependencies();
653 	string[] dependencies;
654 	if (deps is null)
655 		return dependencies;
656 	foreach (dep; deps)
657 		dependencies ~= dep.name;
658 	return dependencies;
659 }