1 module workspaced.com.dub;
2 
3 import core.sync.mutex;
4 import core.exception;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.conv;
9 import std.exception;
10 import std.json : JSONValue, JSON_TYPE;
11 import std.parallelism;
12 import std.regex;
13 import std.stdio;
14 import std.string;
15 
16 import painlessjson : toJSON, fromJSON;
17 
18 import workspaced.api;
19 
20 import dub.description;
21 import dub.dub;
22 import dub.package_;
23 import dub.project;
24 
25 import dub.compilers.compiler;
26 import dub.generators.build;
27 import dub.generators.generator;
28 
29 import dub.compilers.buildsettings;
30 
31 import dub.internal.vibecompat.core.log;
32 import dub.internal.vibecompat.inet.url;
33 
34 @component("dub")
35 class DubComponent : ComponentWrapper
36 {
37 	mixin DefaultComponentWrapper;
38 
39 	static void registered()
40 	{
41 		setLogLevel(LogLevel.none);
42 	}
43 
44 	protected void load()
45 	{
46 		if (!refInstance)
47 			throw new Exception("dub requires to be instanced");
48 
49 		if (config.get!bool("dub", "registerImportProvider", true))
50 			importPathProvider = &imports;
51 		if (config.get!bool("dub", "registerStringImportProvider", true))
52 			stringImportPathProvider = &stringImports;
53 		if (config.get!bool("dub", "registerImportFilesProvider", false))
54 			importFilesProvider = &fileImports;
55 
56 		try
57 		{
58 			start();
59 
60 			_configuration = _dub.project.getDefaultConfiguration(_platform);
61 			if (!_dub.project.configurations.canFind(_configuration))
62 			{
63 				stderr.writeln("Dub Error: No configuration available");
64 				workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("warning"),
65 						"component" : JSONValue("dub"), "detail" : JSONValue("invalid-default-config")]));
66 			}
67 			else
68 				updateImportPaths(false);
69 		}
70 		catch (Exception e)
71 		{
72 			if (!_dub || !_dub.project)
73 				throw e;
74 			stderr.writeln("Dub Error (ignored): ", e);
75 		}
76 		/*catch (AssertError e)
77 		{
78 			if (!_dub || !_dub.project)
79 				throw e;
80 			stderr.writeln("Dub Error (ignored): ", e);
81 		}*/
82 	}
83 
84 	private void start()
85 	{
86 		_dubRunning = false;
87 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
88 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
89 		_dub.loadPackage();
90 		_dub.project.validate();
91 
92 		// mark all packages as optional so we don't crash
93 		int missingPackages;
94 		auto optionalified = optionalifyPackages;
95 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
96 		{
97 			optionalifyRecipe(pkg);
98 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
99 			{
100 				auto d = _dub.project.getDependency(dep.name, true);
101 				if (!d)
102 					missingPackages++;
103 				else
104 					optionalifyRecipe(d);
105 			}
106 		}
107 
108 		if (!_compilerBinaryName.length)
109 			_compilerBinaryName = _dub.defaultCompiler;
110 		setCompiler(_compilerBinaryName);
111 		if (missingPackages > 0)
112 		{
113 			upgrade(false);
114 			optionalifyPackages();
115 		}
116 
117 		_dubRunning = true;
118 	}
119 
120 	private string[] optionalifyPackages()
121 	{
122 		bool[Package] visited;
123 		string[] optionalified;
124 		foreach (pkg; _dub.project.dependencies)
125 			optionalified ~= optionalifyRecipe(cast() pkg);
126 		return optionalified;
127 	}
128 
129 	private string[] optionalifyRecipe(Package pkg)
130 	{
131 		string[] optionalified;
132 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
133 		{
134 			if (!value.optional)
135 			{
136 				value.optional = true;
137 				value.default_ = true;
138 				optionalified ~= key;
139 			}
140 		}
141 		foreach (ref config; pkg.recipe.configurations)
142 			foreach (key, ref value; config.buildSettings.dependencies)
143 			{
144 				if (!value.optional)
145 				{
146 					value.optional = true;
147 					value.default_ = true;
148 					optionalified ~= key;
149 				}
150 			}
151 		return optionalified;
152 	}
153 
154 	private void restart()
155 	{
156 		_dub.destroy();
157 		_dubRunning = false;
158 		start();
159 	}
160 
161 	bool isRunning()
162 	{
163 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
164 			&& _dubRunning;
165 	}
166 
167 	/// Reloads the dub.json or dub.sdl file from the cwd
168 	/// Returns: `false` if there are no import paths available
169 	Future!bool update()
170 	{
171 		restart();
172 		auto ret = new Future!bool;
173 		new Thread({ /**/
174 			try
175 			{
176 				auto result = updateImportPaths(false);
177 				ret.finish(result);
178 			}
179 			catch (Throwable t)
180 			{
181 				ret.error(t);
182 			}
183 		}).start();
184 		return ret;
185 	}
186 
187 	bool updateImportPaths(bool restartDub = true)
188 	{
189 		validateConfiguration();
190 
191 		if (restartDub)
192 			restart();
193 
194 		GeneratorSettings settings;
195 		settings.platform = _platform;
196 		settings.config = _configuration;
197 		settings.buildType = _buildType;
198 		settings.compiler = _compiler;
199 		settings.buildSettings = _settings;
200 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
201 		settings.combined = true;
202 		settings.run = false;
203 
204 		try
205 		{
206 			auto paths = _dub.project.listBuildSettings(settings, ["import-paths",
207 					"string-import-paths", "source-files"], ListBuildSettingsFormat.listNul);
208 			_importPaths = paths[0].split('\0');
209 			_stringImportPaths = paths[1].split('\0');
210 			_importFiles = paths[2].split('\0');
211 			return _importPaths.length > 0 || _importFiles.length > 0;
212 		}
213 		catch (Exception e)
214 		{
215 			workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("error"), "component"
216 					: JSONValue("dub"), "detail"
217 					: JSONValue("Error while listing import paths: " ~ e.toString)]));
218 			_importPaths = [];
219 			_stringImportPaths = [];
220 			return false;
221 		}
222 	}
223 
224 	/// Calls `dub upgrade`
225 	void upgrade(bool save = true)
226 	{
227 		if (save)
228 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
229 		else
230 			_dub.upgrade(UpgradeOptions.noSaveSelections);
231 	}
232 
233 	/// Throws if configuration is invalid, otherwise does nothing.
234 	void validateConfiguration()
235 	{
236 		if (!_dub.project.configurations.canFind(_configuration))
237 			throw new Exception("Cannot use dub with invalid configuration");
238 	}
239 
240 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
241 	void validateBuildConfiguration()
242 	{
243 		if (!_dub.project.configurations.canFind(_configuration))
244 			throw new Exception("Cannot use dub with invalid configuration");
245 		if (_settings.targetType == TargetType.none)
246 			throw new Exception("Cannot build with dub with targetType == none");
247 		if (_settings.targetType == TargetType.sourceLibrary)
248 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
249 	}
250 
251 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
252 	/// Returns: `[{dependencies: [string], ver: string, name: string}]`
253 	auto dependencies() @property
254 	{
255 		validateConfiguration();
256 
257 		return _dub.project.listDependencies();
258 	}
259 
260 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
261 	/// Returns: `[string]`
262 	auto rootDependencies() @property
263 	{
264 		validateConfiguration();
265 
266 		return _dub.project.rootPackage.listDependencies();
267 	}
268 
269 	/// Lists all import paths
270 	string[] imports() @property
271 	{
272 		return _importPaths;
273 	}
274 
275 	/// Lists all string import paths
276 	string[] stringImports() @property
277 	{
278 		return _stringImportPaths;
279 	}
280 
281 	/// Lists all import paths to files
282 	string[] fileImports() @property
283 	{
284 		return _importFiles;
285 	}
286 
287 	/// Lists all configurations defined in the package description
288 	string[] configurations() @property
289 	{
290 		return _dub.project.configurations;
291 	}
292 
293 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
294 	string[] buildTypes() @property
295 	{
296 		string[] types = [
297 			"plain", "debug", "release", "release-debug", "release-nobounds",
298 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
299 		];
300 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
301 			types ~= type;
302 		return types;
303 	}
304 
305 	/// Gets the current selected configuration
306 	string configuration() @property
307 	{
308 		return _configuration;
309 	}
310 
311 	/// Selects a new configuration and updates the import paths accordingly
312 	/// Returns: `false` if there are no import paths in the new configuration
313 	bool setConfiguration(string configuration)
314 	{
315 		if (!_dub.project.configurations.canFind(configuration))
316 			return false;
317 		_configuration = configuration;
318 		return updateImportPaths(false);
319 	}
320 
321 	/// List all possible arch types for current set compiler
322 	string[] archTypes() @property
323 	{
324 		string[] types = ["x86_64", "x86"];
325 
326 		string compilerName = _compiler.name;
327 
328 		version (Windows)
329 		{
330 			if (compilerName == "dmd")
331 			{
332 				types ~= "x86_mscoff";
333 			}
334 		}
335 		if (compilerName == "gdc")
336 		{
337 			types ~= ["arm", "arm_thumb"];
338 		}
339 
340 		return types;
341 	}
342 
343 	/// Returns the current selected arch type
344 	string archType() @property
345 	{
346 		return _archType;
347 	}
348 
349 	/// Selects a new arch type and updates the import paths accordingly
350 	/// Returns: `false` if there are no import paths in the new arch type
351 	bool setArchType(JSONValue request)
352 	{
353 		enforce(request.type == JSON_TYPE.OBJECT && "arch-type" in request, "arch-type not in request");
354 		auto type = request["arch-type"].fromJSON!string;
355 		if (archTypes.canFind(type))
356 		{
357 			_archType = type;
358 			return updateImportPaths(false);
359 		}
360 		else
361 		{
362 			return false;
363 		}
364 	}
365 
366 	/// Returns the current selected build type
367 	string buildType() @property
368 	{
369 		return _buildType;
370 	}
371 
372 	/// Selects a new build type and updates the import paths accordingly
373 	/// Returns: `false` if there are no import paths in the new build type
374 	bool setBuildType(JSONValue request)
375 	{
376 		enforce(request.type == JSON_TYPE.OBJECT && "build-type" in request,
377 				"build-type not in request");
378 		auto type = request["build-type"].fromJSON!string;
379 		if (buildTypes.canFind(type))
380 		{
381 			_buildType = type;
382 			return updateImportPaths(false);
383 		}
384 		else
385 		{
386 			return false;
387 		}
388 	}
389 
390 	/// Returns the current selected compiler
391 	string compiler() @property
392 	{
393 		return _compilerBinaryName;
394 	}
395 
396 	/// Selects a new compiler for building
397 	/// Returns: `false` if the compiler does not exist
398 	bool setCompiler(string compiler)
399 	{
400 		try
401 		{
402 			_compilerBinaryName = compiler;
403 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
404 		}
405 		catch (Exception e)
406 		{
407 			return false;
408 		}
409 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
410 		return true;
411 	}
412 
413 	/// Returns the project name
414 	string name() @property
415 	{
416 		return _dub.projectName;
417 	}
418 
419 	/// Returns the project path
420 	auto path() @property
421 	{
422 		return _dub.projectPath;
423 	}
424 
425 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
426 	bool canBuild() @property
427 	{
428 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
429 				|| !_dub.project.configurations.canFind(_configuration))
430 			return false;
431 		return true;
432 	}
433 
434 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
435 	Future!(BuildIssue[]) build()
436 	{
437 		validateBuildConfiguration();
438 
439 		// copy to this thread
440 		auto compiler = _compiler;
441 		auto buildPlatform = _platform;
442 
443 		GeneratorSettings settings;
444 		settings.platform = buildPlatform;
445 		settings.config = _configuration;
446 		settings.buildType = _buildType;
447 		settings.compiler = compiler;
448 		settings.tempBuild = true;
449 		settings.buildSettings = _settings;
450 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
451 		settings.buildSettings.addDFlags("-o-");
452 
453 		auto ret = new Future!(BuildIssue[]);
454 		new Thread({
455 			try
456 			{
457 				BuildIssue[] issues;
458 
459 				settings.compileCallback = (status, output) {
460 					string[] lines = output.splitLines;
461 					foreach (line; lines)
462 					{
463 						auto match = line.matchFirst(errorFormat);
464 						if (match)
465 						{
466 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
467 								match[1], match[4].to!ErrorType, match[5]);
468 						}
469 						else
470 						{
471 							if (line.canFind("from"))
472 							{
473 								auto contMatch = line.matchFirst(errorFormatCont);
474 								if (contMatch)
475 								{
476 									issues ~= BuildIssue(contMatch[2].to!int,
477 										contMatch[3].toOr!int(1), contMatch[1], ErrorType.Error, contMatch[4]);
478 								}
479 							}
480 							if (line.canFind("is deprecated"))
481 							{
482 								auto deprMatch = line.matchFirst(deprecationFormat);
483 								if (deprMatch)
484 								{
485 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
486 										deprMatch[1], ErrorType.Deprecation,
487 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
488 									// TODO: maybe add special type or output
489 								}
490 							}
491 						}
492 					}
493 				};
494 				try
495 				{
496 					_dub.generateProject("build", settings);
497 				}
498 				catch (Exception e)
499 				{
500 					if (!e.msg.matchFirst(harmlessExceptionFormat))
501 						throw e;
502 				}
503 				ret.finish(issues);
504 			}
505 			catch (Throwable t)
506 			{
507 				ret.error(t);
508 			}
509 		}).start();
510 		return ret;
511 	}
512 
513 private:
514 	Dub _dub;
515 	bool _dubRunning = false;
516 	string _configuration;
517 	string _archType = "x86_64";
518 	string _buildType = "debug";
519 	string _compilerBinaryName;
520 	Compiler _compiler;
521 	BuildSettings _settings;
522 	BuildPlatform _platform;
523 	string[] _importPaths, _stringImportPaths, _importFiles;
524 }
525 
526 ///
527 enum ErrorType : ubyte
528 {
529 	///
530 	Error = 0,
531 	///
532 	Warning = 1,
533 	///
534 	Deprecation = 2
535 }
536 
537 /// Returned by build
538 struct BuildIssue
539 {
540 	///
541 	int line, column;
542 	///
543 	string file;
544 	///
545 	ErrorType type;
546 	///
547 	string text;
548 }
549 
550 private:
551 
552 T toOr(T)(string s, T defaultValue)
553 {
554 	if (!s || !s.length)
555 		return defaultValue;
556 	return s.to!T;
557 }
558 
559 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
560 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
561 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (.*)`, "g");
562 enum deprecationFormat = ctRegex!(
563 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
564 
565 struct DubPackageInfo
566 {
567 	string[string] dependencies;
568 	string ver;
569 	string name;
570 	string path;
571 	string description;
572 	string homepage;
573 	const(string)[] authors;
574 	string copyright;
575 	string license;
576 	DubPackageInfo[] subPackages;
577 
578 	void fill(in PackageRecipe recipe)
579 	{
580 		description = recipe.description;
581 		homepage = recipe.homepage;
582 		authors = recipe.authors;
583 		copyright = recipe.copyright;
584 		license = recipe.license;
585 
586 		foreach (subpackage; recipe.subPackages)
587 		{
588 			DubPackageInfo info;
589 			info.ver = subpackage.recipe.version_;
590 			info.name = subpackage.recipe.name;
591 			info.path = subpackage.path;
592 			info.fill(subpackage.recipe);
593 		}
594 	}
595 }
596 
597 DubPackageInfo getInfo(in Package dep)
598 {
599 	DubPackageInfo info;
600 	info.name = dep.name;
601 	info.ver = dep.version_.toString;
602 	info.path = dep.path.toString;
603 	info.fill(dep.recipe);
604 	foreach (subDep; dep.getAllDependencies())
605 	{
606 		info.dependencies[subDep.name] = subDep.spec.toString;
607 	}
608 	return info;
609 }
610 
611 auto listDependencies(Project project)
612 {
613 	auto deps = project.dependencies;
614 	DubPackageInfo[] dependencies;
615 	if (deps is null)
616 		return dependencies;
617 	foreach (dep; deps)
618 	{
619 		dependencies ~= getInfo(dep);
620 	}
621 	return dependencies;
622 }
623 
624 string[] listDependencies(Package pkg)
625 {
626 	auto deps = pkg.getAllDependencies();
627 	string[] dependencies;
628 	if (deps is null)
629 		return dependencies;
630 	foreach (dep; deps)
631 		dependencies ~= dep.name;
632 	return dependencies;
633 }