1 module workspaced.com.dub;
2 
3 import core.sync.mutex;
4 import core.exception;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.conv;
9 import std.exception;
10 import std.json : JSONValue, JSON_TYPE;
11 import std.parallelism;
12 import std.regex;
13 import std.stdio;
14 import std.string;
15 
16 import painlessjson : toJSON, fromJSON;
17 
18 import workspaced.api;
19 
20 import dub.description;
21 import dub.dub;
22 import dub.package_;
23 import dub.project;
24 
25 import dub.compilers.compiler;
26 import dub.generators.build;
27 import dub.generators.generator;
28 
29 import dub.compilers.buildsettings;
30 
31 import dub.internal.vibecompat.core.log;
32 import dub.internal.vibecompat.inet.url;
33 
34 @component("dub")
35 class DubComponent : ComponentWrapper
36 {
37 	mixin DefaultComponentWrapper;
38 
39 	static void registered()
40 	{
41 		setLogLevel(LogLevel.none);
42 	}
43 
44 	protected void load()
45 	{
46 		if (!refInstance)
47 			throw new Exception("dub requires to be instanced");
48 
49 		if (config.get!bool("dub", "registerImportProvider", true))
50 			importPathProvider = &imports;
51 		if (config.get!bool("dub", "registerStringImportProvider", true))
52 			stringImportPathProvider = &stringImports;
53 		if (config.get!bool("dub", "registerImportFilesProvider", false))
54 			importFilesProvider = &fileImports;
55 
56 		try
57 		{
58 			start();
59 
60 			_configuration = _dub.project.getDefaultConfiguration(_platform);
61 			if (!_dub.project.configurations.canFind(_configuration))
62 			{
63 				stderr.writeln("Dub Error: No configuration available");
64 				workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("warning"),
65 						"component" : JSONValue("dub"), "detail" : JSONValue("invalid-default-config")]));
66 			}
67 			else
68 				updateImportPaths(false);
69 		}
70 		catch (Exception e)
71 		{
72 			if (!_dub || !_dub.project)
73 				throw e;
74 			stderr.writeln("Dub Error (ignored): ", e);
75 		}
76 		/*catch (AssertError e)
77 		{
78 			if (!_dub || !_dub.project)
79 				throw e;
80 			stderr.writeln("Dub Error (ignored): ", e);
81 		}*/
82 	}
83 
84 	private void start()
85 	{
86 		_dubRunning = false;
87 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
88 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
89 		_dub.loadPackage();
90 		_dub.project.validate();
91 
92 		// mark all packages as optional so we don't crash
93 		int missingPackages;
94 		auto optionalified = optionalifyPackages;
95 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
96 		{
97 			optionalifyRecipe(pkg);
98 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
99 			{
100 				auto d = _dub.project.getDependency(dep.name, true);
101 				if (!d)
102 					missingPackages++;
103 				else
104 					optionalifyRecipe(d);
105 			}
106 		}
107 
108 		if (!_compilerBinaryName.length)
109 			_compilerBinaryName = _dub.defaultCompiler;
110 		setCompiler(_compilerBinaryName);
111 		if (missingPackages > 0)
112 		{
113 			upgrade(false);
114 			optionalifyPackages();
115 		}
116 
117 		_dubRunning = true;
118 	}
119 
120 	private string[] optionalifyPackages()
121 	{
122 		bool[Package] visited;
123 		string[] optionalified;
124 		foreach (pkg; _dub.project.dependencies)
125 			optionalified ~= optionalifyRecipe(cast() pkg);
126 		return optionalified;
127 	}
128 
129 	private string[] optionalifyRecipe(Package pkg)
130 	{
131 		string[] optionalified;
132 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
133 		{
134 			if (!value.optional)
135 			{
136 				value.optional = true;
137 				value.default_ = true;
138 				optionalified ~= key;
139 			}
140 		}
141 		foreach (ref config; pkg.recipe.configurations)
142 			foreach (key, ref value; config.buildSettings.dependencies)
143 			{
144 				if (!value.optional)
145 				{
146 					value.optional = true;
147 					value.default_ = true;
148 					optionalified ~= key;
149 				}
150 			}
151 		return optionalified;
152 	}
153 
154 	private void restart()
155 	{
156 		_dub.destroy();
157 		_dubRunning = false;
158 		start();
159 	}
160 
161 	bool isRunning()
162 	{
163 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
164 			&& _dubRunning;
165 	}
166 
167 	/// Reloads the dub.json or dub.sdl file from the cwd
168 	/// Returns: `false` if there are no import paths available
169 	Future!bool update()
170 	{
171 		restart();
172 		auto ret = new Future!bool;
173 		threads.create({ /**/
174 			try
175 			{
176 				auto result = updateImportPaths(false);
177 				ret.finish(result);
178 			}
179 			catch (Throwable t)
180 			{
181 				ret.error(t);
182 			}
183 		});
184 		return ret;
185 	}
186 
187 	bool updateImportPaths(bool restartDub = true)
188 	{
189 		validateConfiguration();
190 
191 		if (restartDub)
192 			restart();
193 
194 		GeneratorSettings settings;
195 		settings.platform = _platform;
196 		settings.config = _configuration;
197 		settings.buildType = _buildType;
198 		settings.compiler = _compiler;
199 		settings.buildSettings = _settings;
200 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
201 		settings.combined = true;
202 		settings.run = false;
203 
204 		try
205 		{
206 			auto paths = _dub.project.listBuildSettings(settings, ["import-paths",
207 					"string-import-paths", "source-files"], ListBuildSettingsFormat.listNul);
208 			_importPaths = paths[0].split('\0');
209 			_stringImportPaths = paths[1].split('\0');
210 			_importFiles = paths[2].split('\0');
211 			return _importPaths.length > 0 || _importFiles.length > 0;
212 		}
213 		catch (Exception e)
214 		{
215 			workspaced.broadcast(refInstance, JSONValue(["type" : JSONValue("error"), "component"
216 					: JSONValue("dub"), "detail"
217 					: JSONValue("Error while listing import paths: " ~ e.toString)]));
218 			_importPaths = [];
219 			_stringImportPaths = [];
220 			return false;
221 		}
222 	}
223 
224 	/// Calls `dub upgrade`
225 	void upgrade(bool save = true)
226 	{
227 		if (save)
228 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
229 		else
230 			_dub.upgrade(UpgradeOptions.noSaveSelections);
231 	}
232 
233 	/// Throws if configuration is invalid, otherwise does nothing.
234 	void validateConfiguration()
235 	{
236 		if (!_dub.project.configurations.canFind(_configuration))
237 			throw new Exception("Cannot use dub with invalid configuration");
238 	}
239 
240 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
241 	void validateBuildConfiguration()
242 	{
243 		if (!_dub.project.configurations.canFind(_configuration))
244 			throw new Exception("Cannot use dub with invalid configuration");
245 		if (_settings.targetType == TargetType.none)
246 			throw new Exception("Cannot build with dub with targetType == none");
247 		if (_settings.targetType == TargetType.sourceLibrary)
248 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
249 	}
250 
251 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
252 	/// Returns: `[{dependencies: [string], ver: string, name: string}]`
253 	auto dependencies() @property
254 	{
255 		validateConfiguration();
256 
257 		return _dub.project.listDependencies();
258 	}
259 
260 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
261 	string[] rootDependencies() @property
262 	{
263 		validateConfiguration();
264 
265 		return _dub.project.rootPackage.listDependencies();
266 	}
267 
268 	/// Returns the path to the root package recipe (dub.json/dub.sdl)
269 	///
270 	/// Note that this can be empty if the package is not in the local file system.
271 	string recipePath() @property
272 	{
273 		return _dub.project.rootPackage.recipePath.toString;
274 	}
275 
276 	/// Lists all import paths
277 	string[] imports() @property
278 	{
279 		return _importPaths;
280 	}
281 
282 	/// Lists all string import paths
283 	string[] stringImports() @property
284 	{
285 		return _stringImportPaths;
286 	}
287 
288 	/// Lists all import paths to files
289 	string[] fileImports() @property
290 	{
291 		return _importFiles;
292 	}
293 
294 	/// Lists all configurations defined in the package description
295 	string[] configurations() @property
296 	{
297 		return _dub.project.configurations;
298 	}
299 
300 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
301 	string[] buildTypes() @property
302 	{
303 		string[] types = [
304 			"plain", "debug", "release", "release-debug", "release-nobounds",
305 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
306 		];
307 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
308 			types ~= type;
309 		return types;
310 	}
311 
312 	/// Gets the current selected configuration
313 	string configuration() @property
314 	{
315 		return _configuration;
316 	}
317 
318 	/// Selects a new configuration and updates the import paths accordingly
319 	/// Returns: `false` if there are no import paths in the new configuration
320 	bool setConfiguration(string configuration)
321 	{
322 		if (!_dub.project.configurations.canFind(configuration))
323 			return false;
324 		_configuration = configuration;
325 		return updateImportPaths(false);
326 	}
327 
328 	/// List all possible arch types for current set compiler
329 	string[] archTypes() @property
330 	{
331 		string[] types = ["x86_64", "x86"];
332 
333 		string compilerName = _compiler.name;
334 
335 		version (Windows)
336 		{
337 			if (compilerName == "dmd")
338 			{
339 				types ~= "x86_mscoff";
340 			}
341 		}
342 		if (compilerName == "gdc")
343 		{
344 			types ~= ["arm", "arm_thumb"];
345 		}
346 
347 		return types;
348 	}
349 
350 	/// Returns the current selected arch type
351 	string archType() @property
352 	{
353 		return _archType;
354 	}
355 
356 	/// Selects a new arch type and updates the import paths accordingly
357 	/// Returns: `false` if there are no import paths in the new arch type
358 	bool setArchType(JSONValue request)
359 	{
360 		enforce(request.type == JSON_TYPE.OBJECT && "arch-type" in request, "arch-type not in request");
361 		auto type = request["arch-type"].fromJSON!string;
362 		if (archTypes.canFind(type))
363 		{
364 			_archType = type;
365 			return updateImportPaths(false);
366 		}
367 		else
368 		{
369 			return false;
370 		}
371 	}
372 
373 	/// Returns the current selected build type
374 	string buildType() @property
375 	{
376 		return _buildType;
377 	}
378 
379 	/// Selects a new build type and updates the import paths accordingly
380 	/// Returns: `false` if there are no import paths in the new build type
381 	bool setBuildType(JSONValue request)
382 	{
383 		enforce(request.type == JSON_TYPE.OBJECT && "build-type" in request,
384 				"build-type not in request");
385 		auto type = request["build-type"].fromJSON!string;
386 		if (buildTypes.canFind(type))
387 		{
388 			_buildType = type;
389 			return updateImportPaths(false);
390 		}
391 		else
392 		{
393 			return false;
394 		}
395 	}
396 
397 	/// Returns the current selected compiler
398 	string compiler() @property
399 	{
400 		return _compilerBinaryName;
401 	}
402 
403 	/// Selects a new compiler for building
404 	/// Returns: `false` if the compiler does not exist
405 	bool setCompiler(string compiler)
406 	{
407 		try
408 		{
409 			_compilerBinaryName = compiler;
410 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
411 		}
412 		catch (Exception e)
413 		{
414 			return false;
415 		}
416 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
417 		return true;
418 	}
419 
420 	/// Returns the project name
421 	string name() @property
422 	{
423 		return _dub.projectName;
424 	}
425 
426 	/// Returns the project path
427 	auto path() @property
428 	{
429 		return _dub.projectPath;
430 	}
431 
432 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
433 	bool canBuild() @property
434 	{
435 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
436 				|| !_dub.project.configurations.canFind(_configuration))
437 			return false;
438 		return true;
439 	}
440 
441 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
442 	Future!(BuildIssue[]) build()
443 	{
444 		import std.process : thisProcessID;
445 		import std.file : tempDir;
446 		import std.random : uniform;
447 
448 		validateBuildConfiguration();
449 
450 		// copy to this thread
451 		auto compiler = _compiler;
452 		auto buildPlatform = _platform;
453 
454 		GeneratorSettings settings;
455 		settings.platform = buildPlatform;
456 		settings.config = _configuration;
457 		settings.buildType = _buildType;
458 		settings.compiler = compiler;
459 		settings.tempBuild = true;
460 		settings.buildSettings = _settings;
461 		settings.buildSettings.targetPath = tempDir;
462 		settings.buildSettings.targetName = "workspace-d-build-"
463 			~ thisProcessID.to!string ~ "-" ~ uniform!uint.to!string(36);
464 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
465 		settings.buildSettings.addDFlags("-o-");
466 
467 		auto ret = new Future!(BuildIssue[]);
468 		new Thread({
469 			try
470 			{
471 				BuildIssue[] issues;
472 
473 				settings.compileCallback = (status, output) {
474 					string[] lines = output.splitLines;
475 					foreach (line; lines)
476 					{
477 						auto match = line.matchFirst(errorFormat);
478 						if (match)
479 						{
480 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
481 								match[1], match[4].to!ErrorType, match[5]);
482 						}
483 						else
484 						{
485 							if (line.canFind("from"))
486 							{
487 								auto contMatch = line.matchFirst(errorFormatCont);
488 								if (contMatch)
489 								{
490 									issues ~= BuildIssue(contMatch[2].to!int,
491 										contMatch[3].toOr!int(1), contMatch[1], ErrorType.Error, contMatch[4]);
492 								}
493 							}
494 							if (line.canFind("is deprecated"))
495 							{
496 								auto deprMatch = line.matchFirst(deprecationFormat);
497 								if (deprMatch)
498 								{
499 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
500 										deprMatch[1], ErrorType.Deprecation,
501 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
502 									// TODO: maybe add special type or output
503 								}
504 							}
505 						}
506 					}
507 				};
508 				try
509 				{
510 					_dub.generateProject("build", settings);
511 				}
512 				catch (Exception e)
513 				{
514 					if (!e.msg.matchFirst(harmlessExceptionFormat))
515 						throw e;
516 				}
517 				ret.finish(issues);
518 			}
519 			catch (Throwable t)
520 			{
521 				ret.error(t);
522 			}
523 		}).start();
524 		return ret;
525 	}
526 
527 	/// Converts the root package recipe to another format.
528 	/// Params:
529 	///     format = either "json" or "sdl".
530 	string convertRecipe(string format)
531 	{
532 		import dub.recipe.io : serializePackageRecipe;
533 		import std.array : appender;
534 
535 		auto dst = appender!string;
536 		serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format);
537 		return dst.data;
538 	}
539 
540 private:
541 	Dub _dub;
542 	bool _dubRunning = false;
543 	string _configuration;
544 	string _archType = "x86_64";
545 	string _buildType = "debug";
546 	string _compilerBinaryName;
547 	Compiler _compiler;
548 	BuildSettings _settings;
549 	BuildPlatform _platform;
550 	string[] _importPaths, _stringImportPaths, _importFiles;
551 }
552 
553 ///
554 enum ErrorType : ubyte
555 {
556 	///
557 	Error = 0,
558 	///
559 	Warning = 1,
560 	///
561 	Deprecation = 2
562 }
563 
564 /// Returned by build
565 struct BuildIssue
566 {
567 	///
568 	int line, column;
569 	///
570 	string file;
571 	///
572 	ErrorType type;
573 	///
574 	string text;
575 }
576 
577 private:
578 
579 T toOr(T)(string s, T defaultValue)
580 {
581 	if (!s || !s.length)
582 		return defaultValue;
583 	return s.to!T;
584 }
585 
586 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
587 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
588 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (.*)`, "g");
589 enum deprecationFormat = ctRegex!(
590 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
591 
592 struct DubPackageInfo
593 {
594 	string[string] dependencies;
595 	string ver;
596 	string name;
597 	string path;
598 	string description;
599 	string homepage;
600 	const(string)[] authors;
601 	string copyright;
602 	string license;
603 	DubPackageInfo[] subPackages;
604 
605 	void fill(in PackageRecipe recipe)
606 	{
607 		description = recipe.description;
608 		homepage = recipe.homepage;
609 		authors = recipe.authors;
610 		copyright = recipe.copyright;
611 		license = recipe.license;
612 
613 		foreach (subpackage; recipe.subPackages)
614 		{
615 			DubPackageInfo info;
616 			info.ver = subpackage.recipe.version_;
617 			info.name = subpackage.recipe.name;
618 			info.path = subpackage.path;
619 			info.fill(subpackage.recipe);
620 		}
621 	}
622 }
623 
624 DubPackageInfo getInfo(in Package dep)
625 {
626 	DubPackageInfo info;
627 	info.name = dep.name;
628 	info.ver = dep.version_.toString;
629 	info.path = dep.path.toString;
630 	info.fill(dep.recipe);
631 	foreach (subDep; dep.getAllDependencies())
632 	{
633 		info.dependencies[subDep.name] = subDep.spec.toString;
634 	}
635 	return info;
636 }
637 
638 auto listDependencies(Project project)
639 {
640 	auto deps = project.dependencies;
641 	DubPackageInfo[] dependencies;
642 	if (deps is null)
643 		return dependencies;
644 	foreach (dep; deps)
645 	{
646 		dependencies ~= getInfo(dep);
647 	}
648 	return dependencies;
649 }
650 
651 string[] listDependencies(Package pkg)
652 {
653 	auto deps = pkg.getAllDependencies();
654 	string[] dependencies;
655 	if (deps is null)
656 		return dependencies;
657 	foreach (dep; deps)
658 		dependencies ~= dep.name;
659 	return dependencies;
660 }