1 module workspaced.com.dub;
2 
3 import core.exception;
4 import core.sync.mutex;
5 import core.thread;
6 
7 import std.algorithm;
8 import std.array : appender;
9 import std.conv;
10 import std.exception;
11 import std.json : JSONType, JSONValue;
12 import std.parallelism;
13 import std.regex;
14 import std.stdio;
15 import std.string;
16 
17 import painlessjson : toJSON, fromJSON;
18 
19 import workspaced.api;
20 
21 import dub.description;
22 import dub.dub;
23 import dub.package_;
24 import dub.project;
25 
26 import dub.compilers.compiler;
27 import dub.generators.build;
28 import dub.generators.generator;
29 
30 import dub.compilers.buildsettings;
31 
32 import dub.internal.vibecompat.core.log;
33 import dub.internal.vibecompat.inet.url;
34 
35 @component("dub")
36 class DubComponent : ComponentWrapper
37 {
38 	mixin DefaultComponentWrapper;
39 
40 	static void registered()
41 	{
42 		setLogLevel(LogLevel.none);
43 	}
44 
45 	protected void load()
46 	{
47 		if (!refInstance)
48 			throw new Exception("dub requires to be instanced");
49 
50 		if (config.get!bool("dub", "registerImportProvider", true))
51 			importPathProvider = &imports;
52 		if (config.get!bool("dub", "registerStringImportProvider", true))
53 			stringImportPathProvider = &stringImports;
54 		if (config.get!bool("dub", "registerImportFilesProvider", false))
55 			importFilesProvider = &fileImports;
56 
57 		try
58 		{
59 			start();
60 
61 			_configuration = _dub.project.getDefaultConfiguration(_platform);
62 			if (!_dub.project.configurations.canFind(_configuration))
63 			{
64 				stderr.writeln("Dub Error: No configuration available");
65 				workspaced.broadcast(refInstance, JSONValue([
66 							"type": JSONValue("warning"),
67 							"component": JSONValue("dub"),
68 							"detail": JSONValue("invalid-default-config")
69 						]));
70 			}
71 			else
72 				updateImportPaths(false);
73 		}
74 		catch (Exception e)
75 		{
76 			if (!_dub || !_dub.project)
77 				throw e;
78 			stderr.writeln("Dub Error (ignored): ", e);
79 		}
80 		/*catch (AssertError e)
81 		{
82 			if (!_dub || !_dub.project)
83 				throw e;
84 			stderr.writeln("Dub Error (ignored): ", e);
85 		}*/
86 	}
87 
88 	private void start()
89 	{
90 		_dubRunning = false;
91 		_dub = new Dub(instance.cwd, null, SkipPackageSuppliers.none);
92 		_dub.packageManager.getOrLoadPackage(NativePath(instance.cwd));
93 		_dub.loadPackage();
94 		_dub.project.validate();
95 
96 		// mark all packages as optional so we don't crash
97 		int missingPackages;
98 		auto optionalified = optionalifyPackages;
99 		foreach (ref pkg; _dub.project.getTopologicalPackageList())
100 		{
101 			optionalifyRecipe(pkg);
102 			foreach (dep; pkg.getAllDependencies().filter!(a => optionalified.canFind(a.name)))
103 			{
104 				auto d = _dub.project.getDependency(dep.name, true);
105 				if (!d)
106 					missingPackages++;
107 				else
108 					optionalifyRecipe(d);
109 			}
110 		}
111 
112 		if (!_compilerBinaryName.length)
113 			_compilerBinaryName = _dub.defaultCompiler;
114 		setCompiler(_compilerBinaryName);
115 
116 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings();
117 
118 		if (missingPackages > 0)
119 		{
120 			upgrade(false);
121 			optionalifyPackages();
122 		}
123 
124 		_dubRunning = true;
125 	}
126 
127 	private string[] optionalifyPackages()
128 	{
129 		bool[Package] visited;
130 		string[] optionalified;
131 		foreach (pkg; _dub.project.dependencies)
132 			optionalified ~= optionalifyRecipe(cast() pkg);
133 		return optionalified;
134 	}
135 
136 	private string[] optionalifyRecipe(Package pkg)
137 	{
138 		string[] optionalified;
139 		foreach (key, ref value; pkg.recipe.buildSettings.dependencies)
140 		{
141 			if (!value.optional)
142 			{
143 				value.optional = true;
144 				value.default_ = true;
145 				optionalified ~= key;
146 			}
147 		}
148 		foreach (ref config; pkg.recipe.configurations)
149 			foreach (key, ref value; config.buildSettings.dependencies)
150 			{
151 				if (!value.optional)
152 				{
153 					value.optional = true;
154 					value.default_ = true;
155 					optionalified ~= key;
156 				}
157 			}
158 		return optionalified;
159 	}
160 
161 	private void restart()
162 	{
163 		_dub.destroy();
164 		_dubRunning = false;
165 		start();
166 	}
167 
168 	bool isRunning()
169 	{
170 		return _dub !is null && _dub.project !is null && _dub.project.rootPackage !is null
171 			&& _dubRunning;
172 	}
173 
174 	/// Reloads the dub.json or dub.sdl file from the cwd
175 	/// Returns: `false` if there are no import paths available
176 	Future!bool update()
177 	{
178 		restart();
179 		mixin(gthreadsAsyncProxy!`updateImportPaths(false)`);
180 	}
181 
182 	bool updateImportPaths(bool restartDub = true)
183 	{
184 		validateConfiguration();
185 
186 		if (restartDub)
187 			restart();
188 
189 		GeneratorSettings settings;
190 		settings.platform = _platform;
191 		settings.config = _configuration;
192 		settings.buildType = _buildType;
193 		settings.compiler = _compiler;
194 		settings.buildSettings = _settings;
195 		settings.buildSettings.addOptions(BuildOption.syntaxOnly);
196 		settings.combined = true;
197 		settings.run = false;
198 
199 		try
200 		{
201 			auto paths = _dub.project.listBuildSettings(settings, [
202 					"import-paths", "string-import-paths", "source-files"
203 					], ListBuildSettingsFormat.listNul);
204 			_importPaths = paths[0].split('\0');
205 			_stringImportPaths = paths[1].split('\0');
206 			_importFiles = paths[2].split('\0');
207 			return _importPaths.length > 0 || _importFiles.length > 0;
208 		}
209 		catch (Exception e)
210 		{
211 			workspaced.broadcast(refInstance, JSONValue([
212 						"type": JSONValue("error"),
213 						"component": JSONValue("dub"),
214 						"detail": JSONValue("Error while listing import paths: " ~ e.toString)
215 					]));
216 			_importPaths = [];
217 			_stringImportPaths = [];
218 			return false;
219 		}
220 	}
221 
222 	/// Calls `dub upgrade`
223 	void upgrade(bool save = true)
224 	{
225 		if (save)
226 			_dub.upgrade(UpgradeOptions.select | UpgradeOptions.upgrade);
227 		else
228 			_dub.upgrade(UpgradeOptions.noSaveSelections);
229 	}
230 
231 	/// Throws if configuration is invalid, otherwise does nothing.
232 	void validateConfiguration() const
233 	{
234 		if (!_dub.project.configurations.canFind(_configuration))
235 			throw new Exception("Cannot use dub with invalid configuration");
236 	}
237 
238 	/// Throws if configuration is invalid or targetType is none or source library, otherwise does nothing.
239 	void validateBuildConfiguration()
240 	{
241 		if (!_dub.project.configurations.canFind(_configuration))
242 			throw new Exception("Cannot use dub with invalid configuration");
243 		if (_settings.targetType == TargetType.none)
244 			throw new Exception("Cannot build with dub with targetType == none");
245 		if (_settings.targetType == TargetType.sourceLibrary)
246 			throw new Exception("Cannot build with dub with targetType == sourceLibrary");
247 	}
248 
249 	/// Lists all dependencies. This will go through all dependencies and contain the dependencies of dependencies. You need to create a tree structure from this yourself.
250 	/// Returns: `[{dependencies: string[string], ver: string, name: string}]`
251 	auto dependencies() @property const
252 	{
253 		validateConfiguration();
254 
255 		return _dub.project.listDependencies();
256 	}
257 
258 	/// Lists dependencies of the root package. This can be used as a base to create a tree structure.
259 	string[] rootDependencies() @property const
260 	{
261 		validateConfiguration();
262 
263 		return _dub.project.rootPackage.listDependencies();
264 	}
265 
266 	/// Returns the path to the root package recipe (dub.json/dub.sdl)
267 	///
268 	/// Note that this can be empty if the package is not in the local file system.
269 	string recipePath() @property
270 	{
271 		return _dub.project.rootPackage.recipePath.toString;
272 	}
273 
274 	/// Lists all import paths
275 	string[] imports() @property nothrow
276 	{
277 		return _importPaths;
278 	}
279 
280 	/// Lists all string import paths
281 	string[] stringImports() @property nothrow
282 	{
283 		return _stringImportPaths;
284 	}
285 
286 	/// Lists all import paths to files
287 	string[] fileImports() @property nothrow
288 	{
289 		return _importFiles;
290 	}
291 
292 	/// Lists all configurations defined in the package description
293 	string[] configurations() @property
294 	{
295 		return _dub.project.configurations;
296 	}
297 
298 	/// Lists all build types defined in the package description AND the predefined ones from dub ("plain", "debug", "release", "release-debug", "release-nobounds", "unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov")
299 	string[] buildTypes() @property
300 	{
301 		string[] types = [
302 			"plain", "debug", "release", "release-debug", "release-nobounds",
303 			"unittest", "docs", "ddox", "profile", "profile-gc", "cov", "unittest-cov"
304 		];
305 		foreach (type, info; _dub.project.rootPackage.recipe.buildTypes)
306 			types ~= type;
307 		return types;
308 	}
309 
310 	/// Gets the current selected configuration
311 	string configuration() @property
312 	{
313 		return _configuration;
314 	}
315 
316 	/// Selects a new configuration and updates the import paths accordingly
317 	/// Returns: `false` if there are no import paths in the new configuration
318 	bool setConfiguration(string configuration)
319 	{
320 		if (!_dub.project.configurations.canFind(configuration))
321 			return false;
322 		_configuration = configuration;
323 		_settingsTemplate = cast() _dub.project.rootPackage.getBuildSettings(configuration);
324 		return updateImportPaths(false);
325 	}
326 
327 	/// List all possible arch types for current set compiler
328 	string[] archTypes() @property
329 	{
330 		string[] types = ["x86_64", "x86"];
331 
332 		string compilerName = _compiler.name;
333 
334 		version (Windows)
335 		{
336 			if (compilerName == "dmd")
337 			{
338 				types ~= "x86_mscoff";
339 			}
340 		}
341 		if (compilerName == "gdc")
342 		{
343 			types ~= ["arm", "arm_thumb"];
344 		}
345 
346 		return types;
347 	}
348 
349 	/// Returns the current selected arch type
350 	string archType() @property
351 	{
352 		return _archType;
353 	}
354 
355 	/// Selects a new arch type and updates the import paths accordingly
356 	/// Returns: `false` if there are no import paths in the new arch type
357 	bool setArchType(JSONValue request)
358 	{
359 		enforce(request.type == JSONType.object && "arch-type" in request, "arch-type not in request");
360 		auto type = request["arch-type"].fromJSON!string;
361 		if (archTypes.canFind(type))
362 		{
363 			_archType = type;
364 			return updateImportPaths(false);
365 		}
366 		else
367 		{
368 			return false;
369 		}
370 	}
371 
372 	/// Returns the current selected build type
373 	string buildType() @property
374 	{
375 		return _buildType;
376 	}
377 
378 	/// Selects a new build type and updates the import paths accordingly
379 	/// Returns: `false` if there are no import paths in the new build type
380 	bool setBuildType(JSONValue request)
381 	{
382 		enforce(request.type == JSONType.object && "build-type" in request, "build-type not in request");
383 		auto type = request["build-type"].fromJSON!string;
384 		if (buildTypes.canFind(type))
385 		{
386 			_buildType = type;
387 			return updateImportPaths(false);
388 		}
389 		else
390 		{
391 			return false;
392 		}
393 	}
394 
395 	/// Returns the current selected compiler
396 	string compiler() @property
397 	{
398 		return _compilerBinaryName;
399 	}
400 
401 	/// Selects a new compiler for building
402 	/// Returns: `false` if the compiler does not exist
403 	bool setCompiler(string compiler)
404 	{
405 		try
406 		{
407 			_compilerBinaryName = compiler;
408 			_compiler = getCompiler(compiler); // make sure it gets a valid compiler
409 		}
410 		catch (Exception e)
411 		{
412 			return false;
413 		}
414 		_platform = _compiler.determinePlatform(_settings, _compilerBinaryName, _archType);
415 		_settingsTemplate.getPlatformSettings(_settings, _platform, _dub.project.rootPackage.path);
416 		return _compiler !is null;
417 	}
418 
419 	/// Returns the project name
420 	string name() @property
421 	{
422 		return _dub.projectName;
423 	}
424 
425 	/// Returns the project path
426 	auto path() @property
427 	{
428 		return _dub.projectPath;
429 	}
430 
431 	/// Returns whether there is a target set to build. If this is false then build will throw an exception.
432 	bool canBuild() @property
433 	{
434 		if (_settings.targetType == TargetType.none || _settings.targetType == TargetType.sourceLibrary
435 				|| !_dub.project.configurations.canFind(_configuration))
436 			return false;
437 		return true;
438 	}
439 
440 	/// Asynchroniously builds the project WITHOUT OUTPUT. This is intended for linting code and showing build errors quickly inside the IDE.
441 	Future!(BuildIssue[]) build()
442 	{
443 		import std.process : thisProcessID;
444 		import std.file : tempDir;
445 		import std.random : uniform;
446 
447 		validateBuildConfiguration();
448 
449 		// copy to this thread
450 		auto compiler = _compiler;
451 		auto buildPlatform = _platform;
452 
453 		GeneratorSettings settings;
454 		settings.platform = buildPlatform;
455 		settings.config = _configuration;
456 		settings.buildType = _buildType;
457 		settings.compiler = compiler;
458 		settings.buildSettings = _settings;
459 
460 		auto ret = new Future!(BuildIssue[]);
461 		new Thread({
462 			try
463 			{
464 				auto issues = appender!(BuildIssue[]);
465 
466 				settings.compileCallback = (status, output) {
467 					string[] lines = output.splitLines;
468 					foreach (line; lines)
469 					{
470 						auto match = line.matchFirst(errorFormat);
471 						if (match)
472 						{
473 							issues ~= BuildIssue(match[2].to!int, match[3].toOr!int(0),
474 								match[1], match[4].to!ErrorType, match[5]);
475 						}
476 						else
477 						{
478 							auto contMatch = line.matchFirst(errorFormatCont);
479 							if (issues.data.length && contMatch)
480 							{
481 								issues ~= BuildIssue(contMatch[2].to!int,
482 									contMatch[3].toOr!int(1), contMatch[1],
483 									issues.data[$ - 1].type, contMatch[4], true);
484 							}
485 							else if (line.canFind("is deprecated"))
486 							{
487 								auto deprMatch = line.matchFirst(deprecationFormat);
488 								if (deprMatch)
489 								{
490 									issues ~= BuildIssue(deprMatch[2].to!int, deprMatch[3].toOr!int(1),
491 										deprMatch[1], ErrorType.Deprecation,
492 										deprMatch[4] ~ " is deprecated, use " ~ deprMatch[5] ~ " instead.");
493 									// TODO: maybe add special type or output
494 								}
495 							}
496 						}
497 					}
498 				};
499 				try
500 				{
501 					import workspaced.dub.lintgenerator : DubLintGenerator;
502 
503 					new DubLintGenerator(_dub.project).generate(settings);
504 				}
505 				catch (Exception e)
506 				{
507 					if (!e.msg.matchFirst(harmlessExceptionFormat))
508 						throw e;
509 				}
510 				ret.finish(issues.data);
511 			}
512 			catch (Throwable t)
513 			{
514 				ret.error(t);
515 			}
516 		}).start();
517 		return ret;
518 	}
519 
520 	/// Converts the root package recipe to another format.
521 	/// Params:
522 	///     format = either "json" or "sdl".
523 	string convertRecipe(string format)
524 	{
525 		import dub.recipe.io : serializePackageRecipe;
526 		import std.array : appender;
527 
528 		auto dst = appender!string;
529 		serializePackageRecipe(dst, _dub.project.rootPackage.rawRecipe, "dub." ~ format);
530 		return dst.data;
531 	}
532 
533 private:
534 	Dub _dub;
535 	bool _dubRunning = false;
536 	string _configuration;
537 	string _archType = "x86_64";
538 	string _buildType = "debug";
539 	string _compilerBinaryName;
540 	Compiler _compiler;
541 	BuildSettingsTemplate _settingsTemplate;
542 	BuildSettings _settings;
543 	BuildPlatform _platform;
544 	string[] _importPaths, _stringImportPaths, _importFiles;
545 }
546 
547 ///
548 enum ErrorType : ubyte
549 {
550 	///
551 	Error = 0,
552 	///
553 	Warning = 1,
554 	///
555 	Deprecation = 2
556 }
557 
558 /// Returned by build
559 struct BuildIssue
560 {
561 	///
562 	int line, column;
563 	///
564 	string file;
565 	/// The error type (Error/Warning/Deprecation) outputted by dmd or inherited from the last error if this is additional information of the last issue. (indicated by cont)
566 	ErrorType type;
567 	///
568 	string text;
569 	/// true if this is additional error information for the last error.
570 	bool cont;
571 }
572 
573 private:
574 
575 T toOr(T)(string s, T defaultValue)
576 {
577 	if (!s || !s.length)
578 		return defaultValue;
579 	return s.to!T;
580 }
581 
582 enum harmlessExceptionFormat = ctRegex!(`failed with exit code`, "g");
583 enum errorFormat = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\): (Deprecation|Warning|Error): (.*)`, "gi");
584 enum errorFormatCont = ctRegex!(`(.*?)\((\d+)(?:,(\d+))?\):[ ]{6,}(.*)`, "g");
585 enum deprecationFormat = ctRegex!(
586 			`(.*?)\((\d+)(?:,(\d+))?\): (.*?) is deprecated, use (.*?) instead.$`, "g");
587 
588 struct DubPackageInfo
589 {
590 	string[string] dependencies;
591 	string ver;
592 	string name;
593 	string path;
594 	string description;
595 	string homepage;
596 	const(string)[] authors;
597 	string copyright;
598 	string license;
599 	DubPackageInfo[] subPackages;
600 
601 	void fill(in PackageRecipe recipe)
602 	{
603 		description = recipe.description;
604 		homepage = recipe.homepage;
605 		authors = recipe.authors;
606 		copyright = recipe.copyright;
607 		license = recipe.license;
608 
609 		foreach (subpackage; recipe.subPackages)
610 		{
611 			DubPackageInfo info;
612 			info.ver = subpackage.recipe.version_;
613 			info.name = subpackage.recipe.name;
614 			info.path = subpackage.path;
615 			info.fill(subpackage.recipe);
616 		}
617 	}
618 }
619 
620 DubPackageInfo getInfo(in Package dep)
621 {
622 	DubPackageInfo info;
623 	info.name = dep.name;
624 	info.ver = dep.version_.toString;
625 	info.path = dep.path.toString;
626 	info.fill(dep.recipe);
627 	foreach (subDep; dep.getAllDependencies())
628 	{
629 		info.dependencies[subDep.name] = subDep.spec.toString;
630 	}
631 	return info;
632 }
633 
634 auto listDependencies(scope const Project project)
635 {
636 	auto deps = project.dependencies;
637 	DubPackageInfo[] dependencies;
638 	if (deps is null)
639 		return dependencies;
640 	foreach (dep; deps)
641 	{
642 		dependencies ~= getInfo(dep);
643 	}
644 	return dependencies;
645 }
646 
647 string[] listDependencies(scope const Package pkg)
648 {
649 	auto deps = pkg.getAllDependencies();
650 	string[] dependencies;
651 	if (deps is null)
652 		return dependencies;
653 	foreach (dep; deps)
654 		dependencies ~= dep.name;
655 	return dependencies;
656 }