1 module workspaced.com.dcd;
2 
3 import std.file : tempDir;
4 
5 import core.thread;
6 import std.algorithm;
7 import std.conv;
8 import std.datetime;
9 import std.json;
10 import std.path;
11 import std.process;
12 import std.random;
13 import std.stdio;
14 import std.string;
15 
16 import painlessjson;
17 
18 import workspaced.api;
19 import workspaced.helpers;
20 
21 version (OSX) version = haveUnixSockets;
22 version (linux) version = haveUnixSockets;
23 version (BSD) version = haveUnixSockets;
24 version (FreeBSD) version = haveUnixSockets;
25 
26 @component("dcd")
27 class DCDComponent : ComponentWrapper
28 {
29 	mixin DefaultComponentWrapper;
30 
31 	enum latestKnownVersion = [0, 10, 2];
32 	void load()
33 	{
34 		string clientPath = this.clientPath;
35 		string serverPath = this.serverPath;
36 
37 		installedVersion = clientPath.getVersionAndFixPath;
38 		string clientPathInfo = clientPath != "dcd-client" ? "(" ~ clientPath ~ ") " : "";
39 		stderr.writeln("Detected dcd-client ", clientPathInfo, installedVersion);
40 
41 		string serverInstalledVersion = serverPath.getVersionAndFixPath;
42 		string serverPathInfo = serverPath != "dcd-server" ? "(" ~ serverPath ~ ") " : "";
43 		stderr.writeln("Detected dcd-server ", serverPathInfo, serverInstalledVersion);
44 
45 		if (serverInstalledVersion != installedVersion)
46 			throw new Exception("client & server version mismatch");
47 
48 		config.set("dcd", "clientPath", clientPath);
49 		config.set("dcd", "serverPath", serverPath);
50 
51 		assert(this.clientPath == clientPath);
52 		assert(this.serverPath == serverPath);
53 
54 		version (haveUnixSockets)
55 			hasUnixDomainSockets = supportsUnixDomainSockets(installedVersion);
56 
57 		//dfmt off
58 		if (isOutdated)
59 			workspaced.broadcast(refInstance, JSONValue([
60 				"type": JSONValue("outdated"),
61 				"component": JSONValue("dcd")
62 			]));
63 		//dfmt on
64 		supportsFullOutput = rawExec([clientPath, "--help"]).output.canFind("--extended");
65 	}
66 
67 	/// Returns: true if DCD version is less than latestKnownVersion or if server and client mismatch or if it doesn't exist.
68 	bool isOutdated()
69 	{
70 		if (!installedVersion)
71 		{
72 			string clientPath = this.clientPath;
73 			string serverPath = this.serverPath;
74 
75 			try
76 			{
77 				installedVersion = clientPath.getVersionAndFixPath;
78 				if (serverPath.getVersionAndFixPath != installedVersion)
79 					return true;
80 			}
81 			catch (ProcessException)
82 			{
83 				return true;
84 			}
85 		}
86 		return !checkVersion(installedVersion, latestKnownVersion);
87 	}
88 
89 	/// Returns: the current detected installed version of dcd-client.
90 	string clientInstalledVersion() @property const
91 	{
92 		return installedVersion;
93 	}
94 
95 	private auto serverThreads()
96 	{
97 		return threads(1, 2);
98 	}
99 
100 	/// This stops the dcd-server instance safely and waits for it to exit
101 	override void shutdown()
102 	{
103 		stopServerSync();
104 		if (_threads)
105 			serverThreads.finish();
106 	}
107 
108 	/// This will start the dcd-server and load import paths from the current provider
109 	void setupServer(string[] additionalImports = [], bool quietServer = false)
110 	{
111 		startServer(importPaths ~ importFiles ~ additionalImports, quietServer);
112 	}
113 
114 	/// This will start the dcd-server
115 	void startServer(string[] additionalImports = [], bool quietServer = false)
116 	{
117 		if (isPortRunning(port))
118 			throw new Exception("Already running dcd on port " ~ port.to!string);
119 		string[] imports;
120 		foreach (i; additionalImports)
121 			if (i.length)
122 				imports ~= "-I" ~ i;
123 		this.runningPort = port;
124 		this.socketFile = buildPath(tempDir,
125 				"workspace-d-sock" ~ thisProcessID.to!string ~ "-" ~ uniform!ulong.to!string(36));
126 		serverPipes = raw([serverPath] ~ clientArgs ~ imports,
127 				Redirect.stdin | Redirect.stderr | Redirect.stdoutToStderr);
128 		while (!serverPipes.stderr.eof)
129 		{
130 			string line = serverPipes.stderr.readln();
131 			if (!quietServer)
132 			{
133 				stderr.writeln("Server: ", line);
134 				stderr.flush();
135 			}
136 			if (line.canFind("Startup completed in "))
137 				break;
138 		}
139 		running = true;
140 		serverThreads.create({
141 			mixin(traceTask);
142 			if (quietServer)
143 				foreach (block; serverPipes.stderr.byChunk(4096))
144 				{
145 				}
146 			else
147 				while (!serverPipes.stderr.eof)
148 				{
149 					stderr.writeln("Server: ", serverPipes.stderr.readln());
150 				}
151 			auto code = serverPipes.pid.wait();
152 			stderr.writeln("DCD-Server stopped with code ", code);
153 			if (code != 0)
154 			{
155 				stderr.writeln("Broadcasting dcd server crash.");
156 				workspaced.broadcast(refInstance, JSONValue([
157 						"type": JSONValue("crash"),
158 						"component": JSONValue("dcd")
159 					]));
160 				running = false;
161 			}
162 		});
163 	}
164 
165 	void stopServerSync()
166 	{
167 		if (!running)
168 			return;
169 		int i = 0;
170 		running = false;
171 		doClient(["--shutdown"]).pid.wait;
172 		while (serverPipes.pid && !serverPipes.pid.tryWait().terminated)
173 		{
174 			Thread.sleep(10.msecs);
175 			if (++i > 200) // Kill after 2 seconds
176 			{
177 				killServer();
178 				return;
179 			}
180 		}
181 	}
182 
183 	/// This stops the dcd-server asynchronously
184 	/// Returns: null
185 	Future!void stopServer()
186 	{
187 		auto ret = new Future!void();
188 		gthreads.create({
189 			mixin(traceTask);
190 			try
191 			{
192 				stopServerSync();
193 				ret.finish();
194 			}
195 			catch (Throwable t)
196 			{
197 				ret.error(t);
198 			}
199 		});
200 		return ret;
201 	}
202 
203 	/// This will kill the process associated with the dcd-server instance
204 	void killServer()
205 	{
206 		if (serverPipes.pid && !serverPipes.pid.tryWait().terminated)
207 			serverPipes.pid.kill();
208 	}
209 
210 	/// This will stop the dcd-server safely and restart it again using setup-server asynchronously
211 	/// Returns: null
212 	Future!void restartServer(bool quiet = false)
213 	{
214 		auto ret = new Future!void;
215 		gthreads.create({
216 			mixin(traceTask);
217 			try
218 			{
219 				stopServerSync();
220 				setupServer([], quiet);
221 				ret.finish();
222 			}
223 			catch (Throwable t)
224 			{
225 				ret.error(t);
226 			}
227 		});
228 		return ret;
229 	}
230 
231 	/// This will query the current dcd-server status
232 	/// Returns: `{isRunning: bool}` If the dcd-server process is not running anymore it will return isRunning: false. Otherwise it will check for server status using `dcd-client --query`
233 	auto serverStatus() @property
234 	{
235 		DCDServerStatus status;
236 		if (serverPipes.pid && serverPipes.pid.tryWait().terminated)
237 			status.isRunning = false;
238 		else if (hasUnixDomainSockets)
239 			status.isRunning = true;
240 		else
241 			status.isRunning = isPortRunning(runningPort);
242 		return status;
243 	}
244 
245 	/// Searches for a symbol across all files using `dcd-client --search`
246 	Future!(DCDSearchResult[]) searchSymbol(string query)
247 	{
248 		auto ret = new Future!(DCDSearchResult[]);
249 		gthreads.create({
250 			mixin(traceTask);
251 			try
252 			{
253 				if (!running)
254 				{
255 					ret.finish(null);
256 					return;
257 				}
258 				auto pipes = doClient(["--search", query]);
259 				scope (exit)
260 				{
261 					pipes.pid.wait();
262 					pipes.destroy();
263 				}
264 				pipes.stdin.close();
265 				DCDSearchResult[] results;
266 				while (pipes.stdout.isOpen && !pipes.stdout.eof)
267 				{
268 					string line = pipes.stdout.readln();
269 					if (line.length == 0)
270 						continue;
271 					string[] splits = line.chomp.split('\t');
272 					if (splits.length >= 3)
273 						results ~= DCDSearchResult(splits[0], splits[2].to!int, splits[1]);
274 				}
275 				ret.finish(results);
276 			}
277 			catch (Throwable t)
278 			{
279 				ret.error(t);
280 			}
281 		});
282 		return ret;
283 	}
284 
285 	/// Reloads import paths from the current provider. Call reload there before calling it here.
286 	void refreshImports()
287 	{
288 		addImports(importPaths ~ importFiles);
289 	}
290 
291 	/// Manually adds import paths as string array
292 	void addImports(string[] imports)
293 	{
294 		knownImports ~= imports;
295 		updateImports();
296 	}
297 
298 	string clientPath() @property @ignoredFunc
299 	{
300 		return config.get("dcd", "clientPath", "dcd-client");
301 	}
302 
303 	string serverPath() @property @ignoredFunc
304 	{
305 		return config.get("dcd", "serverPath", "dcd-server");
306 	}
307 
308 	ushort port() @property @ignoredFunc
309 	{
310 		return cast(ushort) config.get!int("dcd", "port", 9166);
311 	}
312 
313 	/// Searches for an open port to spawn dcd-server in asynchronously starting with `port`, always increasing by one.
314 	/// Returns: 0 if not available, otherwise the port as number
315 	Future!ushort findAndSelectPort(ushort port = 9166)
316 	{
317 		if (hasUnixDomainSockets)
318 		{
319 			return Future!ushort.fromResult(0);
320 		}
321 		auto ret = new Future!ushort;
322 		gthreads.create({
323 			mixin(traceTask);
324 			try
325 			{
326 				auto newPort = findOpen(port);
327 				port = newPort;
328 				ret.finish(port);
329 			}
330 			catch (Throwable t)
331 			{
332 				ret.error(t);
333 			}
334 		});
335 		return ret;
336 	}
337 
338 	/// Finds the declaration of the symbol at position `pos` in the code
339 	Future!DCDDeclaration findDeclaration(scope const(char)[] code, int pos)
340 	{
341 		auto ret = new Future!DCDDeclaration;
342 		gthreads.create({
343 			mixin(traceTask);
344 			try
345 			{
346 				if (!running || pos >= code.length)
347 				{
348 					ret.finish(DCDDeclaration.init);
349 					return;
350 				}
351 
352 				// We need to move by one character on identifier characters to ensure the start character fits.
353 				if (!isIdentifierSeparatingChar(code[pos]))
354 					pos++;
355 
356 				auto pipes = doClient(["-c", pos.to!string, "--symbolLocation"]);
357 				scope (exit)
358 				{
359 					pipes.pid.wait();
360 					pipes.destroy();
361 				}
362 				pipes.stdin.write(code);
363 				pipes.stdin.close();
364 				string line = pipes.stdout.readln();
365 				if (line.length == 0)
366 				{
367 					ret.finish(DCDDeclaration.init);
368 					return;
369 				}
370 				string[] splits = line.chomp.split('\t');
371 				if (splits.length != 2)
372 				{
373 					ret.finish(DCDDeclaration.init);
374 					return;
375 				}
376 				ret.finish(DCDDeclaration(splits[0], splits[1].to!int));
377 			}
378 			catch (Throwable t)
379 			{
380 				ret.error(t);
381 			}
382 		});
383 		return ret;
384 	}
385 
386 	/// Finds the documentation of the symbol at position `pos` in the code
387 	Future!string getDocumentation(scope const(char)[] code, int pos)
388 	{
389 		auto ret = new Future!string;
390 		gthreads.create({
391 			mixin(traceTask);
392 			try
393 			{
394 				if (!running)
395 				{
396 					ret.finish("");
397 					return;
398 				}
399 				auto pipes = doClient(["--doc", "-c", pos.to!string]);
400 				scope (exit)
401 				{
402 					pipes.pid.wait();
403 					pipes.destroy();
404 				}
405 				pipes.stdin.write(code);
406 				pipes.stdin.close();
407 				string data;
408 				while (pipes.stdout.isOpen && !pipes.stdout.eof)
409 				{
410 					string line = pipes.stdout.readln();
411 					if (line.length)
412 						data ~= line.chomp;
413 				}
414 				ret.finish(data.unescapeTabs);
415 			}
416 			catch (Throwable t)
417 			{
418 				ret.error(t);
419 			}
420 		});
421 		return ret;
422 	}
423 
424 	/// Returns the used socket file. Only available on OSX, linux and BSD with DCD >= 0.8.0
425 	/// Throws an error if not available.
426 	string getSocketFile()
427 	{
428 		if (!hasUnixDomainSockets)
429 			throw new Exception("Unix domain sockets not supported");
430 		return socketFile;
431 	}
432 
433 	/// Returns the used running port. Throws an error if using unix sockets instead
434 	ushort getRunningPort()
435 	{
436 		if (hasUnixDomainSockets)
437 			throw new Exception("Using unix domain sockets instead of a port");
438 		return runningPort;
439 	}
440 
441 	/// Queries for code completion at position `pos` in code
442 	/// Raw is anything else than identifiers and calltips which might not be implemented by this point.
443 	/// calltips.symbols and identifiers.definition, identifiers.file, identifiers.location and identifiers.documentation are only available with dcd ~master as of now.
444 	Future!DCDCompletions listCompletion(scope const(char)[] code, int pos)
445 	{
446 		auto ret = new Future!DCDCompletions;
447 		gthreads.create({
448 			mixin(traceTask);
449 			try
450 			{
451 				DCDCompletions completions;
452 				if (!running)
453 				{
454 					stderr.writeln("DCD not running!");
455 					ret.finish(completions);
456 					return;
457 				}
458 				auto pipes = doClient((supportsFullOutput ? ["--extended"] : []) ~ [
459 						"-c", pos.to!string
460 					]);
461 				scope (exit)
462 				{
463 					pipes.pid.wait();
464 					pipes.destroy();
465 				}
466 				pipes.stdin.write(code);
467 				pipes.stdin.close();
468 				string[] data;
469 				while (pipes.stdout.isOpen && !pipes.stdout.eof)
470 				{
471 					string line = pipes.stdout.readln();
472 					stderr.writeln("DCD Client: ", line);
473 					if (line.length == 0)
474 						continue;
475 					data ~= line.chomp;
476 				}
477 				completions.raw = data;
478 				int[] emptyArr;
479 				if (data.length == 0)
480 				{
481 					completions.type = DCDCompletions.Type.identifiers;
482 					ret.finish(completions);
483 					return;
484 				}
485 				if (data[0] == "calltips")
486 				{
487 					if (supportsFullOutput)
488 					{
489 						foreach (line; data[1 .. $])
490 						{
491 							auto parts = line.split("\t");
492 							if (parts.length < 5)
493 								continue;
494 							completions._calltips ~= parts[2];
495 							string location = parts[3];
496 							string file;
497 							int index;
498 							if (location.length)
499 							{
500 								auto space = location.indexOf(' ');
501 								if (space != -1)
502 								{
503 									file = location[0 .. space];
504 									index = location[space + 1 .. $].to!int;
505 								}
506 							}
507 							completions._symbols ~= DCDCompletions.Symbol(file, index, parts[4].unescapeTabs);
508 						}
509 					}
510 					else
511 					{
512 						completions._calltips = data[1 .. $];
513 						completions._symbols.length = completions._calltips.length;
514 					}
515 					completions.type = DCDCompletions.Type.calltips;
516 					ret.finish(completions);
517 					return;
518 				}
519 				else if (data[0] == "identifiers")
520 				{
521 					DCDIdentifier[] identifiers;
522 					foreach (line; data[1 .. $])
523 					{
524 						string[] splits = line.split('\t');
525 						DCDIdentifier symbol;
526 						if (supportsFullOutput)
527 						{
528 							if (splits.length < 5)
529 								continue;
530 							string location = splits[3];
531 							string file;
532 							int index;
533 							if (location.length)
534 							{
535 								auto space = location.indexOf(' ');
536 								if (space != -1)
537 								{
538 									file = location[0 .. space];
539 									index = location[space + 1 .. $].to!int;
540 								}
541 							}
542 							symbol = DCDIdentifier(splits[0], splits[1], splits[2], file,
543 								index, splits[4].unescapeTabs);
544 						}
545 						else
546 						{
547 							if (splits.length < 2)
548 								continue;
549 							symbol = DCDIdentifier(splits[0], splits[1]);
550 						}
551 						identifiers ~= symbol;
552 					}
553 					completions.type = DCDCompletions.Type.identifiers;
554 					completions._identifiers = identifiers;
555 					ret.finish(completions);
556 					return;
557 				}
558 				else
559 				{
560 					completions.type = DCDCompletions.Type.raw;
561 					ret.finish(completions);
562 					return;
563 				}
564 			}
565 			catch (Throwable e)
566 			{
567 				ret.error(e);
568 			}
569 		});
570 		return ret;
571 	}
572 
573 	void updateImports()
574 	{
575 		if (!running)
576 			return;
577 		string[] args;
578 		foreach (path; knownImports)
579 			if (path.length)
580 				args ~= "-I" ~ path;
581 		execClient(args);
582 	}
583 
584 	bool fromRunning(bool supportsFullOutput, string socketFile, ushort runningPort)
585 	{
586 		if (socketFile.length ? isSocketRunning(socketFile) : isPortRunning(runningPort))
587 		{
588 			running = true;
589 			this.supportsFullOutput = supportsFullOutput;
590 			this.socketFile = socketFile;
591 			this.runningPort = runningPort;
592 			this.hasUnixDomainSockets = !!socketFile.length;
593 			return true;
594 		}
595 		else
596 			return false;
597 	}
598 
599 	bool getSupportsFullOutput() @property
600 	{
601 		return supportsFullOutput;
602 	}
603 
604 	bool isUsingUnixDomainSockets() @property
605 	{
606 		return hasUnixDomainSockets;
607 	}
608 
609 	bool isActive() @property
610 	{
611 		return running;
612 	}
613 
614 private:
615 	string installedVersion;
616 	bool supportsFullOutput;
617 	bool hasUnixDomainSockets = false;
618 	bool running = false;
619 	ProcessPipes serverPipes;
620 	ushort runningPort;
621 	string socketFile;
622 	string[] knownImports;
623 
624 	string[] clientArgs()
625 	{
626 		if (hasUnixDomainSockets)
627 			return ["--socketFile", socketFile];
628 		else
629 			return ["--port", runningPort.to!string];
630 	}
631 
632 	auto doClient(string[] args)
633 	{
634 		return raw([clientPath] ~ clientArgs ~ args);
635 	}
636 
637 	auto raw(string[] args, Redirect redirect = Redirect.all)
638 	{
639 		return pipeProcess(args, redirect, null, Config.none, refInstance ? instance.cwd : null);
640 	}
641 
642 	auto execClient(string[] args)
643 	{
644 		return rawExec([clientPath] ~ clientArgs ~ args);
645 	}
646 
647 	auto rawExec(string[] args)
648 	{
649 		return execute(args, null, Config.none, size_t.max, refInstance ? instance.cwd : null);
650 	}
651 
652 	bool isSocketRunning(string socket)
653 	{
654 		if (!hasUnixDomainSockets)
655 			return false;
656 		auto ret = execute([clientPath, "-q", "--socketFile", socket]);
657 		return ret.status == 0;
658 	}
659 
660 	bool isPortRunning(ushort port)
661 	{
662 		if (hasUnixDomainSockets)
663 			return false;
664 		auto ret = execute([clientPath, "-q", "--port", port.to!string]);
665 		return ret.status == 0;
666 	}
667 
668 	ushort findOpen(ushort port)
669 	{
670 		--port;
671 		bool isRunning;
672 		do
673 		{
674 			isRunning = isPortRunning(++port);
675 		}
676 		while (isRunning);
677 		return port;
678 	}
679 }
680 
681 bool supportsUnixDomainSockets(string ver)
682 {
683 	return checkVersion(ver, [0, 8, 0]);
684 }
685 
686 unittest
687 {
688 	assert(supportsUnixDomainSockets("0.8.0-beta2+9ec55f40a26f6bb3ca95dc9232a239df6ed25c37"));
689 	assert(!supportsUnixDomainSockets("0.7.9-beta3"));
690 	assert(!supportsUnixDomainSockets("0.7.0"));
691 	assert(supportsUnixDomainSockets("v0.9.8 c7ea7e081ed9ad2d85e9f981fd047d7fcdb2cf51"));
692 	assert(supportsUnixDomainSockets("1.0.0"));
693 }
694 
695 private string unescapeTabs(string val)
696 {
697 	return val.replace("\\t", "\t").replace("\\n", "\n").replace("\\\\", "\\");
698 }
699 
700 /// Returned by findDeclaration
701 struct DCDDeclaration
702 {
703 	string file;
704 	int position;
705 }
706 
707 /// Returned by listCompletion
708 /// When identifiers: `{type:"identifiers", identifiers:[{identifier:string, type:string, definition:string, file:string, location:number, documentation:string}]}`
709 /// When calltips: `{type:"calltips", calltips:[string], symbols:[{file:string, location:number, documentation:string}]}`
710 /// When raw: `{type:"raw", raw:[string]}`
711 struct DCDCompletions
712 {
713 	/// Type of a completion
714 	enum Type
715 	{
716 		/// Unknown/Unimplemented raw output
717 		raw,
718 		/// Completion after a dot or a variable name
719 		identifiers,
720 		/// Completion for arguments in a function call
721 		calltips,
722 	}
723 
724 	struct Symbol
725 	{
726 		string file;
727 		int location;
728 		string documentation;
729 	}
730 
731 	/// Type of the completion (identifiers, calltips, raw)
732 	Type type;
733 	/// Contains the raw DCD output
734 	string[] raw;
735 	union
736 	{
737 		DCDIdentifier[] _identifiers;
738 		struct
739 		{
740 			string[] _calltips;
741 			Symbol[] _symbols;
742 		}
743 	}
744 
745 	enum DCDCompletions empty = DCDCompletions(Type.identifiers);
746 
747 	/// Only set with type==identifiers.
748 	inout(DCDIdentifier[]) identifiers() inout @property
749 	{
750 		if (type != Type.identifiers)
751 			throw new Exception("Type is not identifiers but attempted to access identifiers");
752 		return _identifiers;
753 	}
754 
755 	/// Only set with type==calltips.
756 	inout(string[]) calltips() inout @property
757 	{
758 		if (type != Type.calltips)
759 			throw new Exception("Type is not calltips but attempted to access calltips");
760 		return _calltips;
761 	}
762 
763 	/// Only set with type==calltips.
764 	inout(Symbol[]) symbols() inout @property
765 	{
766 		if (type != Type.calltips)
767 			throw new Exception("Type is not calltips but attempted to access symbols");
768 		return _symbols;
769 	}
770 }
771 
772 /// Returned by status
773 struct DCDServerStatus
774 {
775 	///
776 	bool isRunning;
777 }
778 
779 /// Type of the identifiers value in listCompletion
780 struct DCDIdentifier
781 {
782 	///
783 	string identifier;
784 	///
785 	string type;
786 	///
787 	string definition;
788 	///
789 	string file;
790 	/// byte location
791 	int location;
792 	///
793 	string documentation;
794 }
795 
796 /// Returned by search-symbol
797 struct DCDSearchResult
798 {
799 	///
800 	string file;
801 	///
802 	int position;
803 	///
804 	string type;
805 }