1 /// Component for adding imports to a file, reading imports at a location of code and sorting imports.
2 module workspaced.com.importer;
3 
4 import dparse.ast;
5 import dparse.parser;
6 import dparse.rollback_allocator;
7 import dparse.lexer;
8 
9 import std.algorithm;
10 import std.array;
11 import std.functional;
12 import std.stdio;
13 import std.string;
14 
15 import workspaced.api;
16 
17 /// ditto
18 @component("importer")
19 class ImporterComponent : ComponentWrapper
20 {
21 	mixin DefaultComponentWrapper;
22 
23 	protected void load()
24 	{
25 		config.stringBehavior = StringBehavior.source;
26 	}
27 
28 	/// Returns all imports available at some code position.
29 	ImportInfo[] get(string code, int pos)
30 	{
31 		auto tokens = getTokensForParser(cast(ubyte[]) code, config, &workspaced.stringCache);
32 		auto mod = parseModule(tokens, "code", &rba, (&doNothing).toDelegate);
33 		auto reader = new ImporterReaderVisitor(pos);
34 		reader.visit(mod);
35 		return reader.imports;
36 	}
37 
38 	/// Returns a list of code patches for adding an import.
39 	/// If `insertOutermost` is false, the import will get added to the innermost block.
40 	ImportModification add(string importName, string code, int pos, bool insertOutermost = true)
41 	{
42 		auto tokens = getTokensForParser(cast(ubyte[]) code, config, &workspaced.stringCache);
43 		auto mod = parseModule(tokens, "code", &rba, (&doNothing).toDelegate);
44 		auto reader = new ImporterReaderVisitor(pos);
45 		reader.visit(mod);
46 		foreach (i; reader.imports)
47 		{
48 			if (i.name.join('.') == importName)
49 			{
50 				if (i.selectives.length == 0)
51 					return ImportModification(i.rename, []);
52 				else
53 					insertOutermost = false;
54 			}
55 		}
56 		string indentation = "";
57 		if (insertOutermost)
58 		{
59 			indentation = reader.outerImportLocation == 0 ? "" : (cast(ubyte[]) code)
60 				.getIndentation(reader.outerImportLocation);
61 			if (reader.isModule)
62 				indentation = '\n' ~ indentation;
63 			return ImportModification("", [CodeReplacement([reader.outerImportLocation, reader.outerImportLocation],
64 					indentation ~ "import " ~ importName ~ ";" ~ (reader.outerImportLocation == 0 ? "\n" : ""))]);
65 		}
66 		else
67 		{
68 			indentation = (cast(ubyte[]) code).getIndentation(reader.innermostBlockStart);
69 			if (reader.isModule)
70 				indentation = '\n' ~ indentation;
71 			return ImportModification("", [CodeReplacement([reader.innermostBlockStart,
72 					reader.innermostBlockStart], indentation ~ "import " ~ importName ~ ";")]);
73 		}
74 	}
75 
76 	/// Sorts the imports in a whitespace separated group of code
77 	/// Returns `ImportBlock.init` if no changes would be done.
78 	ImportBlock sortImports(string code, int pos)
79 	{
80 		bool startBlock = true;
81 		size_t start, end;
82 		// find block of code separated by empty lines
83 		foreach (line; code.lineSplitter!(KeepTerminator.yes))
84 		{
85 			if (startBlock)
86 				start = end;
87 			startBlock = line.strip.length == 0;
88 			if (startBlock && end >= pos)
89 				break;
90 			end += line.length;
91 		}
92 		if (end > start && end + 1 < code.length)
93 			end--;
94 		if (start >= end || end >= code.length)
95 			return ImportBlock.init;
96 		auto part = code[start .. end];
97 		auto tokens = getTokensForParser(cast(ubyte[]) part, config, &workspaced.stringCache);
98 		auto mod = parseModule(tokens, "code", &rba, (&doNothing).toDelegate);
99 		auto reader = new ImporterReaderVisitor(-1);
100 		reader.visit(mod);
101 		auto imports = reader.imports;
102 		auto sorted = imports.map!(a => ImportInfo(a.name, a.rename,
103 				a.selectives.dup.sort!((c, d) => icmp(c.effectiveName, d.effectiveName) < 0).array)).array.sort!((a,
104 				b) => icmp(a.effectiveName, b.effectiveName) < 0).array;
105 		if (sorted == imports)
106 			return ImportBlock.init;
107 		return ImportBlock(cast(int) start, cast(int) end, sorted);
108 	}
109 
110 private:
111 	RollbackAllocator rba;
112 	LexerConfig config;
113 }
114 
115 unittest
116 {
117 	import std.conv : to;
118 
119 	void assertEqual(A, B)(A a, B b)
120 	{
121 		assert(a == b, a.to!string ~ " is not equal to " ~ b.to!string);
122 	}
123 
124 	auto backend = new WorkspaceD();
125 	auto workspace = makeTemporaryTestingWorkspace;
126 	auto instance = backend.addInstance(workspace.directory);
127 	backend.register!ImporterComponent;
128 
129 	string code = `import std.stdio;
130 import std.algorithm;
131 import std.array;
132 import std.experimental.logger;
133 import std.regex;
134 import std.functional;
135 import std.file;
136 import std.path;
137 
138 import core.thread;
139 import core.sync.mutex;
140 
141 import gtk.HBox, gtk.VBox, gtk.MainWindow, gtk.Widget, gtk.Button, gtk.Frame,
142 	gtk.ButtonBox, gtk.Notebook, gtk.CssProvider, gtk.StyleContext, gtk.Main,
143 	gdk.Screen, gtk.CheckButton, gtk.MessageDialog, gtk.Window, gtkc.gtk,
144 	gtk.Label, gdk.Event;
145 
146 import already;
147 import sorted;
148 
149 import std.stdio : writeln, File, stdout, err = stderr;
150 
151 void main() {}`;
152 
153 	//dfmt off
154 	assertEqual(backend.get!ImporterComponent(workspace.directory).sortImports(code, 0), ImportBlock(0, 164, [
155 		ImportInfo(["std", "algorithm"]),
156 		ImportInfo(["std", "array"]),
157 		ImportInfo(["std", "experimental", "logger"]),
158 		ImportInfo(["std", "file"]),
159 		ImportInfo(["std", "functional"]),
160 		ImportInfo(["std", "path"]),
161 		ImportInfo(["std", "regex"]),
162 		ImportInfo(["std", "stdio"])
163 	]));
164 
165 	assertEqual(backend.get!ImporterComponent(workspace.directory).sortImports(code, 192), ImportBlock(166, 209, [
166 		ImportInfo(["core", "sync", "mutex"]),
167 		ImportInfo(["core", "thread"])
168 	]));
169 
170 	assertEqual(backend.get!ImporterComponent(workspace.directory).sortImports(code, 238), ImportBlock(211, 457, [
171 		ImportInfo(["gdk", "Event"]),
172 		ImportInfo(["gdk", "Screen"]),
173 		ImportInfo(["gtk", "Button"]),
174 		ImportInfo(["gtk", "ButtonBox"]),
175 		ImportInfo(["gtk", "CheckButton"]),
176 		ImportInfo(["gtk", "CssProvider"]),
177 		ImportInfo(["gtk", "Frame"]),
178 		ImportInfo(["gtk", "HBox"]),
179 		ImportInfo(["gtk", "Label"]),
180 		ImportInfo(["gtk", "Main"]),
181 		ImportInfo(["gtk", "MainWindow"]),
182 		ImportInfo(["gtk", "MessageDialog"]),
183 		ImportInfo(["gtk", "Notebook"]),
184 		ImportInfo(["gtk", "StyleContext"]),
185 		ImportInfo(["gtk", "VBox"]),
186 		ImportInfo(["gtk", "Widget"]),
187 		ImportInfo(["gtk", "Window"]),
188 		ImportInfo(["gtkc", "gtk"])
189 	]));
190 
191 	assertEqual(backend.get!ImporterComponent(workspace.directory).sortImports(code, 467), ImportBlock.init);
192 
193 	assertEqual(backend.get!ImporterComponent(workspace.directory).sortImports(code, 546), ImportBlock(491, 546, [
194 		ImportInfo(["std", "stdio"], "", [
195 			SelectiveImport("stderr", "err"),
196 			SelectiveImport("File"),
197 			SelectiveImport("stdout"),
198 			SelectiveImport("writeln"),
199 		])
200 	]));
201 	//dfmt on
202 }
203 
204 /// Information about how to add an import
205 struct ImportModification
206 {
207 	/// Set if there was already an import which was renamed. (for example import io = std.stdio; would be "io")
208 	string rename;
209 	/// Array of replacements to add the import to the code
210 	CodeReplacement[] replacements;
211 }
212 
213 /// Name and (if specified) rename of a symbol
214 struct SelectiveImport
215 {
216 	/// Original name (always available)
217 	string name;
218 	/// Rename if specified
219 	string rename;
220 
221 	/// Returns rename if set, otherwise name
222 	string effectiveName() const
223 	{
224 		return rename.length ? rename : name;
225 	}
226 
227 	/// Returns a D source code part
228 	string toString() const
229 	{
230 		return (rename.length ? rename ~ " = " : "") ~ name;
231 	}
232 }
233 
234 /// Information about one import statement
235 struct ImportInfo
236 {
237 	/// Parts of the imported module. (std.stdio -> ["std", "stdio"])
238 	string[] name;
239 	/// Available if the module has been imported renamed
240 	string rename;
241 	/// Array of selective imports or empty if the entire module has been imported
242 	SelectiveImport[] selectives;
243 
244 	/// Returns the rename if available, otherwise the name joined with dots
245 	string effectiveName() const
246 	{
247 		return rename.length ? rename : name.join('.');
248 	}
249 
250 	/// Returns D source code for this import
251 	string toString() const
252 	{
253 		import std.conv : to;
254 
255 		return "import " ~ (rename.length ? rename ~ " = "
256 				: "") ~ name.join('.') ~ (selectives.length
257 				? " : " ~ selectives.to!(string[]).join(", ") : "") ~ ';';
258 	}
259 }
260 
261 /// A block of imports generated by the sort-imports command
262 struct ImportBlock
263 {
264 	/// Start & end byte index
265 	int start, end;
266 	///
267 	ImportInfo[] imports;
268 }
269 
270 private:
271 
272 string getIndentation(ubyte[] code, size_t index)
273 {
274 	import std.ascii : isWhite;
275 
276 	bool atLineEnd = false;
277 	if (index < code.length && code[index] == '\n')
278 	{
279 		for (size_t i = index; i < code.length; i++)
280 			if (!code[i].isWhite)
281 				break;
282 		atLineEnd = true;
283 	}
284 	while (index > 0)
285 	{
286 		if (code[index - 1] == cast(ubyte) '\n')
287 			break;
288 		index--;
289 	}
290 	size_t end = index;
291 	while (end < code.length)
292 	{
293 		if (!code[end].isWhite)
294 			break;
295 		end++;
296 	}
297 	auto indent = cast(string) code[index .. end];
298 	if (!indent.length && index == 0 && !atLineEnd)
299 		return " ";
300 	return "\n" ~ indent.stripLeft('\n');
301 }
302 
303 unittest
304 {
305 	auto code = cast(ubyte[]) "void foo() {\n\tfoo();\n}";
306 	auto indent = getIndentation(code, 20);
307 	assert(indent == "\n\t", '"' ~ indent ~ '"');
308 
309 	code = cast(ubyte[]) "void foo() { foo(); }";
310 	indent = getIndentation(code, 19);
311 	assert(indent == " ", '"' ~ indent ~ '"');
312 
313 	code = cast(ubyte[]) "import a;\n\nvoid foo() {\n\tfoo();\n}";
314 	indent = getIndentation(code, 9);
315 	assert(indent == "\n", '"' ~ indent ~ '"');
316 }
317 
318 class ImporterReaderVisitor : ASTVisitor
319 {
320 	this(int pos)
321 	{
322 		this.pos = pos;
323 		inBlock = false;
324 	}
325 
326 	alias visit = ASTVisitor.visit;
327 
328 	override void visit(const ModuleDeclaration decl)
329 	{
330 		if (pos != -1 && (decl.endLocation + 1 < outerImportLocation || inBlock))
331 			return;
332 		isModule = true;
333 		outerImportLocation = decl.endLocation + 1;
334 	}
335 
336 	override void visit(const ImportDeclaration decl)
337 	{
338 		if (pos != -1 && decl.startIndex >= pos)
339 			return;
340 		isModule = false;
341 		if (inBlock)
342 			innermostBlockStart = decl.endIndex;
343 		else
344 			outerImportLocation = decl.endIndex;
345 		foreach (i; decl.singleImports)
346 			imports ~= ImportInfo(i.identifierChain.identifiers.map!(tok => tok.text.idup)
347 					.array, i.rename.text);
348 		if (decl.importBindings)
349 		{
350 			ImportInfo info;
351 			if (!decl.importBindings.singleImport)
352 				return;
353 			info.name = decl.importBindings.singleImport.identifierChain.identifiers.map!(
354 					tok => tok.text.idup).array;
355 			info.rename = decl.importBindings.singleImport.rename.text;
356 			foreach (bind; decl.importBindings.importBinds)
357 			{
358 				if (bind.right.text)
359 					info.selectives ~= SelectiveImport(bind.right.text, bind.left.text);
360 				else
361 					info.selectives ~= SelectiveImport(bind.left.text);
362 			}
363 			if (info.selectives.length)
364 				imports ~= info;
365 		}
366 	}
367 
368 	override void visit(const BlockStatement content)
369 	{
370 		if (pos == -1 || content && pos >= content.startLocation && pos < content.endLocation)
371 		{
372 			if (content.startLocation + 1 >= innermostBlockStart)
373 				innermostBlockStart = content.startLocation + 1;
374 			inBlock = true;
375 			return content.accept(this);
376 		}
377 	}
378 
379 	private int pos;
380 	private bool inBlock;
381 	ImportInfo[] imports;
382 	bool isModule;
383 	size_t outerImportLocation;
384 	size_t innermostBlockStart;
385 }
386 
387 void doNothing(string, size_t, size_t, string, bool)
388 {
389 }
390 
391 unittest
392 {
393 	import std.conv;
394 
395 	auto backend = new WorkspaceD();
396 	auto workspace = makeTemporaryTestingWorkspace;
397 	auto instance = backend.addInstance(workspace.directory);
398 	backend.register!ImporterComponent;
399 	auto imports = backend.get!ImporterComponent(workspace.directory).get("import std.stdio; void foo() { import fs = std.file; import std.algorithm : map, each2 = each; writeln(\"hi\"); } void bar() { import std.string; import std.regex : ctRegex; }",
400 			81);
401 	bool equalsImport(ImportInfo i, string s)
402 	{
403 		return i.name.join('.') == s;
404 	}
405 
406 	void assertEquals(T)(T a, T b)
407 	{
408 		assert(a == b, "'" ~ a.to!string ~ "' != '" ~ b.to!string ~ "'");
409 	}
410 
411 	assertEquals(imports.length, 3);
412 	assert(equalsImport(imports[0], "std.stdio"));
413 	assert(equalsImport(imports[1], "std.file"));
414 	assertEquals(imports[1].rename, "fs");
415 	assert(equalsImport(imports[2], "std.algorithm"));
416 	assertEquals(imports[2].selectives.length, 2);
417 	assertEquals(imports[2].selectives[0].name, "map");
418 	assertEquals(imports[2].selectives[1].name, "each");
419 	assertEquals(imports[2].selectives[1].rename, "each2");
420 
421 	string code = "void foo() { import std.stdio : stderr; writeln(\"hi\"); }";
422 	auto mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 45);
423 	assertEquals(mod.rename, "");
424 	assertEquals(mod.replacements.length, 1);
425 	assertEquals(mod.replacements[0].apply(code),
426 			"void foo() { import std.stdio : stderr; import std.stdio; writeln(\"hi\"); }");
427 
428 	code = "void foo() {\n\timport std.stdio : stderr;\n\twriteln(\"hi\");\n}";
429 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 45);
430 	assertEquals(mod.rename, "");
431 	assertEquals(mod.replacements.length, 1);
432 	assertEquals(mod.replacements[0].apply(code),
433 			"void foo() {\n\timport std.stdio : stderr;\n\timport std.stdio;\n\twriteln(\"hi\");\n}");
434 
435 	code = "void foo() {\n\timport std.file : readText;\n\twriteln(\"hi\");\n}";
436 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 45);
437 	assertEquals(mod.rename, "");
438 	assertEquals(mod.replacements.length, 1);
439 	assertEquals(mod.replacements[0].apply(code),
440 			"import std.stdio;\nvoid foo() {\n\timport std.file : readText;\n\twriteln(\"hi\");\n}");
441 
442 	code = "void foo() { import io = std.stdio; io.writeln(\"hi\"); }";
443 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 45);
444 	assertEquals(mod.rename, "io");
445 	assertEquals(mod.replacements.length, 0);
446 
447 	code = "import std.file : readText;\n\nvoid foo() {\n\twriteln(\"hi\");\n}";
448 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 45);
449 	assertEquals(mod.rename, "");
450 	assertEquals(mod.replacements.length, 1);
451 	assertEquals(mod.replacements[0].apply(code),
452 			"import std.file : readText;\nimport std.stdio;\n\nvoid foo() {\n\twriteln(\"hi\");\n}");
453 
454 	code = "import std.file;\nimport std.regex;\n\nvoid foo() {\n\twriteln(\"hi\");\n}";
455 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 54);
456 	assertEquals(mod.rename, "");
457 	assertEquals(mod.replacements.length, 1);
458 	assertEquals(mod.replacements[0].apply(code),
459 			"import std.file;\nimport std.regex;\nimport std.stdio;\n\nvoid foo() {\n\twriteln(\"hi\");\n}");
460 
461 	code = "module a;\n\nvoid foo() {\n\twriteln(\"hi\");\n}";
462 	mod = backend.get!ImporterComponent(workspace.directory).add("std.stdio", code, 30);
463 	assertEquals(mod.rename, "");
464 	assertEquals(mod.replacements.length, 1);
465 	assertEquals(mod.replacements[0].apply(code),
466 			"module a;\n\nimport std.stdio;\n\nvoid foo() {\n\twriteln(\"hi\");\n}");
467 }