-
-
Notifications
You must be signed in to change notification settings - Fork 30
Expand file tree
/
Copy pathtest_codegraph.py
More file actions
456 lines (368 loc) · 16.5 KB
/
Copy pathtest_codegraph.py
File metadata and controls
456 lines (368 loc) · 16.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
"""Tests for the code-symbol graph extraction backends.
``RegexSymbolIndexer`` and ``get_code_indexer``/``detect_lang``/``iter_source_files``
are dependency-free and run in the offline numpy-only gate. The tree-sitter-specific
tests skip cleanly when the optional ``tree_sitter_language_pack`` extra isn't
installed, so they never affect that gate.
"""
import os
import pytest
from engraphis.backends.codegraph import (
CompositeSymbolIndexer,
FileIndex,
RegexSymbolIndexer,
Symbol,
detect_lang,
get_code_indexer,
iter_source_files,
normalize_language,
source_path_allowed,
supported_languages,
)
PY_SRC = """
def add(a, b):
return a + b
class Calculator:
def __init__(self):
self.total = 0
def add(self, x):
self.total = add(self.total, x)
return self.total
import os
from collections import OrderedDict
"""
def test_detect_lang_by_extension():
assert detect_lang("foo.py") == "python"
assert detect_lang("foo.tsx") == "typescript"
assert detect_lang("foo.jsx") == "javascript"
assert detect_lang("main.go") == "go"
assert detect_lang("lib.rs") == "rust"
assert detect_lang("Service.java") == "java"
assert detect_lang("schema.sql") == "sql"
assert detect_lang("main.tf") == "terraform"
assert detect_lang("foo.txt") is None
def test_get_code_indexer_regex_forced():
idx = get_code_indexer(prefer="regex")
assert isinstance(idx, RegexSymbolIndexer)
def test_get_code_indexer_auto_never_raises():
# Whatever is installed, "auto" must hand back something usable, never crash —
# the whole point of gating a heavy/fragile optional dependency behind a factory.
idx = get_code_indexer(prefer="auto")
assert idx.supports("python")
def test_regex_indexer_finds_function_and_class():
idx = RegexSymbolIndexer()
fi = idx.index_file("calc.py", PY_SRC, "python")
names = {s.name for s in fi.symbols}
assert "add" in names and "Calculator" in names
def test_regex_indexer_extracts_variables_comments_and_inheritance():
source = """
# Main service implementation.
class Service(BaseService):
pass
# Maximum retry count.
MAX_RETRIES = 4
"""
result = RegexSymbolIndexer().index_file("service.py", source, "python")
by_name = {symbol.name: symbol for symbol in result.symbols}
assert by_name["Service"].docstring == "Main service implementation."
assert by_name["MAX_RETRIES"].kind == "variable"
assert by_name["MAX_RETRIES"].docstring == "Maximum retry count."
assert any(
edge.src == "Service" and edge.dst == "BaseService"
and edge.relation == "inherits"
for edge in result.edges
)
def test_regex_indexer_unsupported_language_returns_empty():
idx = RegexSymbolIndexer()
fi = idx.index_file("calc.rb", "def add(a,b)\n a+b\nend\n", "ruby")
assert fi.symbols == [] and fi.edges == []
def test_iter_source_files_skips_excluded_dirs(tmp_path):
(tmp_path / "src").mkdir()
(tmp_path / "src" / "a.py").write_text("def a(): pass\n")
(tmp_path / "node_modules").mkdir()
(tmp_path / "node_modules" / "b.py").write_text("def b(): pass\n")
(tmp_path / "readme.md").write_text("not code")
found = sorted(iter_source_files(str(tmp_path)))
assert any(f.endswith("a.py") for f in found)
assert not any("node_modules" in f for f in found)
assert not any(f.endswith("readme.md") for f in found)
# ── C# / C / C++ support (regex path — the one the offline gate exercises) ──────
CSHARP_SRC = """
namespace App {
public class Service {
private readonly int _n;
public Service(int n) { _n = n; }
public int Compute(int x) {
if (x > 0) return x;
return 0;
}
internal static string Name() => "svc";
}
public interface IThing { }
}
"""
CPP_SRC = """
class Widget {
public:
int area() {
return w * h;
}
};
int add(int a, int b) {
return a + b;
}
void loop() {
for (int i = 0; i < 10; i++) {
add(i, i);
}
}
"""
def test_detect_lang_covers_csharp_c_cpp():
assert detect_lang("Foo.cs") == "csharp"
assert detect_lang("foo.cpp") == "cpp"
assert detect_lang("foo.hpp") == "cpp"
assert detect_lang("foo.c") == "c"
def test_regex_indexer_csharp_types_and_methods():
idx = RegexSymbolIndexer()
fi = idx.index_file("Service.cs", CSHARP_SRC, "csharp")
names = {s.name for s in fi.symbols}
assert {"Service", "IThing", "Compute", "Name"} <= names
# control-flow must not be mistaken for a definition
assert "if" not in names
def test_regex_indexer_cpp_classes_and_functions_no_false_positives():
idx = RegexSymbolIndexer()
fi = idx.index_file("widget.cpp", CPP_SRC, "cpp")
names = {s.name for s in fi.symbols}
assert {"Widget", "area", "add", "loop"} <= names
# loops/calls are not definitions
assert "for" not in names and "if" not in names
def test_normalize_language_aliases():
assert normalize_language("C#") == "csharp"
assert normalize_language("c++") == "cpp"
assert normalize_language("Python") == "python"
assert normalize_language("rust") == "rust"
assert normalize_language("golang") == "go"
assert normalize_language("hcl") == "terraform"
def test_supported_languages_set():
langs = supported_languages()
assert {
"python", "javascript", "typescript", "go", "rust", "java",
"csharp", "c", "cpp", "sql", "terraform",
} <= langs
assert "ruby" not in langs
@pytest.mark.parametrize(
("lang", "filename", "source", "expected"),
[
("go", "main.go", "func Build() {}\ntype Config struct {}\n", {"Build", "Config"}),
("rust", "lib.rs", "pub fn build() {}\npub struct Config {}\n", {"build", "Config"}),
("java", "App.java", "public class App {}\npublic static void run() {}\n",
{"App", "run"}),
("sql", "schema.sql", "CREATE TABLE public.users (id INTEGER);\n",
{"public.users"}),
("terraform", "main.tf", 'resource "aws_s3_bucket" "logs" {\n}\n',
{"aws_s3_bucket.logs"}),
],
)
def test_regex_indexer_new_languages(lang, filename, source, expected):
fi = RegexSymbolIndexer().index_file(filename, source, lang)
assert expected <= {symbol.name for symbol in fi.symbols}
assert all(edge.relation == "defines" for edge in fi.edges)
# ── the hang fix: build/generated trees are pruned during the walk ──────────────
def test_iter_source_files_skips_build_output_dirs(tmp_path):
(tmp_path / "src").mkdir()
(tmp_path / "src" / "a.cs").write_text("class A {}\n")
for d in ("bin", "obj", "target"):
(tmp_path / d).mkdir()
(tmp_path / d / "g.cs").write_text("class G {}\n")
found = [f.replace(os.sep, "/") for f in iter_source_files(str(tmp_path))]
assert any(f.endswith("src/a.cs") for f in found)
assert not any("/bin/" in f or "/obj/" in f or "/target/" in f for f in found)
# ── .engraphisignore: names, globs, and project-level negations ────────────────
def test_engraphisignore_names_and_globs(tmp_path):
(tmp_path / "src").mkdir()
(tmp_path / "src" / "keep.py").write_text("def keep(): pass\n")
(tmp_path / "generated").mkdir()
(tmp_path / "generated" / "gen.py").write_text("def gen(): pass\n")
(tmp_path / "a.gen.py").write_text("def a(): pass\n")
(tmp_path / ".engraphisignore").write_text(
"# project ignores\n"
"generated\n" # bare name → skip this dir/file anywhere
"*.gen.py\n" # glob → skip generated sources
)
found = [f.replace(os.sep, "/") for f in iter_source_files(str(tmp_path))]
assert any(f.endswith("src/keep.py") for f in found)
assert not any("/generated/" in f for f in found) # name ignore worked
assert not any(f.endswith("a.gen.py") for f in found) # glob ignore worked
def test_engraphisignore_negation_cancels_own_pattern(tmp_path):
# `!name` re-includes a name the ignore file itself excluded (gitignore-style).
(tmp_path / "logs").mkdir()
(tmp_path / "logs" / "nested.py").write_text("def nested(): pass\n")
(tmp_path / "keep.py").write_text("def keep(): pass\n")
(tmp_path / ".engraphisignore").write_text(
"logs\n!logs\nkeep.py\n!keep.py\n"
)
found = {
os.path.relpath(path, tmp_path).replace(os.sep, "/")
for path in iter_source_files(str(tmp_path))
}
assert found == {"keep.py", "logs/nested.py"}
assert source_path_allowed(str(tmp_path), str(tmp_path / "keep.py"))
def test_engraphisignore_bare_negation_overrides_matching_glob(tmp_path):
keep = tmp_path / "keep.py"
drop = tmp_path / "drop.py"
keep.write_text("def keep(): pass\n")
drop.write_text("def drop(): pass\n")
(tmp_path / ".engraphisignore").write_text("*.py\n!keep.py\n")
found = {os.path.basename(path) for path in iter_source_files(str(tmp_path))}
assert found == {"keep.py"}
assert source_path_allowed(str(tmp_path), str(keep))
assert not source_path_allowed(str(tmp_path), str(drop))
def test_engraphisignore_cannot_re_expose_hardcoded_default(tmp_path):
# SECURITY: an untrusted repo's ignore file must NOT be able to un-ignore a default
# excluded dir (that would reintroduce the large-tree hang and pull in vendored code).
(tmp_path / "node_modules").mkdir()
(tmp_path / "node_modules" / "evil.py").write_text("def evil(): pass\n")
(tmp_path / "build").mkdir()
(tmp_path / "build" / "out.py").write_text("def out(): pass\n")
(tmp_path / ".engraphisignore").write_text("!node_modules\n!build\n")
found = [f.replace(os.sep, "/") for f in iter_source_files(str(tmp_path))]
assert not any("/node_modules/" in f for f in found)
assert not any("/build/" in f for f in found)
def test_symlinked_file_is_not_followed_out_of_root(tmp_path):
# SECURITY: a source-extension symlink pointing outside the repo must not be read.
outside = tmp_path.parent / "secret.py"
outside.write_text("SECRET = 1\n")
repo = tmp_path / "repo"
repo.mkdir()
(repo / "real.py").write_text("def real(): pass\n")
try:
os.symlink(outside, repo / "leak.py")
except (OSError, NotImplementedError):
pytest.skip("symlinks not supported on this platform")
found = [f.replace(os.sep, "/") for f in iter_source_files(str(repo))]
assert any(f.endswith("real.py") for f in found)
assert not any(f.endswith("leak.py") for f in found)
def test_engraphisignore_can_be_disabled(tmp_path):
(tmp_path / "generated").mkdir()
(tmp_path / "generated" / "gen.py").write_text("def gen(): pass\n")
(tmp_path / ".engraphisignore").write_text("generated\n")
found = list(iter_source_files(str(tmp_path), respect_ignore_file=False))
assert any(f.replace(os.sep, "/").endswith("generated/gen.py") for f in found)
def test_oversized_ignore_file_is_ignored_safely(tmp_path):
(tmp_path / "src").mkdir()
(tmp_path / "src" / "a.py").write_text("def a(): pass\n")
# a pathological ignore file must not be honoured (and must not hang)
(tmp_path / ".engraphisignore").write_text("src\n" * 200_000)
found = list(iter_source_files(str(tmp_path)))
assert any(f.endswith("a.py") for f in found)
# ── composite indexer: AST where supported, regex fallback per-language ──────────
class _PythonOnlyPrimary:
def supports(self, lang):
return lang == "python"
def index_file(self, file_path, content, lang):
return FileIndex(symbols=[Symbol(kind="function", name="PRIMARY", fqname="PRIMARY",
file=file_path, span="1-1", lang=lang)])
def test_composite_routes_by_language():
comp = CompositeSymbolIndexer(_PythonOnlyPrimary(), RegexSymbolIndexer())
assert comp.supports("python") and comp.supports("csharp")
# python → primary (AST)
assert comp.index_file("a.py", "x", "python").symbols[0].name == "PRIMARY"
# csharp → regex fallback
fi = comp.index_file("A.cs", "public class Foo { }\n", "csharp")
assert any(s.name == "Foo" for s in fi.symbols)
# ── service layer: unsupported language is an actionable error, not a silent 0 ───
def test_index_repo_rejects_unsupported_language(tmp_path):
from engraphis.service import MemoryService, ValidationError
svc = MemoryService.create(":memory:")
(tmp_path / "a.py").write_text("def a(): pass\n")
with pytest.raises(ValidationError):
svc.index_repo(workspace="w", repo="r", root_path=str(tmp_path), languages=["ruby"])
def test_index_repo_accepts_normalized_language_alias(tmp_path):
from engraphis.service import MemoryService
svc = MemoryService.create(":memory:")
(tmp_path / "Svc.cs").write_text("public class Svc { }\n")
res = svc.index_repo(workspace="w", repo="r", root_path=str(tmp_path), languages=["C#"])
assert res["files_indexed"] >= 1 and res["symbols"] >= 1
# ── tree-sitter-specific behavior (skipped if the optional extra isn't installed) ──
#
# NB: this must be a *per-test* skip, not a module-level ``pytest.importorskip`` — the
# latter skips the ENTIRE module when tree-sitter is absent, which silently dropped all
# the dependency-free regex/ignore tests above from the offline numpy-only gate (exactly
# the environment CI runs in). Guard only the AST tests so the offline ones always run.
try: # pragma: no cover - optional package and its grammars may be unavailable
import tree_sitter_language_pack
# The package is installed as an optional extra, but grammar binaries are fetched
# lazily. Treat an unavailable grammar like an unavailable optional extra so a
# transient certificate/network failure does not turn the full offline suite red;
# CompositeSymbolIndexer already guarantees the dependency-free fallback path.
tree_sitter_language_pack.get_parser("python")
_HAS_TREE_SITTER = True
except Exception:
_HAS_TREE_SITTER = False
_needs_tree_sitter = pytest.mark.skipif(
not _HAS_TREE_SITTER,
reason="optional code-graph grammar (tree-sitter) is unavailable",
)
@_needs_tree_sitter
def test_tree_sitter_indexer_extracts_qualified_names_and_edges():
from engraphis.backends.codegraph import TreeSitterSymbolIndexer
idx = TreeSitterSymbolIndexer()
fi = idx.index_file("calc.py", PY_SRC, "python")
fqnames = {s.fqname for s in fi.symbols}
assert "add" in fqnames # top-level function
assert "Calculator" in fqnames # class
assert "Calculator.add" in fqnames # method, qualified by its class
assert any(e.relation == "calls" and e.dst == "add" for e in fi.edges)
assert any(e.relation == "imports" and e.dst == "os" for e in fi.edges)
@_needs_tree_sitter
def test_tree_sitter_indexer_extracts_every_grouped_import_module():
from engraphis.backends.codegraph import TreeSitterSymbolIndexer
indexer = TreeSitterSymbolIndexer()
python = indexer.index_file(
"imports.py",
"import os, sys as system\nfrom pkg import alpha, beta as bee\n",
"python",
)
go = indexer.index_file(
"imports.go",
'package imports\nimport (\n "fmt"\n http "net/http"\n)\n',
"go",
)
assert [
edge.dst for edge in python.edges if edge.relation == "imports"
] == ["os", "sys", "pkg"]
assert [
edge.dst for edge in go.edges if edge.relation == "imports"
] == ["fmt", "net/http"]
@_needs_tree_sitter
def test_tree_sitter_indexer_extracts_docstrings_and_class_variables():
from engraphis.backends.codegraph import TreeSitterSymbolIndexer
source = '''
class Child(Base):
"""Coordinates child operations."""
MAX_RETRIES = 3
def run(self):
"""Run one operation."""
return True
'''
result = TreeSitterSymbolIndexer().index_file("child.py", source, "python")
by_name = {symbol.fqname: symbol for symbol in result.symbols}
assert by_name["Child"].docstring == "Coordinates child operations."
assert by_name["Child.MAX_RETRIES"].kind == "variable"
assert by_name["Child.run"].docstring == "Run one operation."
assert any(
edge.src == "Child" and edge.dst == "Base" and edge.relation == "inherits"
for edge in result.edges
)
@_needs_tree_sitter
def test_tree_sitter_indexer_javascript():
from engraphis.backends.codegraph import TreeSitterSymbolIndexer
idx = TreeSitterSymbolIndexer()
js_src = (
"function add(a, b) { return a + b; }\n"
"class Calc {\n"
" addOne(x) { return add(x, 1); }\n"
"}\n"
)
fi = idx.index_file("calc.js", js_src, "javascript")
fqnames = {s.fqname for s in fi.symbols}
assert "add" in fqnames and "Calc.addOne" in fqnames