".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv", "env", "dist",

"build", "target", ".next", ".nuxt", ".idea", ".vscode", ".tox", ".mypy_cache",

".pytest_cache", ".gradle", "vendor", "coverage", ".cache", "site-packages", "dist-packages",

".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".pdf", ".zip", ".gz", ".tar",

".tgz", ".bz2", ".xz", ".7z", ".jar", ".class", ".exe", ".dll", ".so", ".dylib", ".o",

".a", ".pyc", ".pyo", ".woff", ".woff2", ".ttf", ".eot", ".mp3", ".mp4", ".mov", ".avi",

".wasm", ".lock", ".map", ".svg", ".sqlite", ".db", ".bin", ".pkl", ".npy", ".parquet",

"lazy": ["defer", "deferred", "ondemand", "dynamic"],

"load": ["import", "fetch", "require", "init"],

"auth": ["authenticate", "login", "token", "session", "credential"],

"authentication": ["auth", "login", "token", "session"],

"authorization": ["permission", "role", "policy", "acl"],

"config": ["configuration", "settings", "options", "env"],

"db": ["database", "sql", "query", "orm"],

"database": ["db", "sql", "query", "orm"],

"error": ["exception", "raise", "throw", "fail"],

"cache": ["memoize", "ttl", "lru", "store"],

"log": ["logger", "logging", "trace"],

"test": ["spec", "mock", "assert", "fixture"],

"delete": ["remove", "destroy", "drop"],

"create": ["build", "make", "init", "construct"],

"parse": ["decode", "deserialize", "tokenize"],

"render": ["draw", "paint", "display"],

"route": ["router", "endpoint", "handler", "url"],

"async": ["await", "promise", "future", "coroutine"],

"send": ["emit", "dispatch", "publish"],

"retry": ["backoff", "attempt", "reconnect"],

"validate": ["verify", "check", "sanitize", "schema"],

"queue": ["worker", "job", "task"],

"http": ["request", "response", "client"],

"thread": ["worker", "pool", "lock", "concurrent"],

"serialize": ["encode", "dump", "marshal"],

".py": "python", ".js": "javascript", ".jsx": "jsx", ".ts": "typescript", ".tsx": "tsx",

".java": "java", ".go": "go", ".rs": "rust", ".rb": "ruby", ".php": "php", ".c": "c",

".h": "c", ".cpp": "cpp", ".hpp": "cpp", ".cc": "cpp", ".cs": "csharp", ".kt": "kotlin",

".swift": "swift", ".scala": "scala", ".sh": "bash", ".sql": "sql", ".md": "markdown",

".json": "json", ".yml": "yaml", ".yaml": "yaml", ".toml": "toml", ".html": "html",

".css": "css", ".vue": "vue", ".lua": "lua", ".dart": "dart",

r"^[ \t]*" + _MODS +

r"(?P<kind>def|class|function\*?|func|fn|interface|struct|enum|trait|type|object|module)"

r"[ \t]+(?:\([^)\n]*\)[ \t]*)?(?P<name>[A-Za-z_$][\w$]*)"

r"|^[ \t]*(?:export[ \t]+)?(?:const|let|var)[ \t]+(?P<cname>[A-Za-z_$][\w$]*)[ \t]*"

r"(?::[^=\n]+)?=[ \t]*(?:async[ \t]*)?(?:\([^)\n]*\)|\w+)[ \t]*=>",

w = w.lower()

for suf in ("ations", "ation", "ings", "ing", "ers", "er", "ies", "ed", "es", "s"):

if w.endswith(suf) and len(w) - len(suf) >= 4:

return w[: -len(suf)]

if w.endswith("e") and len(w) >= 5:

return w[:-1]

"""Token -> term frequency. Whole identifiers AND their camel/snake sub-words are indexed."""

raw = Counter(IDENT_RX.findall(text)) # C-speed pass; then work on unique identifiers only

tf: Counter = Counter()

total = 0

for ident, n in raw.items():

total += n

if len(ident) > 64:

continue

low = ident.lower()

if len(low) >= 2:

tf[low] += n

subs = _subwords(ident)

if len(subs) > 1 or (subs and subs[0] != low):

for s in subs:

tf[s] += n

"""(name, kind) if the line declares a function/class-like symbol, else None."""

m = SYMBOL_RX.match(line)

if not m:

return None

name = m.group("name") or m.group("cname")

if not name or name.lower() in _NOT_SYMBOL:

return None

out, last_pos, line = [], 0, 1

for m in SYMBOL_RX.finditer(text):

line += text.count("\n", last_pos, m.start())

last_pos = m.start()

name = m.group("name") or m.group("cname")

if name and name.lower() not in _NOT_SYMBOL:

out.append((name, (m.group("kind") or "function").rstrip("*"), line))

if len(out) >= 3000:

break

root, rel = args

full = os.path.join(root, rel)

try:

st = os.stat(full)

if st.st_size == 0 or st.st_size > MAX_FILE_BYTES:

return rel, st.st_mtime_ns, st.st_size, None

with open(full, "rb") as fh:

data = fh.read()

except OSError:

return rel, 0, 0, None

if b"\0" in data[:4096] or len(data) / (data.count(b"\n") + 1) > 400: # binary / minified

return rel, st.st_mtime_ns, st.st_size, None

text = data.decode("utf-8", "replace")

tf, ntok = index_text(text)

files = None

try:

out = subprocess.run(["git", "-C", root, "ls-files", "-z", "-co", "--exclude-standard"],

capture_output=True, timeout=60)

if out.returncode == 0 and out.stdout:

files = out.stdout.decode("utf-8", "replace").split("\0")

except (OSError, subprocess.SubprocessError):

pass

if files is None:

files, stack = [], [root]

while stack:

d = stack.pop()

try:

with os.scandir(d) as it:

for e in it:

if e.is_dir(follow_symlinks=False):

if e.name not in IGNORE_DIRS and not e.name.startswith("."):

stack.append(e.path)

elif e.is_file(follow_symlinks=False):

files.append(os.path.relpath(e.path, root))

except OSError:

continue

keep = []

for f in files:

if not f:

continue

f = f.replace("\\", "/")

parts = f.split("/")

name = parts[-1]

if any(p in IGNORE_DIRS or p.endswith((".dist-info", ".egg-info")) for p in parts[:-1]) \

or name in SKIP_NAMES:

continue

if os.path.splitext(name)[1].lower() in BINARY_EXT or name.endswith(".min.js"):

continue

keep.append(f)

base = os.environ.get("FC_CACHE_DIR") or os.path.join(

os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache"), "fastcontext")

try:

os.makedirs(base, exist_ok=True)

except OSError:

base = os.path.join(tempfile.gettempdir(), "fastcontext-cache")

os.makedirs(base, exist_ok=True)

"""Substring search across many short strings using ONE big str (C-speed str.find)."""

__slots__ = ("blob", "starts")

def __init__(self, items: list[str]):

self.blob = "\n" + "\n".join(items) + "\n"

self.starts = list(accumulate((len(s) + 1 for s in items), initial=1))

def find(self, sub: str, limit: int = 20000) -> list[int]:

blob, starts, out = self.blob, self.starts, []

pos = blob.find(sub)

while pos != -1:

i = bisect.bisect_right(starts, pos) - 1

out.append(i)

if len(out) >= limit:

break

pos = blob.find(sub, starts[i + 1])

"""Persistent, incrementally-updated inverted index (SQLite)."""

def __init__(self, root: str, use_cache: bool = True, reindex: bool = False, log=lambda *a: None):

self.root, self.log = root, log

self.path = cache_path(root) if use_cache else ":memory:"

if reindex and use_cache:

for suffix in ("", "-wal", "-shm"):

try:

os.remove(self.path + suffix)

except OSError:

pass

self.db = sqlite3.connect(self.path)

self.db.execute("PRAGMA synchronous=OFF")

self.db.execute("PRAGMA cache_size=-131072")

if use_cache:

self.db.execute("PRAGMA journal_mode=WAL")

self._schema()

def _schema(self):

db = self.db

db.execute("CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)")

row = db.execute("SELECT v FROM meta WHERE k='version'").fetchone()

if row and row[0] != str(VERSION):

db.executescript("DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS vocab; "

"DROP TABLE IF EXISTS postings; DROP TABLE IF EXISTS symbols; DELETE FROM meta;")

row = None

db.executescript("""

""")

if not row:

db.execute("INSERT OR REPLACE INTO meta VALUES('version', ?)", (str(VERSION),))

db.commit()

def _load_vocab(self) -> list[str]:

"""Vocabulary is one newline-joined string in meta; token i has id i+1 (append-only)."""

row = self.db.execute("SELECT v FROM meta WHERE k='vocab'").fetchone()

return row[0].split("\n") if row and row[0] else []

# ---- incremental sync ------------------------------------------------- #

def sync(self) -> dict:

db = self.db

now = {}

for rel in list_files(self.root):

try:

st = os.stat(os.path.join(self.root, rel))

except OSError:

continue

now[rel] = (st.st_mtime_ns, st.st_size)

known = {p: (i, m, s) for i, p, m, s in db.execute("SELECT id, path, mtime, size FROM files")}

changed = [p for p, ms in now.items() if p not in known or known[p][1:] != ms]

removed = [known[p][0] for p in known if p not in now]

stats = {"files": len(now), "changed": len(changed), "removed": len(removed)}

for i in range(0, len(removed), 500):

chunk = removed[i:i + 500]

q = ",".join("?" * len(chunk))

for table, col in (("postings", "fid"), ("symbols", "fid"), ("files", "id")):

db.execute(f"DELETE FROM {table} WHERE {col} IN ({q})", chunk)

if changed:

if len(changed) >= 500:

print(f"[FastContext] indexing {len(changed):,} files (cached for next time)...", file=sys.stderr)

args = [(self.root, p) for p in changed]

if len(changed) > 300 and (os.cpu_count() or 1) > 1:

with ProcessPoolExecutor(max_workers=min(os.cpu_count() or 2, 8)) as ex:

results = list(ex.map(_index_worker, args, chunksize=64))

else:

results = [_index_worker(a) for a in args]

bulk = len(changed) >= 1500

if bulk: # cold build: sorted bulk insert is far faster than random B-tree inserts

old = [known[p][0] for p in changed if p in known]

for i in range(0, len(old), 500):

chunk = old[i:i + 500]

q = ",".join("?" * len(chunk))

db.execute(f"DELETE FROM postings WHERE fid IN ({q})", chunk)

db.execute(f"DELETE FROM symbols WHERE fid IN ({q})", chunk)

db.execute("DROP INDEX IF EXISTS postings_fid")

db.execute("CREATE TEMP TABLE post_stage(tid INTEGER, fid INTEGER, tf INTEGER)")

target = "post_stage" if bulk else "postings"

vtoks = self._load_vocab()

tid_map = {tok: i + 1 for i, tok in enumerate(vtoks)}

next_tid = len(vtoks) + 1

vocab_grew = False

for rel, mtime, size, payload in results:

if rel in known:

fid = known[rel][0]

if not bulk:

db.execute("DELETE FROM postings WHERE fid=?", (fid,))

db.execute("DELETE FROM symbols WHERE fid=?", (fid,))

db.execute("UPDATE files SET mtime=?, size=?, ntok=? WHERE id=?",

(mtime, size, payload[1] if payload else 0, fid))

else:

fid = db.execute("INSERT INTO files(path, mtime, size, ntok) VALUES (?,?,?,?)",

(rel, mtime, size, payload[1] if payload else 0)).lastrowid

if not payload:

continue

tf, _, syms = payload

rows = []

for tok, n in tf.items():

tid = tid_map.get(tok)

if tid is None:

tid = tid_map[tok] = next_tid

next_tid += 1

vtoks.append(tok)

vocab_grew = True

rows.append((tid, fid, n))

db.executemany(f"INSERT INTO {target} VALUES (?,?,?)", rows)

if syms:

db.executemany("INSERT INTO symbols VALUES (?,?,?,?,?)",

[(fid, n, n.lower(), k, ln) for n, k, ln in syms])

if vocab_grew:

db.execute("INSERT OR REPLACE INTO meta VALUES('vocab', ?)", ("\n".join(vtoks),))

if bulk:

db.execute("INSERT INTO postings SELECT tid, fid, tf FROM post_stage ORDER BY tid, fid")

db.execute("DROP TABLE post_stage")

db.execute("CREATE INDEX IF NOT EXISTS postings_fid ON postings(fid)")

db.commit()

return stats

# ---- in-memory views used at query time -------------------------------- #

def load(self):

rows = self.db.execute("SELECT id, path, ntok FROM files").fetchall()

self.paths = {i: p for i, p, _ in rows}

self.ntok = {i: n for i, _, n in rows}

indexed = [n for _, _, n in rows if n > 0]

self.N = max(1, len(indexed))

self.avgdl = (sum(indexed) / len(indexed)) if indexed else 1.0

self.fids = [i for i, _, _ in rows]

self.path_blob = Blob([p.lower() for _, p, _ in rows])

self.vtoks = self._load_vocab()

self.vocab = Blob(self.vtoks)

def postings(self, tids: list[int]):

for i in range(0, len(tids), 500):

chunk = tids[i:i + 500]

q = ",".join("?" * len(chunk))

yield from self.db.execute(f"SELECT fid, tid, tf FROM postings WHERE tid IN ({q})", chunk)

def symbols_by_lname(self, names: list[str]):

for i in range(0, len(names), 500):

chunk = names[i:i + 500]

q = ",".join("?" * len(chunk))

yield from self.db.execute(

f"SELECT fid, name, kind, line FROM symbols WHERE lname IN ({q})", chunk)

def stats(self) -> dict:

c = self.db.execute

return {

"db": self.path,

"files": c("SELECT COUNT(*) FROM files").fetchone()[0],

"vocab": len(self._load_vocab()),

"postings": c("SELECT COUNT(*) FROM postings").fetchone()[0],

"symbols": c("SELECT COUNT(*) FROM symbols").fetchone()[0],

"db_mb": round(os.path.getsize(self.path) / 1e6, 1) if os.path.exists(self.path) else 0,

key: str

raw: str

weight: float

kind: str # word | ident | ngram | syn | expand

index_variants: list[str]

text_variants: list[str]

covers: list[str] = field(default_factory=list) # ngram: keys of the word terms it spans

tf: dict = field(default_factory=dict) # fid -> quality-weighted term frequency

syms: list = field(default_factory=list) # (fid, name, kind, line)

path_fids: set = field(default_factory=set)

df: int = 0

idf: float = 0.0

looked_up: bool = False

short: bool = False

rx: re.Pattern | None = None

def __post_init__(self):

self.short = len(self.key) <= 3 and self.kind in ("word", "syn", "expand")

if self.short:

self.rx = re.compile(r"(?<![a-z0-9])" + re.escape(self.key) + r"(?![a-z0-9])")

@property

def in_query(self) -> bool:

stems = [stem(w) for w in words]

variants = set()

for seq in (words, stems):

variants |= {"".join(seq), "_".join(seq), "-".join(seq), " ".join(seq)}

idx = [v for v in variants if re.fullmatch(r"[a-z0-9_]+", v)]

return Term("~" + "_".join(words), " ".join(words), weight, "ngram", idx, sorted(variants),

low = text.lower()

if re.fullmatch(r"[a-z0-9_$]+", low):

idx = [low]

else:

parts = sorted((p for p in re.split(r"[^a-z0-9_$]+", low) if len(p) >= 3), key=len, reverse=True)

idx = parts[:1]

__slots__ = ("lines", "low", "offs")

def __init__(self, text: str):

self.lines = text.split("\n")

low = text.lower()

if len(low) != len(text): # rare unicode case-mapping length changes: keep offsets valid

low = "".join(c.lower() if len(c.lower()) == 1 else c for c in text)

self.low = low

__slots__ = ("fd", "term_lines", "line_w")

def __init__(self):

self.fd: FileData | None = None

self.term_lines: dict[str, list[int]] = {}

low, offs, out = fd.low, fd.offs, set()

if term.rx is not None:

for m in term.rx.finditer(low):

out.add(bisect.bisect_right(offs, m.start()))

if len(out) >= cap:

break

else:

for v in term.text_variants:

pos = low.find(v)

while pos != -1 and len(out) < cap:

ln = bisect.bisect_right(offs, pos)

out.add(ln)

if ln >= len(offs):

break

pos = low.find(v, offs[ln])

lines = sorted(hits)

if len(lines) > 400:

lines = sorted(sorted(lines, key=lambda l: -len(hits[l]))[:400])

best, best_line = 0.0, -1

for i, li in enumerate(lines):

acc, k = set(), i

while k < len(lines) and lines[k] - li <= span:

acc |= hits[lines[k]]

k += 1

w = sum(wmap.get(t, 0.0) for t in acc)

if w > best:

best, best_line = w, li

lo = max(0, idx - max_back)

for i in range(idx, lo - 1, -1):

m = sym_match(lines[i])

if m:

name = m[0]

ind = len(lines[i]) - len(lines[i].lstrip(" \t"))

if ind > 0:

for j in range(i - 1, lo - 1, -1):

mj = sym_match(lines[j])

if mj and mj[1] in CLASS_KINDS and \

(len(lines[j]) - len(lines[j].lstrip(" \t"))) < ind:

return f"{mj[0]}.{name}"

return name

start: int

end: int

hit_lines: list[int]

text: list[str]

symbol: str = ""

terms: list[str] = field(default_factory=list)

fid: int

path: str

base: float

final: float = 0.0

cov: float = 0.0

prox: float = 0.0

prox_cov: float = 0.0

prox_line: int = -1

defsig: float = 0.0

defsym: tuple | None = None

pathsig: float = 0.0

spec: float = 0.0

strength: float = 0.0

confidence: float = 0.0

present: list[str] = field(default_factory=list)

def __init__(self, root, *, use_cache=True, reindex=False, max_turns=MAX_TURNS, context=3,

per_file=3, include_tests=False, exts=None, globs=None, thorough=False, verbose=False):

self.root = os.path.abspath(root)

self.verbose = verbose

self.ctx, self.per_file, self.thorough = context, per_file, thorough

self.max_turns = min(max_turns, MAX_TURNS)

self.include_tests = include_tests

self.exts = {e.strip().lower().lstrip(".") for e in (exts or []) if e.strip()}

self.globs = [g.strip() for g in (globs or []) if g.strip()]

self.workers = min(16, (os.cpu_count() or 4) * 2)

self.idx = Index(self.root, use_cache, reindex)

self.terms: dict[str, Term] = {}

self.analysis: dict[int, FileAnalysis] = {}

self.import_boost: dict[str, float] = {}

self._allowed: dict[int, bool] = {}

self._suffix_map: dict[str, list[str]] | None = None

self.file_syms: dict = {}

self.base: dict[int, float] = {}

self.timings: dict[str, float] = defaultdict(float)

self.turns_used = 0

self.converged = True

def log(self, *a):

if self.verbose:

print(*a, file=sys.stderr)

# ---- query parsing --------------------------------------------------- #

def build_terms(self, query: str) -> None:

T = self.terms

def add(t: Term):

if t.key not in T:

T[t.key] = t

for a, b in re.findall(r'"([^"]+)"|`([^`]+)`', query):

phrase = (a or b).strip().lower()

words = [w for w in re.split(r"[^a-z0-9]+", phrase) if len(w) >= 2]

if len(words) >= 2:

add(make_ngram(words[:3], 3.0))

for w in words:

if w not in STOPWORDS:

add(make_word(w))

q = re.sub(r'"[^"]+"|`[^`]+`', " ", query)

run: list[str] = []

ngrams: list[Term] = []

def flush():

nonlocal run

if len(run) >= 2:

for i in range(len(run) - 1):

ngrams.append(make_ngram(run[i:i + 2], 2.0))

if len(run) >= 3:

ngrams.append(make_ngram(run[:3], 2.5))

run = []

for tok in q.split():

tok = tok.strip(",;:!?[]{}<>\"'").rstrip(".")

cleaned = tok[:-2] if tok.endswith("()") else tok

if not cleaned:

continue

if len(cleaned) >= 3 and looks_like_identifier(cleaned):

flush()

add(make_ident(cleaned))

for part in split_identifier(cleaned):

if len(part) >= 3 and part not in STOPWORDS:

add(make_word(part, 0.7))

continue

for w in re.split(r"[^A-Za-z0-9]+", cleaned):

wl = w.lower()

if len(wl) < 2 or wl.isdigit() or wl in STOPWORDS or stem(wl) in STOPWORDS:

flush()

continue

add(make_word(wl))

run.append(wl)

flush()

for ng in ngrams[:5]:

add(ng)

@property

def qterms(self) -> list[Term]:

return [t for t in self.terms.values() if t.in_query and t.weight > 0]

# ---- index lookups (the "grep/glob" step) ---------------------------- #

def lookup(self, t: Term) -> None:

idx = self.idx

quality: dict[int, float] = {}

for v in t.index_variants:

for i in idx.vocab.find(v):

tok = idx.vtoks[i]

if t.short and tok != v:

continue

q = 1.0 if tok == v else 0.8 if tok.startswith(v) else 0.65 if tok.endswith(v) else 0.5

if q > quality.get(i, 0.0):

quality[i] = q

items = sorted(quality.items(), key=lambda kv: (-kv[1], len(idx.vtoks[kv[0]])))[:MAX_TOKENS_PER_TERM]

tid_q = {i + 1: q for i, q in items}

tf: dict[int, float] = defaultdict(float)

for fid, tid, n in idx.postings(list(tid_q)):

tf[fid] += tid_q[tid] * n

t.tf, t.df = tf, len(tf)

t.idf = math.log(1 + (idx.N - t.df + 0.5) / (t.df + 0.5))

t.syms = list(idx.symbols_by_lname([idx.vtoks[i] for i, _ in items]))

if not t.short:

fids = set()

for v in t.text_variants:

fids.update(idx.fids[i] for i in idx.path_blob.find(v, limit=5000))

t.path_fids = fids

if t.kind in ("syn", "expand") and t.df > 0.15 * idx.N:

t.weight = 0.0 # too common to be informative

t.looked_up = True

# ---- filters / priors ------------------------------------------------ #

def allowed(self, fid: int) -> bool:

ok = self._allowed.get(fid)

if ok is None:

p = self.idx.paths[fid]

ok = True

if self.exts and os.path.splitext(p)[1].lower().lstrip(".") not in self.exts:

ok = False

if ok and self.globs and not any(fnmatch.fnmatch(p, g) for g in self.globs):

ok = False

self._allowed[fid] = ok

return ok

def prior(self, pl: str) -> float:

p = 1.0

if not self.include_tests and TEST_RX.search(pl):

p *= 0.6

if pl.endswith((".md", ".rst", ".txt")):

p *= 0.8

if GENERATED_RX.search(pl):

p *= 0.5

return p

# ---- reading + scanning (the "read" step, threaded) ------------------- #

def read_fd(self, rel: str) -> FileData | None:

try:

with open(os.path.join(self.root, rel), "rb") as fh:

data = fh.read(MAX_FILE_BYTES + 1)

except OSError:

return None

if len(data) > MAX_FILE_BYTES or b"\0" in data[:4096]:

return None

return FileData(data.decode("utf-8", "replace"))

def prepare(self, fid: int) -> None:

fa = self.analysis.get(fid)

if fa is None:

fa = self.analysis[fid] = FileAnalysis()

fa.fd = self.read_fd(self.idx.paths[fid])

if fa.fd is None:

return

for t in self.terms.values():

if t.looked_up and t.key not in fa.term_lines:

fa.term_lines[t.key] = find_lines(t, fa.fd)

# ---- ranking ---------------------------------------------------------- #

def rank(self, top_k: int) -> list[Scored]:

idx = self.idx

t0 = time.perf_counter()

base: dict[int, float] = defaultdict(float)

for t in self.terms.values():

if not t.looked_up or t.weight <= 0:

continue

wi = t.weight * t.idf

for fid, tf in t.tf.items():

base[fid] += wi * tf * (K1 + 1) / (tf + K1 * (1 - B + B * idx.ntok[fid] / idx.avgdl))