def __init__(self, *args, **kwargs):

extra_module_dirs = os.environ.get("PYTHONPATH", "").split(os.pathsep)

kwargs["extra_module_dirs"] = kwargs.get("extra_module_dirs", []) + extra_module_dirs

# HACKy: This isn't super-robust.

from os.path import abspath, normpath

url = normpath(abspath(local_path))

if sys.platform == "win32":

url = "file:///" + url.replace('\\', '/')

else:

url = "file://" + url

"""Allow multiple format string -- depending on the log level.

def __init__(self, fmt=None, datefmt=None, fmtFromLevel=None):

logging.Formatter.__init__(self, fmt, datefmt)

if fmtFromLevel is None:

self.fmtFromLevel = {}

else:

self.fmtFromLevel = fmtFromLevel

def format(self, record):

record.levelname = record.levelname.lower()

if record.levelno in self.fmtFromLevel:

#XXX This is a non-threadsafe HACK. Really the base Formatter

# class should provide a hook accessor for the _fmt

# attribute. *Could* add a lock guard here (overkill?).

_saved_fmt = self._fmt

self._fmt = self.fmtFromLevel[record.levelno]

try:

return logging.Formatter.format(self, record)

finally:

self._fmt = _saved_fmt

else:

hdlr = logging.StreamHandler()

defaultFmt = "%(name)s: %(levelname)s: %(message)s"

infoFmt = "%(name)s: %(message)s"

fmtr = _PerLevelFormatter(fmt=defaultFmt,

fmtFromLevel={logging.INFO: infoFmt})

hdlr.setFormatter(fmtr)

r"""Return escaped version of text.

#TODO:

# - Add 'c-string' style.

# - Add _escaped_html_from_text() with a similar call sig.

import re

if isinstance(escapes, basestring):

if escapes == "eol":

escapes = {'\r\n': "\\r\\n\r\n", '\n': "\\n\n", '\r': "\\r\r"}

elif escapes == "whitespace":

escapes = {'\r\n': "\\r\\n\r\n", '\n': "\\n\n", '\r': "\\r\r",

'\t': "\\t", ' ': "."}

elif escapes == "eol-one-line":

escapes = {'\n': "\\n", '\r': "\\r"}

elif escapes == "whitespace-one-line":

escapes = {'\n': "\\n", '\r': "\\r", '\t': "\\t", ' ': '.'}

else:

raise ValueError("unknown text escape style: %r" % escapes)

# Sort longer replacements first to allow, e.g. '\r\n' to beat '\r' and

# '\n'.

escapes_keys = escapes.keys()

escapes_keys.sort(key=lambda a: len(a), reverse=True)

def repl(match):

val = escapes[match.group(0)]

return val

escaped = re.sub("(%s)" % '|'.join([re.escape(k) for k in escapes_keys]),

repl,

text)

brief=False, doSort=False, encoding=None):

"""Dump an outline of the given codeintel tree element."""

outline = outline_ci_elem(elem)

r"""ci2 -- the new Code Intel, a tool for working with source code

name = "ci2"

version = __version__

_do_profiling = False

#XXX There is a bug in cmdln.py alignment when using this. Leave it off

# until that is fixed. -- I think this may be fixed now.

#helpindent = ' '*4

def _set_profiling(self, option, opt_str, value, parser):

self._do_profiling = True

def get_optparser(self):

optparser = cmdln.Cmdln.get_optparser(self)

optparser.add_option("-v", "--verbose",

action="callback", callback=_set_verbosity,

help="More verbose output. Repeat for more and more output.")

optparser.add_option("-L", "--log-level",

action="callback", callback=_set_logger_level, nargs=1, type="str",

help="Specify a logger level via '<logname>:<levelname>'.")

optparser.add_option("-p", "--profile",

action="callback", callback=self._set_profiling,

help="Enable code profiling, prints out a method summary.")

return optparser

def do_test(self, argv):

"""Run the ci2 test suite.

import subprocess

testdir = join(dirname(__file__), "test2")

if self._do_profiling:

cmd = '"%s" -m cProfile -s time test.py %s' % (sys.executable, ' '.join(argv[1:]))

else:

cmd = '"%s" test.py %s' % (sys.executable, ' '.join(argv[1:]))

env = os.environ.copy()

env["CODEINTEL_NO_PYXPCOM"] = "1"

p = subprocess.Popen(cmd, cwd=testdir, env=env, shell=True)

p.wait()

return p.returncode

def do_clean_unicode_directories(self, argv):

""" Remove the unicode directories after running `ci2 test`."""

import subprocess

testdir = join(dirname(__file__), "test2")

cmd = '"%s" clean_tests.py' % (sys.executable,)

env = os.environ.copy()

p = subprocess.Popen(cmd, cwd=testdir, env=env, shell=True)

p.wait()

return p.returncode

def do_doctest(self, subcmd, opts):

"""Run the ci2 internal doctests.

import doctest

doctest.testmod()

def _cidb_path_from_kover(self, kover):

ko_user_data_dir = applib.user_data_dir("Komodo", "ActiveState",

"%s.%s" % kover)

if not exists(ko_user_data_dir):

raise OSError("`%s' does not exist" % ko_user_data_dir)

return join(ko_user_data_dir, "codeintel.db")

def do_play(self, subcmd, opts):

"""Run my current play/dev code.

if False:

lang = "CSS"

markedup_content = dedent("""

""")

content, data = unmark_text(markedup_content)

pos = data["pos"]

mgr = Manager()

#mgr.upgrade() # Don't need it for just CSS usage.

mgr.initialize()

try:

buf = mgr.buf_from_content(content, lang=lang, path="play.css")

trg = buf.trg_from_pos(pos)

if trg is None:

raise Error("unexpected trigger: %r" % trg)

completions = buf.cplns_from_trg(trg)

print("COMPLETIONS: %r" % completions)

finally:

mgr.finalize()

elif False:

lang = "Python"

path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))

markedup_content = dedent("""

""")

content, data = unmark_text(markedup_content)

print(banner(path))

print(_escaped_text_from_text(content, "whitespace"))

pos = data["pos"]

mgr = Manager()

mgr.upgrade()

mgr.initialize()

try:

buf = mgr.buf_from_content(content, lang=lang, path=path)

print(banner("cix", '-'))

print(buf.cix)

trg = buf.trg_from_pos(pos)

if trg is None:

raise Error("unexpected trigger: %r" % trg)

print(banner("completions", '-'))

ctlr = LogEvalController(log)

buf.async_eval_at_trg(trg, ctlr)

ctlr.wait(2) #XXX

if not ctlr.is_done():

ctlr.abort()

raise Error("XXX async eval timed out")

pprint(ctlr.cplns)

print(banner(None))

finally:

mgr.finalize()

elif False:

lang = "Ruby"

path = join("<Unsaved>", "rand%d.py" % random.randint(0, 100))

markedup_content = dedent("""\

""")

content, data = unmark_text(markedup_content)

print(banner(path))

print(_escaped_text_from_text(content, "whitespace"))

pos = data[1]

mgr = Manager()

mgr.upgrade()

mgr.initialize()

try:

buf = mgr.buf_from_content(content, lang=lang, path=path)

print(banner("cix", '-'))

cix = buf.cix

print(ET.tostring(pretty_tree_from_tree(tree_from_cix(buf.cix))))

trg = buf.trg_from_pos(pos, implicit=False)

if trg is None:

raise Error("unexpected trigger: %r" % trg)

print(banner("completions", '-'))

ctlr = LogEvalController(log)

buf.async_eval_at_trg(trg, ctlr)

ctlr.wait(30) #XXX

if not ctlr.is_done():

ctlr.abort()

raise Error("XXX async eval timed out")

pprint(ctlr.cplns)

print(banner(None))

finally:

mgr.finalize()

@cmdln.alias("up")

def do_unpickle(self, subcmd, opts, *path_patterns):

"""Unpickle and dump the given paths.

import cPickle as pickle

for path in _paths_from_path_patterns(path_patterns):

fin = open(path, 'rb')

try:

obj = pickle.load(fin)

finally:

fin.close()

pprint(obj)

@cmdln.option("-d", "--db-base-dir",

def do_dbcheck(self, subcmd, opts):

"""Run an internal consistency check on the database.

mgr = Manager(opts.db_base_dir)

try:

errors = mgr.db.check()

finally:

mgr.finalize()

for error in errors:

print(error)

return len(errors)

@cmdln.option("-l", "--language", dest="lang",

@cmdln.option("-b", "--brief", dest="brief", action="store_true",

@cmdln.option("-s", "--sorted", dest="doSort", action="store_true",

def do_outline(self, subcmd, opts, path):

"""Print code outline of the given file.

mgr = Manager()

mgr.upgrade()

mgr.initialize()

try:

if '#' in path:

path, anchor = path.rsplit('#', 1)

else:

anchor = None

if path.endswith(".cix"):

tree = tree_from_cix(open(path, 'r').read())

#buf = mgr.buf_from_content("", tree[0].get("lang"), path=path)

else:

buf = mgr.buf_from_path(path, lang=opts.lang)

tree = buf.tree

if anchor is not None:

# Lookup the anchor in the codeintel CIX tree.

lpath = re.split(r'\.|::', anchor)

def blobs_from_tree(tree):

for file_elem in tree:

for blob in file_elem:

yield blob

for elem in blobs_from_tree(tree):

# Generally have 3 types of codeintel trees:

# 1. single-lang file: one <file>, one <blob>

# 2. multi-lang file: one <file>, one or two <blob>'s

# 3. CIX stdlib/catalog file: possibly multiple

# <file>'s, likely multiple <blob>'s

# Allow the first token to be the blob name or lang.

# (This can sometimes be weird, but seems the most

# convenient solution.)

if lpath[0] in (elem.get("name"), elem.get("lang")):

remaining_lpath = lpath[1:]

else:

remaining_lpath = lpath

for name in remaining_lpath:

try:

elem = elem.names[name]

except KeyError:

elem = None

break # try next lang blob

if elem is not None:

break # found one

else:

log.error("could not find `%s' definition (or blob) in `%s'",

anchor, path)

return 1

else:

elem = tree

try:

_outline_ci_elem(elem, brief=opts.brief, doSort=opts.doSort)

except IOError as ex:

if ex.errno == 0:

# Ignore this error from aborting 'less' of 'ci2 outline'

# output:

# IOError: (0, 'Error')

pass

else:

raise

finally:

mgr.finalize()

def do_cix_check(self, subcmd, opts, *path_patterns):

"""Check the given CIX file(s) for warnings, errors.

num_results = 0

for path in _paths_from_path_patterns(path_patterns):

tree = None

cix = open(path, 'r').read()

tree = tree_from_cix(cix)

for sev, msg in check_tree(tree):

num_results += 1

print("%s: %s: %s" % (path, sev, msg))

return num_results

@cmdln.option("-2", dest="convert", action="store_true",

@cmdln.option("-p", "--pretty-print", action="store_true",

def do_cix(self, subcmd, opts, *path_patterns):

"""Read in and print a CIX file (possibly converting to CIX 2.0

if opts.pretty_print:

opts.convert = True

for path in _paths_from_path_patterns(path_patterns):

tree = None

cix = open(path, 'r').read()

if opts.convert:

tree = tree_from_cix(cix)

if opts.pretty_print:

tree = pretty_tree_from_tree(tree)

ET.dump(tree)

else:

sys.stdout.write(cix)

@cmdln.option("-l", "--language", dest="lang",

@cmdln.option("-q", "--quiet", dest="quiet", action="store_true",

@cmdln.option("-p", "--pretty-print", action="store_true",

@cmdln.option("-f", "--force", action="store_true",

@cmdln.option("-i", "--include", dest="includes", action="append",

@cmdln.option("-t", dest="time_it", action="store_true",

@cmdln.option("-T", dest="time_details", action="store_true",

@cmdln.option("-r", dest="recursive", action="store_true",

@cmdln.option("-n", dest="stripfuncvars", action="store_true",

def do_scan(self, subcmd, opts, *path_patterns):

"""Scan and print the CIX for the given path(s).

mgr = Manager()

mgr.upgrade()

mgr.initialize()

try:

if opts.time_it:

start = time.time()

quiet = opts.quiet

if opts.time_it or opts.time_details:

opts.force = True

scan_count = 0

lang_warnings = set()

tree = None

for path in _paths_from_path_patterns(path_patterns,

recursive=opts.recursive,

includes=opts.includes):

if opts.time_it:

sys.stderr.write(path+"\n")

if opts.time_details:

start1 = time.time()

try:

lang = opts.lang or guess_lang_from_path(path)

except CodeIntelError:

log.info("skip `%s': couldn't determine language", path)

continue

try:

buf = mgr.buf_from_path(path, lang=lang)

except OSError as ex:

# Couldn't access the file.

if not opts.recursive:

raise

# Ignore files we don't really care about.

log.warn("%r - %r", ex, path)

continue

if not isinstance(buf, CitadelBuffer):

if opts.recursive:

# Ignore files that scanning isn't provided for.

continue

raise CodeIntelError("`%s' (%s) is not a language that "

"uses CIX" % (path, buf.lang))

scan_count += 1

if scan_count % 10 == 0:

log.info("%d scanning %r", scan_count, path)

try:

if opts.force:

buf.scan()

if tree is None:

tree = ET.Element("codeintel", version="2.0")

file_elem = ET.SubElement(tree, "file",

lang=buf.lang,

mtime=str(int(time.time())),

path=os.path.basename(path))

for lang, blob in sorted(buf.blob_from_lang.items()):

blob = buf.blob_from_lang[lang]

file_elem.append(blob)

except KeyError as ex:

# Unknown cile language.

if not opts.recursive:

raise

message = str(ex)

if message not in lang_warnings:

lang_warnings.add(message)

log.warn("Skipping unhandled language %s", message)

if opts.time_details:

delta = time.time() - start1

sys.stderr.write("%.3f %s\n" % (delta, path))

sys.stderr.flush()

if tree is not None:

if opts.stripfuncvars:

# For stdlibs, we don't care about variables inside of

# functions and they take up a lot of space.

for function in tree.getiterator('scope'):

if function.get('ilk') == 'function':

function[:] = [child for child in function

if child.tag != 'variable']

if opts.pretty_print:

tree = pretty_tree_from_tree(tree)

if not quiet:

sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')

ET.dump(tree)

if opts.time_it:

end = time.time()

sys.stderr.write("scan took %.3fs\n" % (end-start))

finally:

mgr.finalize()

@cmdln.option("-o", "--output",

@cmdln.option("-b", "--browse", action="store_true",

@cmdln.option("-f", "--force", action="store_true",

@cmdln.option("-e", "--do-eval", action="store_true",

@cmdln.option("-t", "--do-trg", action="store_true",

@cmdln.option("-l", "--lang",

def do_html(self, subcmd, opts, path):

"""Convert the given path to styled HTML.

mgr = Manager()

try:

if opts.browse:

htmls = []

buf = mgr.buf_from_path(path, lang=opts.lang)

html = buf.to_html(True, True, title=path,

do_trg=opts.do_trg,

do_eval=opts.do_eval)

finally:

mgr.finalize()

if opts.output == '-':

output_path = None

output_file = sys.stdout

else:

if opts.output:

output_path = opts.output

else:

output_path = path+".html"

if exists(output_path):

if opts.force:

os.remove(output_path)

else:

raise Error("`%s' exists: use -f|--force option to "

"allow overwrite" % output_path)

output_file = open(output_path, 'w')

#else:

# output_path = None

# output_file = sys.stdout

# #XXX Disable writing t

# output_file = None

if output_file:

output_file.write(html)

if output_path:

output_file.close()

if opts.browse:

if not output_path:

raise Error("cannot open in browser if stdout used "

"for output")

import webbrowser

url = _url_from_local_path(output_path)

webbrowser.open_new(url)

@cmdln.option("-c", "--css", dest="css_reference_files", action="append",

@cmdln.option("-o", "--output", dest="output",

@cmdln.option("-t", "--toc-file", dest="toc_file",

@cmdln.option("-l", "--language", dest="lang",

def do_cix2html(self, subcmd, opts, path):

"""Turn cix file into html API documentation.

cix2html.cix2html(opts, path)

@cmdln.option("-o", "--output",

@cmdln.option("-f", "--force", action="store_true",

def do_json(self, subcmd, opts, path):

"""Convert cix XML file into json format.

import json

if opts.output == '-':

output_path = None

output_file = sys.stdout

else:

if opts.output:

output_path = opts.output

else:

output_path = splitext(path)[0]+".json"

if exists(output_path):

if opts.force:

os.remove(output_path)

else:

raise Error("`%s' exists: use -f|--force option to "

"allow overwrite" % output_path)

output_file = open(output_path, 'w')

mgr = Manager()

mgr.upgrade()

mgr.initialize()

try:

if path.endswith(".cix"):

tree = tree_from_cix(open(path, 'r').read())

else:

buf = mgr.buf_from_path(path, lang=opts.lang)

tree = buf.tree

result = {}

ci = result["codeintel"] = defaultdict(list)

def _elemToDict(parent, elem):

data = defaultdict(list)

name = elem.get("name")

if name is not None:

data["name"] = name

data["tag"] = elem.tag

for attr_name, attr in elem.attrib.items():

data[attr_name] = attr

parent["children"].append(data)

for child in elem:

_elemToDict(data, child)

for child in tree:

_elemToDict(ci, child)

json.dump(result, output_file, indent=2)

finally:

"""Return True iff the given path should be included."""

from os.path import basename

from fnmatch import fnmatch

base = basename(path)

if includes:

for include in includes:

if fnmatch(base, include):

try:

log.debug("include `%s' (matches `%s')", path, include)

except (NameError, AttributeError):

pass

break

else:

log.debug("exclude `%s' (matches no includes)", path)

return False

for exclude in excludes:

if fnmatch(base, exclude):

try:

log.debug("exclude `%s' (matches `%s')", path, exclude)

except (NameError, AttributeError):

pass

return False

recursive=True, includes=[], excludes=[],

on_error=_NOT_SPECIFIED):

"""_paths_from_path_patterns([<path-patterns>, ...]) -> file paths

from os.path import basename, exists, isdir, join

from glob import glob

GLOB_CHARS = '*?['

for path_pattern in path_patterns:

# Determine the set of paths matching this path_pattern.

for glob_char in GLOB_CHARS:

if glob_char in path_pattern:

paths = glob(path_pattern)

break

else:

paths = exists(path_pattern) and [path_pattern] or []

if not paths:

if on_error is None:

pass

elif on_error is _NOT_SPECIFIED:

try:

log.error("`%s': No such file or directory", path_pattern)

except (NameError, AttributeError):

pass

else:

on_error(path_pattern)

for path in paths:

if isdir(path):

# 'includes' SHOULD affect whether a dir is yielded.

if (dirs == "always"

or (dirs == "if-not-recursive" and not recursive)

) and _should_include_path(path, includes, excludes):

yield path

# However, if recursive, 'includes' should NOT affect

# whether a dir is recursed into. Otherwise you could

# not:

# script -r --include="*.py" DIR

if recursive and _should_include_path(path, [], excludes):

for dirpath, dirnames, filenames in os.walk(path):

dir_indeces_to_remove = []

for i, dirname in enumerate(dirnames):

d = join(dirpath, dirname)

if dirs == "always" \

and _should_include_path(d, includes, excludes):

yield d

if not _should_include_path(d, [], excludes):

dir_indeces_to_remove.append(i)

for i in dir_indeces_to_remove:

del dirnames[i]

if files:

for filename in sorted(filenames):

f = join(dirpath, filename)

if _should_include_path(f, includes, excludes):

yield f

elif files and _should_include_path(path, includes, excludes):

global _v_count, log