if isinstance(function, int):

def memoize_wrapper(f):

return memoize(f, function)

return memoize_wrapper

dict = {}

list = []

def memoize_wrapper(*args, **kwargs):

key = pickle.dumps((args, kwargs))

try:

list.append(list.pop(list.index(key)))

except ValueError:

dict[key] = function(*args, **kwargs)

list.append(key)

if limit is not None and len(list) > limit:

del dict[list.pop(0)]

return dict[key]

memoize_wrapper._memoize_dict = dict

memoize_wrapper._memoize_list = list

memoize_wrapper._memoize_limit = limit

memoize_wrapper._memoize_origfunc = function

memoize_wrapper.__name__ = function.__name__

"A wrapper around compiled regexen that keeps track of timing data"

def __init__(self, re, orig_re):

self._re = re

self._orig_re = orig_re

def sub(self, *args):

return self._timing_operation('sub', *args)

def match(self, *args):

return self._timing_operation('match', *args)

def search(self, *args):

return self._timing_operation('search', *args)

def split(self, *args):

return self._timing_operation('split', *args)

def findall(self, *args):

return self._timing_operation('findall', *args)

def _timing_operation(self, methodname, *args):

start = time.time()

retval = getattr(self._re, methodname)(*args)

end = time.time()

delta = end-start

# if delta > 0.01:

# print delta,'\t', self._orig_re, str(args)[:80]

if self._orig_re not in REGEXEN:

REGEXEN[self._orig_re] = 0.0

REGEXEN[self._orig_re] += delta

"""A version of re.compile which memoizes and optionally keeps track of timing

if TIMING:

return TimingRe(re.compile(regex, *args), regex)

else:

"""a version of re.sub which deals with TimingRe objects and prints out

if TIMING:

start = time.time()

if isinstance(args[0], TimingRe):

retval = args[0].sub(*args[1:])

else:

retval = re.sub(*args)

end = time.time()

delta = end-start

if delta > 0.01: # adjust as needed.

print(delta, '\t', args)

else:

retval = re.sub(*args)

"""Mixin class containing classifier callbacks"""

def is_array_cb(self, tok):

tval = tok['text']

return len(tval) >= 2 and tval[0] == '@' and tval[1] != '$'

# @$name is more like an expression -- don't return it

def is_scalar_cb(self, tok):

tval = tok['text']

return len(tval) > 1 and tval[0] == '$' and (tval[1].isalnum or tval[1] == "_")

def is_pod_cb(self, tok):

return tok['text'][0] == '=' and tok['text'][1].isalnum and tok['text'].find("\n=cut", 5) > 0

def is_string_qw_cb(self, tok):

return re_compile(r'^qw\s*[^\w\d_]').match(tok['text'])

# Used for stripping the quotes off a string

_quote_patterns = {SCE_PL_STRING: re.compile('^[\'\"](.*)[\'\"]$'),

SCE_PL_CHARACTER: re.compile('^\'(.*)\'$'),

SCE_PL_STRING_Q: re.compile(r'^q\s*.(.*).$'),

SCE_PL_STRING_QQ: re.compile(r'^q\w\s*.(.*).$'),

SCE_PL_DEFAULT: re.compile('^.(.*).$'), # fallback

}

def quote_patterns_cb(self, tok):

# Caller wants an array.

return [self.quote_patterns_cb_aux(tok)]

def quote_patterns_cb_aux(self, tok):

tval = tok['text']

if tval[0] == '"':

return self._quote_patterns[SCE_PL_STRING]

elif tval[0] == '\'':

return self._quote_patterns[SCE_PL_CHARACTER]

elif tval.startswith("Q"):

return self._quote_patterns[SCE_PL_STRING_QQ]

elif tval.startswith("q"):

return self._quote_patterns[SCE_PL_STRING_Q]

else:

def get_builtin_type(self, tok, callback):

raise CILEError("Unexpected call to perl_parser.get_builtin_type")

def is_any_operator(self, tok):

return tok['style'] == ScintillaConstants.SCE_PL_OPERATOR

def is_comment(self, tok):

return tok['style'] in (ScintillaConstants.SCE_PL_COMMENT,

ScintillaConstants.SCE_PL_POD)

def is_comment_structured(self, tok, callback):

return tok['style'] == ScintillaConstants.SCE_PL_POD

def is_identifier(self, tok, allow_keywords=False):

return (tok['style'] == ScintillaConstants.SCE_PL_IDENTIFIER or

(allow_keywords and

tok['style'] == ScintillaConstants.SCE_PL_WORD))

def is_index_op(self, tok, pattern=None):

if not (tok['style'] in (SCE_PL_OPERATOR, SCE_PL_VARIABLE_INDEXER)):

return False

elif not pattern:

return True

return len(tok['text']) > 0 and pattern.search(tok['text'])

def is_interpolating_string(self, tok, callback):

return tok['style'] in [ScintillaConstants.SCE_PL_STRING,

ScintillaConstants.SCE_PL_REGEX,

ScintillaConstants.SCE_PL_HERE_QQ,

ScintillaConstants.SCE_PL_STRING_QQ,

ScintillaConstants.SCE_PL_STRING_QR,

ScintillaConstants.SCE_PL_STRING_QX

]

def is_keyword(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_PL_WORD and tok['text'] == target

def is_number(self, tok):

return tok['style'] == ScintillaConstants.SCE_PL_NUMBER

def is_operator(self, tok, target):

return tok['style'] == ScintillaConstants.SCE_PL_OPERATOR and tok['text'] == target

def is_string(self, tok):

return tok['style'] in [ScintillaConstants.SCE_PL_STRING,

ScintillaConstants.SCE_PL_CHARACTER,

ScintillaConstants.SCE_PL_HERE_Q,

ScintillaConstants.SCE_PL_HERE_QQ,

ScintillaConstants.SCE_PL_STRING_Q,

ScintillaConstants.SCE_PL_STRING_QQ,

ScintillaConstants.SCE_PL_STRING_QX,

]

def is_string_qw(self, tok, callback):

return tok['style'] == ScintillaConstants.SCE_PL_STRING_QW

def is_symbol(self, tok):

return False

def is_variable(self, tok):

return SCE_PL_SCALAR <= tok['style'] <= SCE_PL_SYMBOLTABLE

# Types of variables

def is_variable_array(self, tok, callback=None):

return tok['style'] == ScintillaConstants.SCE_PL_ARRAY and \

len(tok['text']) > 1 and tok['text'][1] != '$'

def is_variable_scalar(self, tok, callback=None):

return tok['style'] == ScintillaConstants.SCE_PL_SCALAR and \

len(tok['text']) > 1 and tok['text'][1] != '$'

# Accessors for where we'd rather work with a style than call a predicate

# fn

@property

def style_identifier(self):

return ScintillaConstants.SCE_PL_IDENTIFIER

@property

def style_word(self):

def __init__(self, provide_full_docs):

self.provide_full_docs = provide_full_docs

self.modules = {}

self.currentFunction = None

self.currentNS = None

self.textWrapper = textwrap.TextWrapper()

self.textWrapper.width = 60

self.max_doclet_low_water_mark = 80

self.max_doclet_high_water_mark = 100

self.pod_escape_seq = {'lt': "&lt;",

'gt': "&gt;",

'verbar': "|",

'sol': "/"}

# Things for attrs, etc.

self.export_string = '__exported__'

self.export_ok_string = '__exportable__'

self.local_string = '__local__'

# Cached regular expressions

self.re_bl = r'\r?\n\s*\r?\n'

self.tryGettingDoc_Sig_re3 = re_compile(

r'^=(?:item|head)\w*\s*((?:(?!\n=).)*)(?!\n=)',

re.M | re.S)

self.printDocInfo_re4 = re_compile(

r'^=(?:item|head)\w*\s*((?:(?!\n=).)*)(?!\n=)', re.S | re.M)

self.printDocInfo_re2 = re_compile(

r'^=\w+\s+DESCRIPTION%s(.*?)(?:%s|^=)' % (self.re_bl, self.re_bl), re.M)

self.printDocInfo_re6 = re_compile(

r'^=\w+\s+SYNOPSIS' + self.re_bl + '(.*?)^=',

re.M | re.S)

self.printDocInfo_bdot_re = re_compile(r'\.\s+[A-Z].*\Z')

self._get_first_sentence_re1 = re_compile(r'\.\s+[A-Z].*\Z', re.S)

self._simple_depod_e_re = re_compile(r'E<(.*?)>', re.S)

self._simple_depod_c_re = re_compile(r'C<{2,}\s*(.*?)\s*>{2,}', re.S)

self._simple_depod_ibcfsxl_re1 = re_compile(r'[IBCFSXL]<[^>\n]*>')

self._simple_depod_ibcfsxl_re2 = re_compile(

r'[IBCFSX]<(<*[^<>]*?>*)>', re.S)

self._simple_depod_l_re = re_compile(r'L<\/?(.*?)>')

self._simple_depod_rest_re = re_compile(r'\w<\/?(<*.*?>*)>')

self._depod_re1 = re_compile(

r'^=begin\s+man\s+.*?^=end\s+man\s*', re.M | re.S)

self._depod_re2 = re_compile(r'^=\w+\s*', re.M)

self._depod_re3 = re_compile(r'\]\]>')

self._depod_re4 = re_compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')

self.trim_ws_re1 = re_compile(r'(?<=\w[\.\!\?])\s+')

self.trim_ws_re2 = re_compile(r'[\r\n\t]')

self.trim_ws_re3 = re_compile(r' {2,}')

self.printFunctions_re1 = re_compile(r'(\S)\s*\n(?:\s*\n)*\s*(\S)')

def doStartNS(self, ns):

name = ns.name

if name not in self.modules:

self.modules[name] = ns

self.currentNS = ns

def doEndNS(self, **attrInfo):

if 'lineNo' in attrInfo:

self.currentNS.lineend = attrInfo['lineNo']

self.currentNS = None

def getNS(self, name, **attrInfo):

if name in self.modules:

return self.modules[name]

else:

return NamespaceInfo(name, **attrInfo)

def doSetArg(self, name):

self.currentFunction.aArg[name] = []

self.currentFunction.argList.append(name)

def doSetParent(self, **attrInfo):

ns = attrInfo.get('ns')

if ns:

self.currentNS.aParent.append(ns)

def doStartFn(self, fn):

self.currentFunction = fn

def doEndFn(self, **attrInfo):

if 'lineNo' in attrInfo:

self.currentFunction.lineend = attrInfo.get('lineNo')

self.currentNS.aFunc.append(self.currentFunction)

self.currentFunction = None

def doStartVar(self, **attrInfo):

self.thisVar = {}

self.thisVar['name'] = attrInfo.get('name')

for field in ['line', 'aType', 'scope']:

if field in attrInfo:

self.thisVar[field] = attrInfo[field]

def doEndVar(self, forceGlobal):

name = self.thisVar['name']

if (not forceGlobal) and self.currentFunction:

if name in self.currentFunction.aArg:

self.currentFunction.aArg[name].append(self.thisVar)

else:

self.set_or_append(

self.currentFunction.aVar, name, self.thisVar)

else:

self.set_or_append(self.currentNS.aVar, name, self.thisVar)

del self.thisVar

def set_or_append(self, obj, name, val):

if name in obj:

obj[name].append(val)

else:

obj[name] = [val]

def doSetVar(self, **args):

if 'forceGlobal' in args:

forceGlobal = args['forceGlobal']

del args['forceGlobal']

else:

forceGlobal = False

self.doStartVar(**args)

self.doEndVar(forceGlobal)

def add_imported_module(self, args, **kwargs):

args2 = copy.copy(args)

args2.update(kwargs)

if self.currentFunction:

self.currentFunction.aImports.append(args2)

else:

self.currentNS.aImports.append(args2)

def printDocInfo(self, modInfo, funcInfo, currNode):

docs = modInfo.hDocs['modules']

modName = modInfo.name

# These REs need rebuilding each time, as their values change on each

# call.

printDocInfo_re1 = re_compile(r'^=\w+\s+NAME%s%s[\s-]+(.*?)(?:%s|^=)' %

(self.re_bl, modName, self.re_bl), re.M)

try:

mainDocs = self.modules['main'].hDocs['modules'] or []

except:

mainDocs = []

finalDoc = None

if not funcInfo:

# Just dump the module-level docs for now,

# but favor extracting the synopsis.

#

# First, find the first item with a synopsis

#

# Otherwise, go with the first one.

for doc in docs:

m1 = printDocInfo_re1.search(doc)

if m1:

finalDoc = self.trim_ws(m1.group(1), True)

break

else:

m2 = self.printDocInfo_re2.search(doc)

if m2:

finalDoc = self.trim_ws(m2.group(1), True)

break

if finalDoc is None:

# Look only for a qualified name in the NAME section,

# but don't look at the DESCRIPTION part until we can

# select the main class in a module.

for doc in mainDocs:

m1 = printDocInfo_re1.search(doc)

if m1:

finalDoc = self.trim_ws(m1.group(1), True)

break

else:

# First look for the function name in an item

# Then look in the synopsis.

# Look for a top-level synopsis first

funcName = funcInfo.name

# Allow for datasection-based POD\

if docs:

re3_s = (r'''

((?:\r?\n|.)*?)(?:^=|\Z)''' %

(modName, modName, funcName))

printDocInfo_re3 = re_compile(re3_s, re.X | re.M)

for doc in docs:

# Speed up: do index before doing a reg

if doc.find(funcName) == -1:

continue

# Find a function definition in an item or head thing

# Very general-purpose, might pick up false-positives

# The gist:

# Look for one or more =item/head lines,

# separate by blank lines,

# followed by a paragraph description

m1 = printDocInfo_re3.search(doc)

if m1:

part1 = m1.group(1)

finalDoc = self.trim_ws(m1.group(2), True)

part2 = [self.trim_ws(

s, True) for s in self.printDocInfo_re4.findall(part1)]

finalDoc = "\n\n".join(part2) + "\n\n" + finalDoc

finalDoc = self._get_first_sentence(finalDoc)

break

if not finalDoc:

# Look in the __END__ section

# XXXX VERY SLOW

re1_s = r'''(?:^(?:item|head)\w*\s*

((?:\r?\n|.)*?)(?=$|^=)''' % (funcName, self.re_bl)

printDocInfo_re1 = re_compile(re1_s, re.M | re.X)

for doc in mainDocs:

m1 = printDocInfo_re1.search(doc)

if m1:

before_period = self.printDocInfo_bdot_re.sub(

'.', m1.group(1))

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(1)), True)

break

if not finalDoc:

# Try to find a synopsis entry

printDocInfo_re7 = re_compile(

r'(.*(?:::|->|\b)%s\b.*)' % (funcName,))

for doc in docs + mainDocs:

m1 = self.printDocInfo_re6.search(doc)

if m1:

synopsis = m1.group(1)

m2 = printDocInfo_re7.search(synopsis)

if m2:

finalDoc = self.trim_ws(

self._get_first_sentence(m2.group(1)))

break

if finalDoc:

self.printDocString(finalDoc, currNode)

def _get_first_sentence(self, s1):

s2 = self._get_first_sentence_re1.sub('.', s1)

return s2

def printDocString(self, finalDoc, currNode):

finalDoc2 = self._depod(finalDoc)

if finalDoc2:

currNode.set('doc', finalDoc2)

def _process_e_pod(self, src):

val = self.pod_escape_seq.get(src.lower())

if val:

return val

if re.search(r'^\d+$', src):

return '&#%s;' % src

m1 = re.search(r'^0[Xx]([0-9a-fA-F]+)$', src)

if m1:

return '&#x%s;' % src

else:

# Assume it's an entity name of somekind,

# and return it as an escaped representation

# For example, pod E<eacute> will be encoded as

# &amp;eacute;

# and will appear as

# &eacute;

# Not great, but it causes no breakage.

return '&amp;%s' % src

def _wrap_process_e_pod(self, m):

return self._process_e_pod(m.group(1))

def _simple_depod(self, doc):

# Simple inline-markup removal (doesn't handle nested inlines)

# In Perl, do this:

# $doc =~ s/E<(.*?)>/_process_e_pod($1)/eg;

doc = re_sub(self._simple_depod_e_re, self._wrap_process_e_pod, doc)

# And handle the inline codes-- thse nest with E codes...

doc = self._simple_depod_c_re.sub(r'\1', doc)

# Above code replaces this:

# doc = re_sub(r'C<{2,}\s+(.*?)\s+>{2,}', r'\1', doc)

# doc = XmlAttrEscape(doc) -- No longer needed with ElementTree

# Allow the other sequences to nest, and loop until there

# aren't any left.

old_doc = doc

while self._simple_depod_ibcfsxl_re1.search(doc):

# Most formatting sequences wrap a single clump of code

doc = self._simple_depod_ibcfsxl_re2.sub(r'\1', doc)

# Handling of links - this is more complicated.

doc = self._simple_depod_l_re.sub(r'\1', doc)

# We need to make sure we pull out when nothing changes.

# XXX A log message would be useful here.

if old_doc != doc:

old_doc = doc

else:

break

# Remove any unrecognized sequences

doc = self._simple_depod_rest_re.sub(r'\1', doc)

# And shrink entities back into strings.

# This is done because we can't convert constructs like

# E<gt> into ">" directly, because they'll prevent

# proper handling of outer C<...> in strings like 'C<if (a E<gt> b) {

# ...>'

doc = doc.replace("&lt;", "<").replace("&gt;", ">")

return doc

def _depod(self, doc):

# Remove embedded man directives

doc1 = self._depod_re1.sub('', doc)

# Pull out leading equal signs and the directives

doc2 = self._depod_re2.sub('', doc1)

doc3 = self._simple_depod(doc2)

# Handle strings that could cause the XML parser trouble

doc4 = self._depod_re3.sub(']<!>]>', doc3)

doc5 = self._depod_re4.sub('?', doc4)

return doc5

def trim_ws(self, str1, truncate=False):

# First split into sentences

str2 = str1.strip()

if truncate:

sentences = self.trim_ws_re1.split(str2)

keep_sentences = []

sum = 0

# Keep sentences until we pass max_doclet_high_water_mark chars.

for s in sentences:

thisLen = len(s)

if sum and sum + thisLen > self.max_doclet_high_water_mark:

break

s1 = self.trim_ws_re2.sub(' ', s)

s2 = self.trim_ws_re3.sub(' ', s1)

keep_sentences.append(s2)

sum += thisLen

if sum > self.max_doclet_high_water_mark:

break

str2 = " ".join(keep_sentences)

if str2.find("\n") >= 0 or len(str2) > self.textWrapper.width * 1.1:

str2 = "\n".join(self.textWrapper.wrap(str2))

return str2

def printClassParents(self, modInfo, currNode):

if not hasattr(modInfo, 'aParent'):

return

classrefs = [info[0] for info in modInfo.aParent]

if len(classrefs) > 0:

currNode.set('classrefs', " ".join(classrefs))

def printImports(self, modInfo, currNode):

# -- this will be correct only when there are deliberate conflicts

# better to use object inheritance to choose methods dynamically.

# imports = getattr(modInfo, 'aImports', [])

# imports.reverse()

for _import in getattr(modInfo, 'aImports', []):

attrs = list(_import.keys())

importNode = SubElement(currNode, "import")

for k in attrs:

importNode.set(k, str(_import[k]))

def printTypeInfo(self, argInfo, currNode):

types = {}

for type_ in argInfo:

typeInfo = type_.get('aType')

if not typeInfo:

continue

elif 'assign' not in typeInfo:

continue

tp = typeInfo['assign']

if tp in types:

continue

types[tp] = None

currNode.set('citdl', tp)

def printVariables(self, modInfo, currNode):

if not hasattr(modInfo, 'aVar'):

return

def sorter1(a, b):

return (cmp(a[0]['line'], b[0]['line']) or

cmp(a[0]['name'].lower(), b[0]['name'].lower()))

variables = list(modInfo.aVar.values())

variables.sort(sorter1)

try:

export_info = modInfo.export_info

except:

if not hasattr(self, 'default_export_info'):

self.default_export_info = {'@EXPORT': {}, '@EXPORT_OK': {}}

export_info = self.default_export_info

for varInfo in variables:

var_name = varInfo[0]['name']

varNode = SubElement(

currNode, 'variable', line=str(varInfo[0]['line']),

name=var_name)

attr_parts = []

if var_name in export_info['@EXPORT']:

attr_parts.append(self.export_string)

if var_name in export_info['@EXPORT_OK']:

attr_parts.append(self.export_ok_string)

try:

if varInfo[0]['scope'] == 'my':

attr_parts.append(self.local_string)

except:

pass

if attr_parts:

varNode.set('attributes', ' '.join(attr_parts))

self.printTypeInfo(varInfo, varNode)

def printFunctions(self, modInfo, currNode):

for funcInfo in getattr(modInfo, 'aFunc', []):

sig, docString = self.tryGettingDoc_Sig(modInfo, funcInfo)

funcName = funcInfo.name

if not sig:

sig = '%s(%s)' % (funcName, ', '.join(funcInfo.argList))

else:

sig = self._simple_depod(sig.strip())

sig = self.printFunctions_re1.sub('\\1\n\\2', sig)

funcNode = SubElement(

currNode, 'scope', ilk='function', name=funcName)

for attr_name in ['line', 'lineend']:

ln = getattr(funcInfo, attr_name, None)

if ln:

funcNode.set(attr_name, str(ln))

attr_parts = []

if funcInfo.isConstructor:

attr_parts.append("__ctor__")

export_info = modInfo.export_info

for tuple in [['@EXPORT', self.export_string],

['@EXPORT_OK', self.export_ok_string]]:

if funcName in export_info[tuple[0]]:

attr_parts.append(tuple[1])

if attr_parts:

funcNode.set('attributes', ' '.join(attr_parts))

funcNode.set('signature', sig)

for argName in funcInfo.argList:

argInfo = funcInfo.aArg.get(argName)

if argInfo:

kwargs = {'ilk': 'argument', 'name': argInfo[0]['name']}

if 'line' in argInfo[0]:

kwargs['line'] = str(argInfo[0]['line'])

argNode = SubElement(funcNode, 'variable', **kwargs)

self.printTypeInfo(argInfo, argNode)

if self.provide_full_docs:

if not docString:

self.printDocInfo(modInfo, funcInfo, funcNode)

else:

self.printDocString(docString, funcNode)

self.printImports(funcInfo, funcNode)

self.printVariables(funcInfo, funcNode)

def tryGettingDoc_Sig(self, modInfo, funcInfo):

if not self.provide_full_docs:

return (None, None)

modName = modInfo.name

funcName = funcInfo.name

re1_s = r"""((?:^=(?:item|head)\w*\s*

((?:\r?\n|.)*?)(?:^=|\Z)""" % \

(modName, modName, funcName, self.re_bl)

re1 = re_compile(re1_s, re.X | re.M)

re2a = re_compile(funcName + r'\s+\w')

re2b = re_compile(r'\s\w+\s+' + funcName + r'\b')

re4_s = (r'''^=(?:item|head)\s*\*?\s*\r?\n

((?:.*\r?\n(?!=))+)''' % (funcName,))

re4 = re_compile(re4_s, re.M | re.X)

for doc in modInfo.hDocs['modules'] + self.modules['main'].hDocs['modules']:

if doc.find(funcName) == -1:

continue

m1 = re1.search(doc)

if m1:

part1 = m1.group(1)

# If the function name is followed by text, it's probably

# an English description. We need to check for a whole word

# before the name to avoid the pod directive

if (re2a.search(part1) or

re2b.search(part1)):

continue

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(2)), True)

part2 = [

s.strip() for s in self.tryGettingDoc_Sig_re3.findall(part1)]

finalSig = (part2 and "\n\n".join(part2)) or None

return (finalSig, finalDoc)

else:

m1 = re4.search(doc)

if m1:

finalSig = m1.group(1)

finalDoc = self.trim_ws(

self._get_first_sentence(m1.group(2)), True)

return (finalSig, finalDoc)

def __init__(self, name, **attrInfo):

self.name = name

self.line = attrInfo.get('lineNo') or 0

self.aFunc = []

self.aVar = {}

self.aParent = []

self.hDocs = {'modules': [], # hash of modules => array of docs,

'subs': {} # subs => subname => array of docs

}

self.aImports = []

self._isProbablyClass = False

self.export_info = {'@EXPORT': {},

'@EXPORT_OK': {}}

def isProbablyClass(self, val):

def __init__(self, name, **attrInfo):

self.name = name

self.aArg = {}

self.aVar = {}

self.resultType = []

self.argList = []

self.aImports = []

self.isConstructor = attrInfo.get('isConstructor', False)

if 'lineNo' in attrInfo:

def __init__(self, tokenizer, lang="Perl", provide_full_docs=True):

self.tokenizer = tokenizer

self._provide_full_docs = provide_full_docs

self.block_stack = []

self.bracket_matchers = {"[": "]", "{": "}", "(": ")"}

self.classifier = _get_classifier(lang)

# Use simple knowledge of Perl's syntax

# to skip quickly through code to skip.

self.opHash = {"(": [0, 1],

")": [0, -1],

"{": [1, 1],

"}": [1, -1],

"[": [2, 1],

"]": [2, -1]}

self.pragmaNames = {'attributes': None,

'attrs': None,

'autouse': None,

'bigint': None,

'bignum': None,

'bigrat': None,

'blib': None,

'bytes': None,

'charnames': None,

'constant': None,

'diagnostics': None,

'encoding': None,

'fields': None,

'filetest': None,

'if': None,

'integer': None,

'less': None,

'lib': None,

'locale': None,

'open': None,

'ops': None,

'overload': None,

're': None,

'sigtrap': None,

'sort': None,

'strict': None,

'subs': None,

'threads': None,

'utf8': None,

'vmsish': None,

'warnings': None,

}

self.find_open_indexer_re = re_compile(r'[\[{]')

self.provide_full_docs = provide_full_docs

def _is_stmt_end_op(self, tok):

tval = tok['text']

if self.classifier.is_keyword(tok, 'or'):

return True

elif self.classifier.is_any_operator(tok):

return tval in (';', '||')

return False

def _is_string(self, tok):

return tok['style'] in self.tokenizer.string_types

def printHeader(self, mtime):

moduleName = self.moduleName

root = Element("codeintel", version="2.0")

fileNode = Element("file", lang="Perl",

path=moduleName)

if mtime:

fileNode.set('mtime', str(mtime))

root.append(fileNode)

return (root, fileNode)

def printContents(self, moduleContentsName, currNode):

name = os.path.splitext(os.path.basename(self.moduleName))[0]

moduleNode = SubElement(

currNode, 'scope', ilk='blob', lang="Perl", name=name)

innerModules = self.moduleInfo.modules

mainInfo = innerModules.get('main', None)

if mainInfo:

if self.provide_full_docs:

self.moduleInfo.printDocInfo(mainInfo, None, moduleNode)

self.moduleInfo.printImports(mainInfo, moduleNode)

self.moduleInfo.printVariables(mainInfo, moduleNode)

def sorter1(a, b):

amod = innerModules.get(a)

bmod = innerModules.get(b)

aline = getattr(amod, 'line', None)

if aline:

bline = getattr(bmod, 'line', None)

if aline and bline:

return cmp(aline, bline)

return cmp(getattr(amod, 'name', ""), getattr(bmod, 'name', ""))

# Sub-packages need to updated their parent blob name - bug 88814.

# I.e. when parsing "XML/Simple.pm" the blob name is "Simple", but we

# need it be "XML::Simple" in this case. The bestPackageName is

# used to find the best matching name.

packages = [x for x in list(self.moduleInfo.modules.keys()) if x != 'main']

packages.sort(sorter1)

bestPackageName = None

for k in packages:

modInfo = innerModules[k]

if name in k and \

(bestPackageName is None or len(bestPackageName) > k):

bestPackageName = k

moduleNode.set("name", bestPackageName)

classNode = SubElement(

moduleNode, 'scope', ilk='class', name=modInfo.name, line=str(modInfo.line))

if hasattr(modInfo, 'lineend'):

classNode.set('lineend', str(modInfo.lineend))

self.moduleInfo.printClassParents(modInfo, classNode)

if self.provide_full_docs:

self.moduleInfo.printDocInfo(modInfo, None, classNode)

self.moduleInfo.printImports(modInfo, classNode)

self.moduleInfo.printVariables(modInfo, classNode)

self.moduleInfo.printFunctions(modInfo, classNode)

# And do main's functions after its classes

if mainInfo:

self.moduleInfo.printFunctions(mainInfo, moduleNode)

def printTrailer(self, root):

pass