CodeIntel/codeintel/codeintel2/manager.py at master · SublimeCodeIntel/CodeIntel

580 lines (499 loc) · 22.7 KB
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
# The contents of this file are subject to the Mozilla Public License
# Version 1.1 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
# Software distributed under the License is distributed on an "AS IS"
# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
# License for the specific language governing rights and limitations
# under the License.
# The Original Code is Komodo code.
# The Initial Developer of the Original Code is ActiveState Software Inc.
# Portions created by ActiveState Software Inc are Copyright (C) 2000-2007
# ActiveState Software Inc. All Rights Reserved.
# Contributor(s):
#   ActiveState Software Inc
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
# ***** END LICENSE BLOCK *****
"""The "Manager" is the controlling instance for a codeintel system."""
from __future__ import absolute_import
from os.path import dirname, join, abspath, splitext, basename, isabs
import logging
from collections import defaultdict
from glob import glob
import threading
from six.moves.queue import Queue
import warnings
import traceback
import codecs
from SilverCity import ScintillaConstants
import codeintel2
from codeintel2.common import *
from codeintel2.accessor import *
from codeintel2.citadel import Citadel, BinaryBuffer
from codeintel2.buffer import ImplicitBuffer
from codeintel2.langintel import ImplicitLangIntel
from codeintel2.database.database import Database
from codeintel2.environment import DefaultEnvironment
from codeintel2 import indexer
from codeintel2.util import guess_lang_from_path
from codeintel2 import hooks
from codeintel2.udl import XMLParsingBufferMixin, UDLBuffer
import langinfo
if _xpcom_:
    from xpcom.server import UnwrapObject
#---- global variables
log = logging.getLogger("codeintel.manager")
#log.setLevel(logging.INFO)
#---- public interface
class Manager(threading.Thread, Queue):
    # See the module docstring for usage information.
    def __init__(self, db_base_dir=None, on_scan_complete=None,
                 extra_module_dirs=None, env=None,
                 db_event_reporter=None, db_catalog_dirs=None,
                 db_import_everything_langs=None):
        """Create a CodeIntel manager.
            "db_base_dir" (optional) specifies the base directory for
                the codeintel database. If not given it will default to
                '~/.codeintel'.
            "on_scan_complete" (optional) is a callback for Citadel scan
                completion. It will be passed the ScanRequest instance
                as an argument.
            "extra_module_dirs" (optional) is a list of extra dirs
                in which to look for and use "codeintel_*.py"
                support modules (and "lang_*.py" modules, DEPRECATED).
            "env" (optional) is an Environment instance (or subclass).
                See environment.py for details.
            "db_event_reporter" (optional) is a callback that will be called
                    db_event_reporter(<event-desc-string>)
                before "significant" long processing events in the DB. This
                may be useful to forward to a status bar in a GUI.
            "db_catalog_dirs" (optional) is a list of catalog dirs in
                addition to the std one to use for the CatalogsZone. All
                *.cix files in a catalog dir are made available.
            "db_import_everything_langs" (optional) is a set of langs for which
                the extra effort to support Database
                `lib.hits_from_lpath()' should be made. See class
                Database for more details.
        """
        threading.Thread.__init__(self, name="CodeIntel Manager")
        self.setDaemon(True)
        Queue.__init__(self)
        self.citadel = Citadel(self)
        # Module registry bits.
        self._registered_module_canon_paths = set()
        self.silvercity_lexer_from_lang = {}
        self.buf_class_from_lang = {}
        self.langintel_class_from_lang = {}
        self._langintel_from_lang_cache = {}
        self.import_handler_class_from_lang = {}
        self._is_citadel_from_lang = {} # registered langs that are Citadel-based
        self._is_cpln_from_lang = {} # registered langs for which completion is supported
        self._hook_handlers_from_lang = defaultdict(list)
        self.env = env or DefaultEnvironment() 
        # The database must be enabled before registering modules.
        self.db = Database(self, base_dir=db_base_dir,
                           catalog_dirs=db_catalog_dirs,
                           event_reporter=db_event_reporter,
                           import_everything_langs=db_import_everything_langs)
        self.lidb = langinfo.get_default_database()
        self._register_modules(extra_module_dirs)
        self.idxr = indexer.Indexer(self, on_scan_complete)
    def upgrade(self):
        """Upgrade the database, if necessary.
        It blocks until the upgrade is complete.  Alternatively, if you
        want more control over upgrading use:
            Database.upgrade_info()
            Database.upgrade()
            Database.reset()
        """
        log.debug("upgrade db if necessary")
        status, reason = self.db.upgrade_info()
        if status == Database.UPGRADE_NECESSARY:
            log.info("db upgrade is necessary")
            self.db.upgrade()
        elif status == Database.UPGRADE_NOT_POSSIBLE:
            log.warn("%s (resetting db)", reason)
            log.info("reset db at `%s' (creating backup)", self.db.base_dir)
            self.db.reset()
        elif status == Database.UPGRADE_NOT_NECESSARY:
            log.debug("no upgrade necessary")
        else:
            raise CodeIntelError("unknown db upgrade status: %r" % status)
    def initialize(self):
        """Initialize the codeintel system."""
        # TODO: Implement DB cleaning.
        #self.db.clean()
        self.idxr.start()
    def _register_modules(self, extra_module_dirs=None):
        """Register codeintel/lang modules.
        @param extra_module_dirs {sequence} is an optional list of extra
            dirs in which to look for and use "codeintel|lang_*.py"
            support modules. By default just the codeintel2 package
            directory is used.
        """
        dirs = [dirname(__file__)]
        if extra_module_dirs:
            dirs += extra_module_dirs
        import_hook = self._ImportHook(self._registered_module_canon_paths.union(dirs))
        sys.meta_path.append(import_hook)
        try:
            for dir in dirs:
                for module_path in glob(join(dir, "codeintel_*.py")):
                    self._register_module(module_path)
                for module_path in glob(join(dir, "lang_*.py")):
                    warnings.warn("%s: `lang_*.py' codeintel modules are deprecated, "
                                  "use `codeintel_*.py'. Support for `lang_*.py' "
                                  "will be dropped in Komodo 5.1." % module_path,
                                  CodeIntelDeprecationWarning)
                    self._register_module(module_path)
        finally:
            sys.meta_path.remove(import_hook)
    class _ImportHook(object):
        """This is an import hook for __import__ to look for modules in the
        extra module paths as necessary.  This is needed because a bunch of the
        modules assume they're in the codeintel2 package.
        """
        _suffixes = None
        def __init__(self, paths):
            """Create an import hook
            @param paths {set} The paths to scan in
            self._paths = paths
            self._cache = None
        def find_module(self, fullname, path=None):
            parts = fullname.split(".")
            if len(parts) != 2 or parts[0] != "codeintel2":
                return None
            name = parts[-1]
            for path in self._paths:
                fullpath = join(path, name + ".py")
                if not os.path.exists(fullpath):
                    continue
                self._cache = fullpath
                return self
        def load_module(self, fullname):
            if fullname in sys.modules:
                return sys.modules[fullname]
            parts = fullname.split(".")
            if len(parts) != 2 or parts[0] != "codeintel2":
                raise ImportError("Did not expect to handle import for %s" %
                                  fullname)
            name = parts[-1]
            if self._cache and basename(self._cache) == name + ".py":
                fullpath = self._cache
            else:
                # stale cache
                for path in self._paths:
                    fullpath = join(path, name + ".py")
                    if os.path.exists(fullpath):
                else:
                    raise ImportError("Failed to locate %s" % fullname)
            try:
                module = imp.load_source(fullname, fullpath)
                sys.modules[fullname] = module
                setattr(codeintel2, name, module)
                return module
            except:
                log.exception("Failed to load %s", fullpath)
                raise
    def _register_module(self, module_path):
        """Register the given codeintel support module.
        @param module_path {str} is the path to the support module.
        @exception ImportError, CodeIntelError
        This will import the given module path and call its top-level
        `register` function passing it the Manager instance. That is
        expected to callback to one or more of:
            mgr.set_lang_info(...)
            mgr.add_hooks_handler(...)
        """
        module_canon_path = canonicalizePath(module_path)
        if module_canon_path in self._registered_module_canon_paths:
            return
        module_dir, module_name = os.path.split(module_path)
        module_name = splitext(module_name)[0]
        module_full_name = "codeintel2." + module_name
        if module_full_name in sys.modules:
            module = sys.modules[module_full_name]
        else:
            iinfo = imp.find_module(module_name, [module_dir])
            module = imp.load_module(module_name, *iinfo)
            sys.modules[module_full_name] = module
            setattr(codeintel2, module_name, module)
        if hasattr(module, "register"):
            log.debug("register `%s' support module", module_path)
            try:
                module.register(self)
            except CodeIntelError as ex:
                log.warn("error registering `%s' support module: %s",
                         module_path, ex)
            except:
                log.exception("unexpected error registering `%s' "
                              "support module", module_path)
        self._registered_module_canon_paths.add(module_canon_path)
    def set_lang_info(self, lang, silvercity_lexer=None, buf_class=None,
                      import_handler_class=None, cile_driver_class=None,
                      is_cpln_lang=False, langintel_class=None,
                      import_everything=False):
        """Called by register() functions in language support modules."""
        if silvercity_lexer:
            self.silvercity_lexer_from_lang[lang] = silvercity_lexer
        if buf_class:
            self.buf_class_from_lang[lang] = buf_class
        if langintel_class:
            self.langintel_class_from_lang[lang] = langintel_class
        if import_handler_class:
            self.import_handler_class_from_lang[lang] = import_handler_class
        if cile_driver_class is not None:
            self._is_citadel_from_lang[lang] = True
            self.citadel.set_lang_info(lang, cile_driver_class,
                                       is_cpln_lang=is_cpln_lang)
        if is_cpln_lang:
            self._is_cpln_from_lang[lang] = True
        if import_everything:
            self.db.import_everything_langs.add(lang)
    def add_hook_handler(self, hook_handler):
        """Add a handler for various codeintel hooks.
        @param hook_handler {hooks.HookHandler}
        """
        assert isinstance(hook_handler, hooks.HookHandler)
        assert hook_handler.name is not None, \
            "hook handlers must have a name: %r.name is None" % hook_handler
        for lang in hook_handler.langs:
            self._hook_handlers_from_lang[lang].append(hook_handler)
    def finalize(self, timeout=None):
        if self.citadel is not None:
            self.citadel.finalize()
        if self.isAlive():
            self.stop()
            self.join(timeout)
        self.idxr.finalize()
        if self.db is not None:
            try:
                self.db.save()
            except Exception:
                log.exception("error saving database")
            self.db = None # break the reference
    # Proxy the batch update API onto our Citadel instance.
    def batch_update(self, join=True, updater=None):
        return self.citadel.batch_update(join=join, updater=updater)
    def report_message(self, msg, details=None, notification_name="codeintel-message"):
        """Reports a unique codeintel message."""
        log.info("%s: %s: %r", notification_name, msg, details)
    def is_multilang(self, lang):
        """Return True iff this is a multi-lang language.
        I.e. Is this a language that supports embedding of different
        programming languages. For example RHTML can have Ruby and
        JavaScript content, HTML can have JavaScript content.
        """
        try:
            return issubclass(self.buf_class_from_lang[lang], UDLBuffer)
        except KeyError:
            return False # This typically happens if lang is Text
    def is_xml_lang(self, lang):
        try:
            buf_class = self.buf_class_from_lang[lang]
        except KeyError:
            return False
        return issubclass(buf_class, XMLParsingBufferMixin)
    def is_cpln_lang(self, lang):
        """Return True iff codeintel supports completion (i.e. autocomplete
        and calltips) for this language."""
        return lang in self._is_cpln_from_lang
    def get_cpln_langs(self):
        return list(self._is_cpln_from_lang.keys())
    def is_citadel_lang(self, lang):
        """Returns True if the given lang has been registered and
        is a Citadel-based language.
        A "Citadel-based" language is one that uses CIX/CIDB/CITDL tech for
        its codeintel. Note that currently not all Citadel-based langs use
        the Citadel system for completion (e.g. Tcl).
        """
        return lang in self._is_citadel_from_lang
    def get_citadel_langs(self):
        return list(self._is_citadel_from_lang.keys())
    def langintel_from_lang(self, lang):
        if lang not in self._langintel_from_lang_cache:
            try:
                langintel_class = self.langintel_class_from_lang[lang]
            except KeyError:
                langintel = ImplicitLangIntel(lang, self)
            else:
                langintel = langintel_class(self)
            self._langintel_from_lang_cache[lang] = langintel
        return self._langintel_from_lang_cache[lang] 
    def hook_handlers_from_lang(self, lang):
        return self._hook_handlers_from_lang.get(lang, []) \
               + self._hook_handlers_from_lang.get("*", [])
    def buf_from_content(self, content, lang, env=None, path=None,
                         encoding=None):
        lexer = self.silvercity_lexer_from_lang.get(lang)
        accessor = SilverCityAccessor(lexer, content)
        try:
            buf_class = self.buf_class_from_lang[lang]
        except KeyError:
            buf = ImplicitBuffer(lang, self, accessor, env, path, encoding)
        else:
            buf = buf_class(self, accessor, env, path, encoding)
        return buf
    def binary_buf_from_path(self, path, lang=None, env=None):
        buf = BinaryBuffer(lang, self, env, path)
        return buf
    MAX_FILESIZE = 1 * 1024 * 1024   # 1MB
    def buf_from_path(self, path, lang=None, env=None, encoding=None):
        if lang != "Ruby":
            path = os.path.normpath(path) # reduce ./, ../, etc.
            # TODO: Ruby handles relative imports really weirdly. If
            # `normpath()` is used in Ruby, many of the Rails tests will fail.
        # Detect and abort on large files - to avoid memory errors, bug 88487.
        # The maximum size is 1MB - someone uses source code that big?
        filestat = os.stat(path)
        if filestat.st_size > self.MAX_FILESIZE:
            log.warn("File %r has size greater than 1MB (%d)", path, filestat.st_size)
            raise CodeIntelError('File too big. Size: %d bytes, path: %r' % (
                                 filestat.st_size, path))
        if lang is None or encoding is None:
            import textinfo
            ti = textinfo.textinfo_from_path(path, encoding=encoding,
                    follow_symlinks=True)
            if lang is None:
                lang = (hasattr(ti.langinfo, "komodo_name")
                        and ti.langinfo.komodo_name
                        or ti.langinfo.name)
            if not ti.is_text:
                return self.binary_buf_from_path(path, lang, env)
            encoding = ti.encoding
            content = ti.text
        else:
            try:
                content = codecs.open(path, 'rb', encoding).read()
            except:
                log.info("Unable to read %s (likely not a text file)" % path)
                content = ''
        #TODO: Re-instate this when have solution for CILE test failures
        #      that this causes.
        #if not isabs(path) and not path.startswith("<Unsaved>"):
        #    path = abspath(path)
        return self.buf_from_content(content, lang, env, path, encoding)
    #---- Completion Evaluation Session/Queue handling
    # The current eval session (an Evaluator instance). A current session's
    # lifetime is as follows:
    # - [self._get()] Starts when the evaluator thread (this class) takes it
    #   off the queue.
    # - [self._put()] Can be aborted (via sess.ctlr.abort()) if a new eval
    #   request comes in.
    # - [eval_sess.eval()] Done when the session completes either by
    #   (1) an unexpected error during sess.eval() or (2) sess.ctlr.is_done()
    #   after sess.eval().
    _curr_eval_sess = None
    def request_eval(self, evalr):
        """Request evaluation of the given completion.
            "evalr" is the Evaluator instance.
        The manager has an evaluation thread on which this evalr will be
        scheduled. Only one request is ever eval'd at one time. A new
        request will cause an existing on to be aborted and requests made in
        the interim will be trumped by this new one.
        Dev Notes:
        - XXX Add a timeout to the put and raise error on timeout?
        """
        #evalr.eval(self)
        self.put((evalr, False))
    def request_reeval(self, evalr):
        """Occassionally evaluation will need to defer until something (e.g.
        scanning into the CIDB) is one. These sessions will re-request
        evaluation via this method.
        """
        self.put((evalr, True))
    def stop(self):
        self.put((None, None)) # Sentinel to tell thread mainloop to stop.
    def run(self):
        while 1:
            eval_sess, is_reeval = self.get()
            if eval_sess is None: # Sentinel to stop.
                break
            try:
                eval_sess.eval(self)
            except:
                    self._handle_eval_sess_error(eval_sess)
                except:
            finally:
                self._curr_eval_sess = None
        self.db.report_event(None)
    def _handle_eval_sess_error(self, eval_sess):
        exc_info = sys.exc_info()
        tb_path, tb_lineno, tb_func \
            = traceback.extract_tb(exc_info[2])[-1][:3]
        if hasattr(exc_info[0], "__name__"):
            exc_str = "%s: %s" % (exc_info[0].__name__, exc_info[1])
        else: # string exception
            exc_str = exc_info[0]
        eval_sess.ctlr.error("error evaluating %s: %s "
                             "(%s#%s in %s)", eval_sess, exc_str,
                             tb_path, tb_lineno, tb_func)
        log.exception("error evaluating %s" % eval_sess)
        eval_sess.ctlr.done("unexpected eval error")
    def _put(self, xxx_todo_changeme):
        # Only consider re-evaluation if we are still on the same eval
        # session.
        (eval_sess, is_reeval) = xxx_todo_changeme
        if is_reeval and self._curr_eval_sess is not eval_sess:
            return
        replace = True
        if hasattr(eval_sess, "ctlr") and eval_sess.ctlr and eval_sess.ctlr.keep_existing:
            # Allow multiple eval sessions; currently used for variable
            # highlighting (bug 80095), may pick up additional uses.  Note that
            # these sessions can still get wiped out by a single replace=False
            # caller.
            replace = False
        if replace:
            # We only allow *one* eval session at a time.
            # - Drop a possible accumulated eval session.
            if len(self.queue):
                pending = list(self.queue)
                self.queue.clear()
                for evalr, _ in pending:
                        evalr.close()
                    except:
                        log.exception("Failed to close evalr")
            ## - Abort the current eval session.
            if not is_reeval and self._curr_eval_sess is not None:
                self._curr_eval_sess.ctlr.abort()
        # Lazily start the eval thread.
        if not self.isAlive():
            self.start()
        Queue._put(self, (eval_sess, is_reeval))
        if replace:
            assert len(self.queue) == 1
    def _get(self):
        eval_sess, is_reeval = Queue._get(self)
        if is_reeval:
            assert self._curr_eval_sess is eval_sess
        else:
            self._curr_eval_sess = eval_sess
        return eval_sess, is_reeval
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

manager.py

Latest commit

History

manager.py

File metadata and controls
