instant.cache

Package instant :: Module cache |
[hide private]
[frames] | no frames]

# Source Code for Module instant.cache

      1  """This module contains helper functions for working with the module cache.
      2
      3  Example operations:
      4    - modulename = modulename_from_checksum(checksum)
      5    - modulename = modulename_from_checksum(compute_checksum(signature))
      6    - module = import_module_directly(path, modulename)
      7    - module = import_module(modulename)
      8    - module = import_module(checksum)
      9    - module = import_module(compute_checksum(signature))
     10    - modules = cached_modules()
     11    - modules = cached_modules(cache_dir)
     12  """
     13
     14  import os, sys, re
     15  from output import instant_warning, instant_assert, instant_debug
     16  from paths import get_default_cache_dir, validate_cache_dir
     17  from signatures import compute_checksum
     18  from locking import get_lock, release_lock
     19
     20  # TODO: We could make this an argument, but it's used indirectly several places so take care.
     21  _modulename_prefix = "instant_module_"
     22 -def modulename_from_checksum(checksum):
     23      "Construct a module name from a checksum for use in cache."
     24      return _modulename_prefix + checksum
     25
     26
     27 -def checksum_from_modulename(modulename):
     28      "Construct a module name from a checksum for use in cache."
     29      return modulename.remove(_modulename_prefix)
     30
     31
     32 -def import_module_directly(path, modulename):
     33      "Import a module with the given module name that resides in the given path."
     34      sys.path.insert(0, path)
     35      try:
     36          module = __import__(modulename)
     37      except:
     38          instant_warning("In instant.import_module_directly: Failed to import module '%s' from '%s'." % (modulename, path))
     39          module = None
     40      finally:
     41          sys.path.pop(0)
     42      return module
     43
     44
     45  _memory_cache = {}
     46 -def memory_cached_module(moduleid):
     47      "Returns the cached module if found."
     48      module = _memory_cache.get(moduleid, None)
     49      instant_debug("Found '%s' in memory cache with key '%r'." % (module, moduleid))
     50      return module
     51
     52
     53 -def place_module_in_memory_cache(moduleid, module):
     54      "Place a compiled module in cache with given id."
     55      _memory_cache[moduleid] = module
     56      instant_debug("Added module '%s' to cache with key '%r'." % (module, moduleid))
     57
     58
     59 -def is_valid_module_name(name):
     60      NAMELENGTHLIMIT = 100
     61      return len(name) < NAMELENGTHLIMIT and bool(re.search(r"^[a-zA-Z_][\w]*$", name))
     62
     63
     64 -def import_and_cache_module(path, modulename, moduleids):
     65      module = import_module_directly(path, modulename)
     66      instant_assert(module is not None, "Failed to import module found in cache. Modulename: '%s'; Path: '%s'." % (modulename, path))
     67      for moduleid in moduleids:
     68          place_module_in_memory_cache(moduleid, module)
     69      return module
     70
     71
     72 -def check_memory_cache(moduleid):
     73      # Check memory cache first with the given moduleid
     74      moduleids = [moduleid]
     75      module = memory_cached_module(moduleid)
     76      if module: return module, moduleids
     77
     78      # Get signature from moduleid if it isn't a string,
     79      # and check memory cache again
     80      if hasattr(moduleid, "signature"):
     81          moduleid = moduleid.signature()
     82          instant_debug("In instant.check_memory_cache: Got signature "\
     83                        "'%s' from moduleid.signature()." % moduleid)
     84          module = memory_cached_module(moduleid)
     85          if module:
     86              for moduleid in moduleids:
     87                  place_module_in_memory_cache(moduleid, module)
     88              return module, moduleids
     89          moduleids.append(moduleid)
     90
     91      # Construct a filename from the checksum of moduleid if it
     92      # isn't already a valid name, and check memory cache again
     93      if not is_valid_module_name(moduleid):
     94          moduleid = modulename_from_checksum(compute_checksum(moduleid))
     95          instant_debug("In instant.check_memory_cache: Constructed module name "\
     96                        "'%s' from moduleid '%s'." % (moduleid, moduleids[-1]))
     97          module = memory_cached_module(moduleid)
     98          if module: return module, moduleids
     99          moduleids.append(moduleid)
    100
    101      instant_debug("In instant.check_memory_cache: Failed to find module.")
    102      return None, moduleids
    103
    104
    105 -def check_disk_cache(modulename, cache_dir, moduleids):
    106      # Get file lock to avoid race conditions in cache
    107      lock = get_lock(cache_dir, modulename)
    108
    109      # Ensure a valid cache_dir
    110      cache_dir = validate_cache_dir(cache_dir)
    111
    112      # Check on disk, in current directory and cache directory
    113      for path in (os.getcwd(), cache_dir):
    114          if os.path.isdir(os.path.join(path, modulename)):
    115              # Found existing directory, try to import and place in memory cache
    116              module = import_and_cache_module(path, modulename, moduleids)
    117              if module:
    118                  instant_debug("In instant.check_disk_cache: Imported module "\
    119                                "'%s' from '%s'." % (modulename, path))
    120                  release_lock(lock)
    121                  return module
    122              else:
    123                  instant_debug("In instant.check_disk_cache: Failed to imported "\
    124                                "module '%s' from '%s'." % (modulename, path))
    125
    126      # All attempts failed
    127      instant_debug("In instant.check_disk_cache: Can't import module with modulename "\
    128                    "%r using cache directory %r." % (modulename, cache_dir))
    129      release_lock(lock)
    130      return None
    131
    132
    133 -def import_module(moduleid, cache_dir=None):
    134      """Import module from cache given its moduleid and an optional cache directory.
    135
    136      The moduleid can be either
    137        - the module name
    138        - a signature string, of which a checksum is taken to look up in the cache
    139        - a checksum string, which is used directly to look up in the cache
    140        - a hashable non-string object with a function moduleid.signature() which is used to get a signature string
    141      The hashable object is used to look up in the memory cache before signature() is called.
    142      If the module is found on disk, it is placed in the memory cache.
    143      """
    144      # Look for module in memory cache
    145      module, moduleids = check_memory_cache(moduleid)
    146      if module: return module
    147
    148      # Look for module in disk cache
    149      modulename = moduleids[-1]
    150      return check_disk_cache(modulename, cache_dir, moduleids)
    151
    152
    153 -def cached_modules(cache_dir=None):
    154      "Return a list with the names of all cached modules."
    155      cache_dir = validate_cache_dir(cache_dir)
    156      return os.listdir(cache_dir)
    157