instant.build

Package instant :: Module build |
[hide private]
[frames] | no frames]

# Source Code for Module instant.build

      1  """This module contains the main part of Instant, the build_module function."""
      2
      3  import os, sys, shutil, glob
      4  from itertools import chain
      5
      6  # TODO: Import only the official interface
      7  from output import *
      8  from config import header_and_libs_from_pkgconfig
      9  from paths import *
     10  from signatures import *
     11  from cache import *
     12  from codegeneration import *
     13  from locking import get_lock, release_lock
     14
     15 -def assert_is_str(x):
     16      instant_assert(isinstance(x, str),
     17          "In instant.build_module: Expecting string.")
     18
     19 -def assert_is_bool(x):
     20      instant_assert(isinstance(x, bool),
     21          "In instant.build_module: Expecting bool.")
     22
     23 -def assert_is_str_list(x):
     24      instant_assert(isinstance(x, (list, tuple)),
     25          "In instant.build_module: Expecting sequence.")
     26      instant_assert(all(isinstance(i, str) for i in x),
     27          "In instant.build_module: Expecting sequence of strings.")
     28
     29 -def strip_strings(x):
     30      assert_is_str_list(x)
     31      return [s.strip() for s in x]
     32
     33 -def arg_strings(x):
     34      if isinstance(x, str):
     35          x = x.split()
     36      return strip_strings(x)
     37
     38
     39 -def copy_files(source, dest, files):
     40      """Copy a list of files from a source directory to a destination directory.
     41      This may seem a bit complicated, but a lot of this code is error checking."""
     42      if os.path.exists(dest):
     43          overwriting = set(files) & set(glob.glob(os.path.join(dest, "*")))
     44          if overwriting:
     45              instant_warning("In instant.copy_files: Path '%s' already exists, "\
     46                  "overwriting existing files: %r." % (dest, list(overwriting)))
     47      else:
     48          os.mkdir(dest)
     49
     50      if source != dest:
     51          instant_debug("In instant.copy_files: Copying files %r from %r to %r"\
     52              % (files, source, dest))
     53
     54          for f in files:
     55              a = os.path.join(source, f)
     56              b = os.path.join(dest, f)
     57              instant_assert(a != b, "In instant.copy_files: Seems like the "\
     58                  "input files are absolute paths, should be relative to "\
     59                  "source. (%r, %r)" % (a, b))
     60              instant_assert(os.path.isfile(a), "In instant.copy_files: "\
     61                  "Missing source file '%s'." % a)
     62              if os.path.isfile(b):
     63                  os.remove(b)
     64              shutil.copyfile(a, b)
     65
     66
     67 -def recompile(modulename, module_path, setup_name, new_compilation_checksum):
     68      """Recompile module if the new checksum is different from
     69      the one in the checksum file in the module directory."""
     70      # Check if the old checksum matches the new one
     71      compilation_checksum_filename = "%s.checksum" % modulename
     72      if os.path.exists(compilation_checksum_filename):
     73          checksum_file = open(compilation_checksum_filename)
     74          old_compilation_checksum = checksum_file.readline()
     75          checksum_file.close()
     76          if old_compilation_checksum == new_compilation_checksum:
     77              return
     78
     79      # Verify that SWIG is on the system
     80      (swig_stat, swig_out) = get_status_output("swig -version")
     81      if swig_stat != 0:
     82          instant_error("In instant.recompile: Could not find swig!"\
     83              " You can download swig from http://www.swig.org")
     84
     85      # Create log file for logging of compilation errors
     86      compile_log_filename = os.path.join(module_path, "compile.log")
     87      compile_log_file = open(compile_log_filename, "w")
     88      try:
     89          # Build module
     90          cmd = "python %s build_ext" % setup_name
     91          instant_info("--- Instant: compiling ---")
     92          instant_debug("cmd = %s" % cmd)
     93          ret, output = get_status_output(cmd)
     94          compile_log_file.write(output)
     95          compile_log_file.flush()
     96          if ret != 0:
     97              if os.path.exists(compilation_checksum_filename):
     98                  os.remove(compilation_checksum_filename)
     99              instant_error("In instant.recompile: The module did not "\
    100                  "compile, see '%s'" % compile_log_filename)
    101
    102          # 'Install' module
    103          cmd = "python %s install --install-platlib=." % setup_name
    104          instant_debug("cmd = %s" % cmd)
    105          ret, output = get_status_output(cmd)
    106          compile_log_file.write(output)
    107          compile_log_file.flush()
    108          if ret != 0:
    109              if os.path.exists(compilation_checksum_filename):
    110                  os.remove(compilation_checksum_filename)
    111              instant_error("In instant.recompile: Could not 'install' "\
    112                  "the module, see '%s'" % compile_log_filename)
    113      finally:
    114          compile_log_file.close()
    115
    116      # Compilation succeeded, write new_compilation_checksum to checksum_file
    117      write_file(compilation_checksum_filename, new_compilation_checksum)
    118
    119
    120 -def copy_to_cache(module_path, cache_dir, modulename):
    121      "Copy module directory to cache."
    122      # Get lock, check if the module exists, _otherwise_ copy the
    123      # finished compiled module from /tmp/foo to the cache directory,
    124      # and then release lock
    125      lock = get_lock(cache_dir, modulename)
    126
    127      # Validate the path
    128      cache_module_path = os.path.join(cache_dir, modulename)
    129      if os.path.exists(cache_module_path):
    130          # This indicates a race condition has happened (and is being avoided!).
    131          instant_warning("In instant.build_module: Path '%s' already exists,"\
    132              " but module wasn't found in cache previously. Not overwriting,"\
    133              " assuming this module is valid." % cache_module_path)
    134
    135          release_lock(lock)
    136          return cache_module_path
    137
    138          # Not deleting anymore, relying on locking system
    139          #shutil.rmtree(cache_module_path, ignore_errors=True)
    140
    141      # Error checks
    142      instant_assert(os.path.isdir(module_path), "In instant.build_module:"\
    143          " Cannot copy non-existing directory %r!" % module_path)
    144      instant_assert(not os.path.isdir(cache_module_path),
    145          "In instant.build_module: Cache directory %r shouldn't exist "\
    146          "at this point!" % cache_module_path)
    147      instant_debug("In instant.build_module: Copying built module from %r"\
    148          " to cache at %r" % (module_path, cache_module_path))
    149
    150      # Do the copying
    151      shutil.copytree(module_path, cache_module_path)
    152      delete_temp_dir()
    153      release_lock(lock)
    154      return cache_module_path
    155
    156
    157 -def build_module(modulename=None, source_directory=".",
    158                   code="", init_code="",
    159                   additional_definitions="", additional_declarations="",
    160                   sources=[], wrap_headers=[],
    161                   local_headers=[], system_headers=[],
    162                   include_dirs=['.'], library_dirs=[], libraries=[],
    163                   swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'],
    164                   swig_include_dirs = [],
    165                   cppargs=['-O2'], lddargs=[],
    166                   object_files=[], arrays=[],
    167                   generate_interface=True, generate_setup=True,
    168                   signature=None, cache_dir=None):
    169      """Generate and compile a module from C/C++ code using SWIG.
    170
    171      Arguments:
    172      ==========
    173      The keyword arguments are as follows:
    174        - B{modulename}:
    175          - The name you want for the module.
    176            If specified, the module will not be cached.
    177            If missing, a name will be constructed based on
    178            a checksum of the other arguments, and the module
    179            will be placed in the global cache. String.
    180        - B{source_directory}:
    181          - The directory where user supplied files reside. The files
    182          given in B{sources}, B{wrap_headers}, and B{local_headers}
    183          are expected to exist in this directory. String.
    184        - B{code}:
    185          - A string containing C or C++ code to be compiled and wrapped. String.
    186        - B{init_code}:
    187          - Code that should be executed when the Instant module is
    188          imported. This code is inserted in the SWIG interface file, and is
    189          used for instance for calling C{import_array()} used for the
    190          initialization of NumPy arrays. String.
    191        - B{additional_definitions}:
    192          - Additional definitions (typically needed for inheritance)
    193          for interface file. These definitions should be given as triple-quoted
    194          strings in the case they span multiple lines, and are placed both in the
    195          initial block for C/C++ code (C{%{,%}}-block), and the main section
    196          of the interface file. String.
    197        - B{additional_declarations}:
    198          - Additional declarations (typically needed for inheritance)
    199          for interface file. These declarations should be given as triple-quoted
    200          strings in the case they span multiple lines, and are plaves in the main
    201          section of the interface file. String.
    202        - B{sources}:
    203          - Source files to compile and link with the module. These
    204          files are compiled togehter with the SWIG-generated wrapper file into
    205          the final library file. Should reside in directory specified in
    206          B{source_directory}. List of strings.
    207        - B{wrap_headers}:
    208          - Local header files that should be wrapped by SWIG. The
    209          files specified will be included both in the initial block for C/C++ code
    210          (with a C directive) and in the main section of the interface file (with
    211          a SWIG directive). Should reside in directory specified in
    212          B{source_directory}. List of strings.
    213        - B{local_headers}:
    214          - Local header files required to compile the wrapped
    215          code. The files specified will be included in the initial block for
    216          C/C++ code (with a C directive). Should reside in directory specified in
    217          B{source_directory}. List of strings.
    218        - B{system_headers}:
    219          - System header files required to compile the wrapped
    220          code. The files specified will be included in the initial block for C/C++
    221          code (with a C directive). List of strings.
    222        - B{include_dirs}:
    223          - Directories to search for header files for building the
    224          extension module. Needs to be absolute path names. List of strings.
    225        - B{library_dirs}:
    226          - Directories to search for libraries (C{-l}) for building
    227          the extension module. Needs to be absolute paths. List of strings.
    228        - B{libraries}:
    229          -  Libraries needed by the Instant module. The libraries will
    230          be linked in from the shared object file. The initial C{-l} is added
    231          automatically. List of strings.
    232        - B{swigargs}:
    233          - List of arguments to swig, e.g. C{["-lpointers.i"]}
    234            to include the SWIG pointers.i library.
    235        - B{swig_include_dirs}:
    236          - A list of directories to include in the 'swig' command.
    237        - B{cppargs}:
    238          - List of arguments to the compiler, e.g. C{["-Wall", "-fopenmp"]}.
    239        - B{lddargs}:
    240          - List of arguments to the linker, e.g. C{["-E", "-U"]}.
    241        - B{object_files}:
    242          - If you want to compile the files yourself. TODO: Not yet supported.
    243        - B{arrays}:
    244          - A nested list describing the C arrays to be made from NumPy arrays.
    245            If the NumPy array is 1D, the inner list should contain strings with
    246            the variable names for length of the array and the array itself.
    247            If the NumPy array is a matrix or a tensor, the inner list should
    248            contain strings with variable names for the number of dimensions,
    249            the length in each dimension, and the array itself, respectively.
    250        - B{generate_interface}:
    251          - A bool to indicate if you want to generate the interface files.
    252        - B{generate_setup}:
    253          - A bool to indicate if you want to generate the setup.py file.
    254        - B{signature}:
    255          - A signature string to identify the form instead of the source code.
    256        - B{cache_dir}:
    257          - A directory to look for cached modules and place new ones.
    258            If missing, a default directory is used. Note that the module
    259            will not be cached if B{modulename} is specified.
    260            The cache directory should not be used for anything else.
    261      """
    262
    263      # Store original directory to be able to restore later
    264      original_path = os.getcwd()
    265
    266      # --- Validate arguments
    267
    268      instant_assert(modulename is None or isinstance(modulename, str),
    269          "In instant.build_module: Expecting modulename to be string or None.")
    270      assert_is_str(source_directory)
    271      source_directory = os.path.abspath(source_directory)
    272      assert_is_str(code)
    273      assert_is_str(init_code)
    274      assert_is_str(additional_definitions)
    275      assert_is_str(additional_declarations)
    276      sources           = strip_strings(sources)
    277      wrap_headers      = strip_strings(wrap_headers)
    278      local_headers     = strip_strings(local_headers)
    279      system_headers    = strip_strings(system_headers)
    280      include_dirs      = strip_strings(include_dirs)
    281      library_dirs      = strip_strings(library_dirs)
    282      libraries         = strip_strings(libraries)
    283      swigargs          = arg_strings(swigargs)
    284      swig_include_dirs = strip_strings(swig_include_dirs)
    285      cppargs           = arg_strings(cppargs)
    286      lddargs           = arg_strings(lddargs)
    287      object_files      = strip_strings(object_files)
    288      arrays            = [strip_strings(a) for a in arrays]
    289      assert_is_bool(generate_interface)
    290      assert_is_bool(generate_setup)
    291      instant_assert(   signature is None \
    292                     or isinstance(signature, str) \
    293                     or hasattr(signature, "signature"),
    294          "In instant.build_module: Expecting modulename to be string or None.")
    295      instant_assert(not (signature is not None and modulename is not None),
    296          "In instant.build_module: Can't have both modulename and signature.")
    297
    298      # --- Replace arguments with defaults if necessary
    299
    300      cache_dir = validate_cache_dir(cache_dir)
    301
    302      # Split sources by file-suffix (.c or .cpp)
    303      csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')]
    304      cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')]
    305      instant_assert(len(csrcs) + len(cppsrcs) == len(sources),
    306          "In instant.build_module: Source files must have '.c' or '.cpp' suffix")
    307
    308      # --- Debugging code
    309      instant_debug('In instant.build_module:')
    310      instant_debug('::: Begin Arguments :::')
    311      instant_debug('    modulename: %r' % modulename)
    312      instant_debug('    source_directory: %r' % source_directory)
    313      instant_debug('    code: %r' % code)
    314      instant_debug('    init_code: %r' % init_code)
    315      instant_debug('    additional_definitions: %r' % additional_definitions)
    316      instant_debug('    additional_declarations: %r' % additional_declarations)
    317      instant_debug('    sources: %r' % sources)
    318      instant_debug('    csrcs: %r' % csrcs)
    319      instant_debug('    cppsrcs: %r' % cppsrcs)
    320      instant_debug('    wrap_headers: %r' % wrap_headers)
    321      instant_debug('    local_headers: %r' % local_headers)
    322      instant_debug('    system_headers: %r' % system_headers)
    323      instant_debug('    include_dirs: %r' % include_dirs)
    324      instant_debug('    library_dirs: %r' % library_dirs)
    325      instant_debug('    libraries: %r' % libraries)
    326      instant_debug('    swigargs: %r' % swigargs)
    327      instant_debug('    swig_include_dirs: %r' % swig_include_dirs)
    328      instant_debug('    cppargs: %r' % cppargs)
    329      instant_debug('    lddargs: %r' % lddargs)
    330      instant_debug('    object_files: %r' % object_files)
    331      instant_debug('    arrays: %r' % arrays)
    332      instant_debug('    generate_interface: %r' % generate_interface)
    333      instant_debug('    generate_setup: %r' % generate_setup)
    334      instant_debug('    signature: %r' % signature)
    335      instant_debug('    cache_dir: %r' % cache_dir)
    336      instant_debug('::: End Arguments :::')
    337
    338      # --- Setup module directory, making it and copying
    339      #     files to it if necessary, and compute a modulename
    340      #     if it isn't specified explicitly
    341
    342      if modulename is None:
    343          # Compute a signature if we have none passed by the user:
    344          if signature is None:
    345              # Collect arguments used for checksum creation,
    346              # including everything that affects the interface
    347              # file generation and module compilation.
    348              checksum_args = ( \
    349                  # We don't care about the modulename, that's what we're trying to construct!
    350                  #modulename,
    351                  # We don't care where the user code resides:
    352                  #source_directory,
    353                  code, init_code,
    354                  additional_definitions,
    355                  additional_declarations,
    356                  # Skipping filenames, since we use the file contents:
    357                  #sources, wrap_headers,
    358                  #local_headers,
    359                  system_headers,
    360                  include_dirs, library_dirs, libraries,
    361                  swig_include_dirs, swigargs, cppargs, lddargs,
    362                  object_files, arrays,
    363                  generate_interface, generate_setup,
    364                  # The signature isn't defined, and the cache_dir doesn't affect the module:
    365                  #signature, cache_dir)
    366                  )
    367              allfiles = sources + wrap_headers + local_headers
    368              allfiles = [os.path.join(source_directory, file) for file in allfiles]
    369              text = "\n".join((str(a) for a in checksum_args))
    370              signature = modulename_from_checksum(compute_checksum(text, allfiles))
    371              modulename = signature
    372              moduleids = [signature]
    373          else:
    374              module, moduleids = check_memory_cache(signature)
    375              if module: return module
    376              modulename = moduleids[-1]
    377
    378          # Look for module in disk cache
    379          module = check_disk_cache(modulename, cache_dir, moduleids)
    380          if module: return module
    381
    382          # Make a temporary module path for compilation
    383          module_path = os.path.join(get_temp_dir(), modulename)
    384          instant_assert(not os.path.exists(module_path),
    385              "In instant.build_module: Not expecting module_path to exist: '%s'"\
    386              % module_path)
    387          os.mkdir(module_path)
    388          use_cache = True
    389      else:
    390          use_cache = False
    391          moduleids = []
    392          module_path = os.path.join(original_path, modulename)
    393          if not os.path.exists(module_path):
    394              os.mkdir(module_path)
    395
    396          ## Look for module in memory cache
    397          #module, moduleids = check_memory_cache(modulename)
    398          #if module: return module
    399          #instant_assert(modulename == moduleids[-1] and len(moduleids) == 1, "Logic breach.")
    400          ## Look for module in local directory
    401          #module = check_disk_cache(modulename, original_path, moduleids)
    402          #if module: return module
    403
    404      # Wrapping rest of code in try-block to
    405      # clean up at the end if something fails.
    406      try:
    407          # --- Copy user-supplied files to module path
    408
    409          module_path = os.path.abspath(module_path)
    410          files_to_copy = sources + wrap_headers + local_headers + object_files
    411          copy_files(source_directory, module_path, files_to_copy)
    412          # At this point, all user input files should reside in module_path.
    413
    414          # --- Generate additional files in module directory
    415          os.chdir(module_path)
    416
    417          # Generate __init__.py which imports compiled module contents
    418          write_file("__init__.py", "from %s import *" % modulename)
    419
    420          # Generate SWIG interface if wanted
    421          ifile_name = "%s.i" % modulename
    422          if generate_interface:
    423              write_interfacefile(ifile_name, modulename, code, init_code,
    424                  additional_definitions, additional_declarations, system_headers,
    425                  local_headers, wrap_headers, arrays)
    426
    427          # Generate setup.py if wanted
    428          setup_name = "setup.py"
    429          if generate_setup:
    430              write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, \
    431                          include_dirs, library_dirs, libraries, swig_include_dirs, \
    432                          swigargs, cppargs, lddargs)
    433
    434          # --- Build module
    435
    436          # At this point we have all the files, and can make the
    437          # total checksum from all file contents. This is used to
    438          # decide whether the module needs recompilation or not.
    439
    440          # Compute new_compilation_checksum
    441          # Collect arguments used for checksum creation,
    442          # including everything that affects the module compilation.
    443          # Since the interface file is included in allfiles,
    444          # we don't need stuff that modifies it here.
    445          checksum_args = ( \
    446                           # We don't care about the modulename, that's what
    447                           # we're trying to construct!
    448                           #modulename,
    449                           # We don't care where the user code resides:
    450                           #source_directory,
    451                           #code, init_code,
    452                           #additional_definitions, additional_declarations,
    453                           # Skipping filenames, since we use the file contents:
    454                           #sources, wrap_headers,
    455                           #local_headers,
    456                           system_headers,
    457                           include_dirs, library_dirs, libraries,
    458                           swigargs, swig_include_dirs, cppargs, lddargs,
    459                           object_files, #arrays,
    460                           #generate_interface, generate_setup,
    461                           # The signature isn't defined, and the
    462                           # cache_dir doesn't affect the module:
    463                           #signature, cache_dir)
    464                           )
    465          text = "\n".join((str(a) for a in checksum_args))
    466          allfiles = sources + wrap_headers + local_headers + [ifile_name]
    467          new_compilation_checksum = compute_checksum(text, allfiles)
    468
    469          # Recompile if necessary
    470          recompile(modulename, module_path, setup_name, new_compilation_checksum)
    471
    472          # --- Load, cache, and return module
    473
    474          # Copy compiled module to cache
    475          if use_cache:
    476              module_path = copy_to_cache(module_path, cache_dir, modulename)
    477
    478          # Import module and place in memory cache
    479          # Do not use locks if use_cache is False:
    480          if use_cache:
    481              lock = get_lock(cache_dir, modulename)
    482          module = import_and_cache_module(module_path, modulename, moduleids)
    483          if use_cache:
    484              release_lock(lock)
    485          if not module:
    486              instant_error("Failed to import newly compiled module!")
    487
    488          instant_debug("In instant.build_module: Returning %s from build_module."\
    489              % module)
    490          return module
    491          # The end!
    492
    493      finally:
    494          # Always get back to original directory.
    495          os.chdir(original_path)
    496
    497      instant_error("In instant.build_module: Should never reach this point!")
    498      # end build_module
    499