[Pyrex] New Distutils implementation for Pyrex.
Robert Bradshaw
robertwb at math.washington.edu
Fri Aug 3 22:12:25 CEST 2007
On Aug 1, 2007, at 5:55 PM, Billy G. Allie wrote:
> Greetings,
>
> I am re-implementing my PostgreSQL interface (pyPgSQL) using Pyrex
> to implement an extension for the libpq C-API. When using the
> supplied build_ext, I notices that I had to have the .pxd files in
> the same directory as the source .pyx file. Also, the generated c
> source file was placed in the same directory as the Pyrex source.
> I also could not specify Pyrex options to create a listing file,
> etc. I wanted to have the .pxd files in an separate include
> directory and I also did not want to clean up the generated files
> from the Pyrex source directory. To correct these (minor)
> annoyances, I've implesmentd a new version of buld_ext.py and
> created a version of core.py and extension.py.
>
> This implementation of Pyrex.Distutils will:
> put the generated files (*.c, *.h, *.pxi) into the [build_temp]/
> pyrex directory (unless the --inplace option is used).
> add the following Pyrex specific command-line options:
I'm not sure I like this (especially for the default).
> pyrex-include-dirs: a list of directories that Pyrex will search
> for include files (the -I option of pyrexc)
This is only necessary in the above case, right?
> pyrex-cplus: tells Pyrex to use the C++ compiler to compile the
> generated file.
Is this not handled by the "language" parameter in the
distutils.core.Extension used in setup.py
> pyrex-create-listing: tells Pyrex to place error messages in a
> listing (.lis) file.
I'm not sure this should be an extra option--it can be accomplished
easier by piping to a file (or something else for that manner).
Currently in Cython one can do cross-directory imports (like in
python--essential for a project with 100's of extension modules), and
this doesn't look like it respects that.
- Robert
> add the following Pyrex specific Extension parameters:
> pyrex_include_dirs: a python list of include directories to search.
> pyrex_cplus: a boolean parameter - =1 tells Pyrex to use the C++
> compiler (i.e. generates .cpp files instead of .c files)
> pyres_create_listing: a boolean parameter - =1 tells Pyres to
> generate a listing (.lis) file.
> These parameters can also be used in a distutils configuration file
> (i.e. setup.cfg).
>
> I am also supplying a patch to Pyrex's Main.py that will place the
> listing file in the same directory as the output file, instead of
> in the same directory as the source file.
>
> Thanks for a very useful product Greg.
>
> Billy G. Allie.
> """Pyrex.Distutils.build_ext
>
> Implements a version of the Distutils 'build_ext' command, for
> building Pyrex extension modules."""
>
> # This module should be kept compatible with Python 2.1.
>
> __revision__ = "$Id:$"
>
> import sys, os, string, re
> from types import *
> from Pyrex.Distutils.core import Command
> from distutils.errors import *
> from distutils.sysconfig import customize_compiler, get_python_version
> from distutils.dep_util import newer_group
> from Pyrex.Distutils.extension import Extension
> from distutils import log
> from distutils.dir_util import mkpath
> try:
> from Pyrex.Compiler.Main \
> import CompilationOptions, \
> default_options as pyrex_default_options, \
> compile as pyrex_compile
> from Pyrex.Compiler.Errors import PyrexError
> except ImportError:
> PyrexError = None
>
> from distutils.command import build_ext as _build_ext
>
> extension_name_re = _build_ext.extension_name_re
>
> show_compilers = _build_ext.show_compilers
>
> class build_ext(_build_ext.build_ext):
>
> description = "build C/C++ and Pyrex extensions (compile/link
> to build directory)"
>
> sep_by = _build_ext.build_ext.sep_by
> user_options = _build_ext.build_ext.user_options
> boolean_options = _build_ext.build_ext.boolean_options
> help_options = _build_ext.build_ext.help_options
>
> # Add the pyrex specific data.
> user_options.extend([
> ('pyrex-cplus', None,
> "generate C++ source files"),
> ('pyrex-create-listing', None,
> "write errors to a listing file"),
> ('pyrex-include-dirs=', None,
> "path to the Pyrex include files" + sep_by),
> ])
>
> boolean_options.extend(['pyrex-cplus', 'pyrex-create-listing'])
>
> def initialize_options(self):
> _build_ext.build_ext.initialize_options(self)
> self.pyrex_cplus = 0
> self.pyrex_create_listing = 0
> self.pyrex_include_dirs = None
>
> def finalize_options (self):
> _build_ext.build_ext.finalize_options(self)
>
> if self.pyrex_include_dirs is None:
> self.pyrex_include_dirs = []
> elif type(self.pyrex_include_dirs) is StringType:
> self.pyrex_include_dirs = \
> string.split(self.pyrex_include_dirs, os.pathsep)
> # finalize_options ()
>
>
> def build_extensions(self):
> # First, sanity-check the 'extensions' list
> self.check_extensions_list(self.extensions)
>
> for ext in self.extensions:
> ext.sources = self.pyrex_sources(ext.sources, ext)
> self.build_extension(ext)
>
> def pyrex_sources (self, sources, extension):
>
> """
> Walk the list of source files in 'sources', looking for Pyrex
> source (.pyx) files. Run Pyrex on all that are found, and
> return
> a modified 'sources' list with Pyrex source files replaced
> by the
> generated C (or C++) files.
> """
>
> if PyrexError == None:
> raise DistutilsPlatformError, \
> ("Pyrex does not appear to be installed "
> "on platform '%s'") % os.name
>
> new_sources = []
> pyrex_sources = []
> pyrex_targets = {}
>
> create_listing = self.pyrex_create_listing or \
> extension.pyrex_create_listing
> cplus = self.pyrex_cplus or extension.pyrex_cplus
>
> #Set up the include_path for the Pyres compiler:
> # 1. Start with the command line option.
> # 2. Add in any (unique) paths from the extension
> # pyrex_include_dirs.
> # 3. Add in any (unique) paths from the extension
> include_dirs
> # if and only if there there were no
> pyrex_include_dirs defined..
> includes = self.pyrex_include_dirs
> for i in extension.pyrex_include_dirs:
> if not i in includes:
> includes.append(i)
> if len(includes) < 1:
> for i in extension.include_dirs:
> if not i in includes:
> includes.append(i)
>
> # Set the target_ext to '.c'. Pyrex will change this to
> '.cpp' if
> # needed.
> if cplus:
> target_ext = '.cpp'
> else:
> target_ext = '.c'
>
> # Drop the generated C files into the temp dir, unless the
> inplace
> # option is present. In that case, drop them into the
> source tree.
>
> if self.inplace:
> target_dir = ""
> else:
> target_dir = os.path.join(self.build_temp, "pyrex")
>
> for source in sources:
> (base, ext) = os.path.splitext(source)
> if ext == ".pyx": # Pyrex source file
> new_sources.append(os.path.join(target_dir, base +
> target_ext))
> pyrex_sources.append(source)
> pyrex_targets[source] = new_sources[-1]
> else:
> new_sources.append(source)
>
> if not pyrex_sources:
> return new_sources
>
> for source in pyrex_sources:
> target = pyrex_targets[source]
> log.info("pyrexing %s to %s", source, target)
> self.mkpath(os.path.dirname(target))
> options = CompilationOptions(pyrex_default_options,
> use_listing_file =
> create_listing,
> include_path = includes,
> output_file = target,
> cplus = cplus)
> result = pyrex_compile(source, options=options)
>
> return new_sources
>
> # pyrex_sources ()
>
> # class build_ext
> """Pyrex.Distutils.core
>
> This module wraps the 'distutils.core' module and extends it to
> support
> Pyrex extension modules.
>
> This is the only module that needs to be imported to use the
> Distutils;
> provides the 'setup' function (which is to be called from the setup
> script).
> Also indirectly provides the Distribution and Command classes,
> although
> they are really defined in distutils.dist and distutils.cmd.
> """
>
> # This module should be kept compatible with Python 2.1.
>
> __revision__ = "$Id:$"
>
> import sys, os
> from types import *
>
> from distutils.debug import DEBUG
> from distutils.errors import *
> from distutils.util import grok_environment_error
>
> # Mainly import these so setup scripts can "from distutils.core
> import" them.
> from distutils.dist import Distribution
> from distutils.cmd import Command
>
> # Make sure we get Pyrex's Extension class.
> #from Pyrex.Distutils.extension import Extension
> from extension import Extension
>
> from distutils.core import *
>
> extension_keywords = extension_keywords + \
> ('pyrex_include_dirs', 'pyrex_create_listing',
> 'pyrex_cplus')
> """Pyrex.Distutils.extension
>
> Provides a modified Extension class, that understands hou to describe
> Pyrex extension modules in setup scripts."""
>
> __revision__ = "$Id:$"
>
> import os, string, sys
> from types import *
> import distutils.extension as _Extension
>
> try:
> import warnings
> except ImportError:
> warnings = None
>
> class Extension(_Extension.Extension):
> _Extension.Extension.__doc__ + \
> """pyrex_include_dirs : [string]
> list of directories to search for Pyrex header files (.pxd)
> (in
> Unix form for portability)
> pyrex_create_listing_file : boolean
> write pyrex error messages to a listing (.lis) file.
> pyrex_cplus : boolean
> use the C++ compiler for compiling and linking.
> """
>
> # When adding arguments to this constructor, be sure to update
> # setup_keywords in core.py.
> def __init__ (self, name, sources,
> include_dirs=None,
> define_macros=None,
> undef_macros=None,
> library_dirs=None,
> libraries=None,
> runtime_library_dirs=None,
> extra_objects=None,
> extra_compile_args=None,
> extra_link_args=None,
> export_symbols=None,
> swig_opts = None,
> depends=None,
> language=None,
> pyrex_include_dirs=None,
> pyrex_create_listing=0,
> pyrex_cplus=0,
> **kw # To catch unknown
> keywords
> ):
>
> _Extension.Extension.__init__(self, name, sources,
> include_dirs,
> define_macros,
> undef_macros,
> library_dirs,
> libraries,
> runtime_library_dirs,
> extra_objects,
> extra_compile_args,
> extra_link_args,
> export_symbols,
> swig_opts ,
> depends,
> language,
> **kw
> )
>
> self.pyrex_include_dirs = pyrex_include_dirs or []
> self.pyrex_create_listing = pyrex_create_listing
> self.pyrex_cplus = pyrex_cplus
>
> # class Extension
>
> read_setup_file = _Extension.read_setup_file
> # July 2002, Graham Fawcett
> #
> # this hack was inspired by the way Thomas Heller got py2exe
> # to appear as a distutil command
> #
> # we replace distutils.command.build_ext with our own version
> # and keep the old one under the module name _build_ext,
> # so that *our* build_ext can make use of it.
>
> #from build_ext import build_ext
>
> __revision__ = "$Id:$"
>
> __all__ = [ 'core.py', 'build_ext.py', 'extension.py' ]
>
> import core, build_ext, extension
>
>
> *** Main.py 2007-01-26 23:21:03.000000000 -0500
> --- /usr/lib/python2.4/site-packages/Pyrex/Compiler/Main.py
> 2007-07-30 18:42:05.000000000 -0400
> ***************
> *** 158,164 ****
> cwd = os.getcwd()
> source = os.path.join(cwd, source)
> if options.use_listing_file:
> ! result.listing_file = replace_suffix(source, ".lis")
> Errors.open_listing_file(result.listing_file,
> echo_to_stderr = options.errors_to_stderr)
> else:
> --- 158,171 ----
> cwd = os.getcwd()
> source = os.path.join(cwd, source)
> if options.use_listing_file:
> ! if options.output_file:
> ! # If an output_file is given, place the listing
> file in the
> ! # same directory as the output_file.
> ! result.listing_file = \
> ! replace_suffix(os.path.join(cwd,
> options.output_file), ".lis")
> ! else:
> ! # Otherwise, place it in the same directory as
> the source file.
> ! result.listing_file = replace_suffix(source, ".lis")
> Errors.open_listing_file(result.listing_file,
> echo_to_stderr = options.errors_to_stderr)
> else:
> _______________________________________________
> Pyrex mailing list
> Pyrex at lists.copyleft.no
> http://lists.copyleft.no/mailman/listinfo/pyrex
More information about the Pyrex
mailing list