Python-enabled bjam

From: Vladimir Prus (ghost_at_[hidden])
Date: 2004-12-15 04:49:06

Folks, 
I've tried to do some bjam/Python integration, and got something working very 
quickly. I've implemented an ability to import a Python function into bjam, 
which can then be called from bjam code.

Here's example python module I've used:

print "Python module loaded"

def foo(first, second):
print "Python function called"
print "First argument:", first
print "Second argument:", second
return ['a', 'b', 'c']

and here's jam code:

PYTHON_IMPORT_RULE aaa : foo : Python : foo ;

module Python
{
local x = [ foo 1 : 2 3 4 ] ;
ECHO "Python function returned" '$(x)' ;
} 

And the output is:

$ bin.linuxx86.debug/bjam -fpytest.jam
Python module loaded
Python function called
First argument: ['1']
Second argument: ['2', '3', '4']
Python function returned 'a' 'b' 'c'

Known problems:
1. When importing Python module we don't look in BOOST_BUILD_PATH, or in the 
directory of importing jam module.
2. It's not possible (yet) to call bjam rules from Python (but should be 
equally simple to implement)
3. I don't know what to do with BACKTRACE rule, if bjam code calls Python and 
then Python calls back bjam code.
4. Error reporting could be better.

On the other hand 
1. It's already possible to use all of Python standard library, which is cool. 
I even thinking this feature should be included in next milestone (optionally 
enabled during bjam build), so that, say, capturing an output from a command 
is possible.

2. Even if we decide to try wholesale rewrite into Python, we can keep bjam 
and 
- retain existing syntax
- retain existing tests

And finally, the patch is attached for those interested. The code at the 
beginning of build.jam configures Python location, and otherwise bjam built 
is not altered.

- Volodya

 --Boundary-00=_SiAwBFVDSP8UA81 Content-Type: text/x-diff;
charset="us-ascii";
name="bjam_python.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename="bjam_python.diff"

? A.diff
? ChangeLog
? aaa.py
? aaa.pyc
? bin.linuxx86
? bin.linuxx86.debug
? bjam_python.diff
? bootstrap.gcc
? pytest.jam
Index: build.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/build.jam,v
retrieving revision 1.62
diff -u -r1.62 build.jam
--- build.jam	21 Jun 2004 00:12:38 -0000	1.62
+++ build.jam	15 Dec 2004 09:34:15 -0000
@@ -17,6 +17,11 @@
RELEASE = 1 ;
LICENSE = 1_0 ;

+HAVE_PYTHON = 1 ;
+PYTHON_INCUDES = /usr/include/python2.3 ;
+PYTHON_LIB = -lpython2.3 ;
+
+
# Generate development debug binaries?
if --debug in $(ARGV)
{
@@ -326,6 +331,16 @@
}
--defs += YYSTACKSIZE=5000 ;

+if $(HAVE_PYTHON)
+{
+ --defs += HAVE_PYTHON ;
+ --flags += -I$(PYTHON_INCUDES) ;
+ --flags += -Wno-long-long ;
+ --libs += $(PYTHON_LIB) ;
+}
+
+
+
# The basic symbolic targets...
NOTFILE all clean dist ;
ALWAYS clean ;
Index: builtins.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/builtins.c,v
retrieving revision 1.37
diff -u -r1.37 builtins.c
--- builtins.c	7 Dec 2004 10:21:25 -0000	1.37
+++ builtins.c	15 Dec 2004 09:34:16 -0000
@@ -275,6 +275,20 @@
builtin_nearest_user_location, 0, args );
}

+#ifdef HAVE_PYTHON
+ {
+ char * args[] = { "module", 0 };
+ bind_builtin( "PYTHON_IMPORT",
+ builtin_python_import, 0, args );
+ }
+ {
+ char * args[] = { "python-module", ":", "function", ":", 
+ "jam-module", ":", "rule-name", 0 };
+ bind_builtin( "PYTHON_IMPORT_RULE",
+ builtin_python_import_rule, 0, args );
+ }
+#endif
+

# ifdef OS_NT
@@ -1172,6 +1186,61 @@
}
}

+#ifdef HAVE_PYTHON
+LIST *builtin_python_import( PARSE *parse, FRAME *frame )
+{
+ LIST* module_name = lol_get( frame->args, 0 ); 
+ static char buffer[1000];
+ snprintf(buffer, 1000, "import %s", module_name->string);
+ PyRun_SimpleString(buffer); 
+}
+
+
+LIST *builtin_python_import_rule( PARSE *parse, FRAME *frame )
+{
+ char* python_module = lol_get( frame->args, 0 )->string; 
+ char* python_function = lol_get( frame->args, 1 )->string; 
+ char* jam_module = lol_get( frame->args, 2 )->string; 
+ char* jam_rule = lol_get( frame->args, 3 )->string; 
+
+ PyObject *pName, *pModule, *pDict, *pFunc;
+
+ pName = PyString_FromString(python_module);
+ 
+ pModule = PyImport_Import(pName);
+ Py_DECREF(pName);
+
+ if (pModule != NULL) {
+ pDict = PyModule_GetDict(pModule);
+ pFunc = PyDict_GetItemString(pDict, python_function);
+
+ if (pFunc && PyCallable_Check(pFunc)) {
+
+ module_t* m = bindmodule(jam_module);
+ RULE* r = bindrule( jam_rule, m );
+
+ /* Make pFunc owned */
+ Py_INCREF(pFunc);
+
+ r->python_function = pFunc;
+ }
+ else {
+ if (PyErr_Occurred())
+ PyErr_Print();
+ fprintf(stderr, "Cannot find function \"%s\"\n", python_function);
+ }
+ Py_DECREF(pModule);
+ }
+ else {
+ PyErr_Print();
+ fprintf(stderr, "Failed to load \"%s\"\n", python_module);
+ }
+ return L0;
+
+}
+
+#endif
+

void lol_build( LOL* lol, char** elements )
{
Index: builtins.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/builtins.h,v
retrieving revision 1.15
diff -u -r1.15 builtins.h
--- builtins.h	7 Dec 2004 10:21:25 -0000	1.15
+++ builtins.h	15 Dec 2004 09:34:16 -0000
@@ -42,6 +42,8 @@
LIST *builtin_native_rule( PARSE *parse, FRAME *frame );
LIST *builtin_user_module( PARSE *parse, FRAME *frame );
LIST *builtin_nearest_user_location( PARSE *parse, FRAME *frame );
+LIST *builtin_python_import( PARSE *parse, FRAME *frame );
+LIST *builtin_python_import_rule( PARSE *parse, FRAME *frame );

void backtrace( FRAME *frame );

Index: compile.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/compile.c,v
retrieving revision 1.32
diff -u -r1.32 compile.c
--- compile.c	7 Dec 2004 10:21:25 -0000	1.32
+++ compile.c	15 Dec 2004 09:34:16 -0000
@@ -889,6 +889,65 @@
}
}

+static LIST*
+call_python_function(RULE* r, FRAME* frame)
+{
+ LIST* result = 0;
+ PyObject* arguments = PyTuple_New(frame->args->count);
+ int i ;
+ PyObject* py_result;
+
+ for(i = 0; i < frame->args->count; ++i)
+ {
+ PyObject* arg = PyList_New(0);
+ LIST* l = lol_get( frame->args, i);
+
+ for(; l; l = l->next)
+ {
+ PyObject* v = PyString_FromString(l->string);
+ /* Steals reference to 'v' */
+ PyList_Append(arg, v); 
+ }
+ /* Steals reference to 'arg' */
+ PyTuple_SetItem(arguments, i, arg);
+ }
+
+ py_result = PyObject_CallObject(r->python_function, arguments);
+ Py_DECREF(arguments);
+ if (py_result != NULL) {
+ 
+ if (PyList_Check(py_result)) {
+ int size = PyList_Size(py_result);
+ int i;
+ for(i = 0; i < size; ++i)
+ {
+ PyObject* item = PyList_GetItem(py_result, i);
+ if (PyString_Check(item))
+ {
+ result = list_new(result, 
+ newstr(PyString_AsString(item)));
+ }
+ else
+ {
+ fprintf(stderr, "Non-string object returned by Python call\n");
+ }
+ }
+ }
+ else
+ {
+ fprintf(stderr, "Non-list object returned by Python call\n");
+ }
+
+ Py_DECREF(py_result);
+ }
+ else {
+ PyErr_Print();
+ fprintf(stderr,"Call failed\n");
+ }
+ 
+ return result;
+}
+
/*
* evaluate_rule() - execute a rule invocation
*/
@@ -925,6 +984,11 @@
rulename = l->string;
rule = bindrule( l->string, frame->module );

+ if (rule->python_function)
+ {
+ return call_python_function(rule, frame);
+ }
+
/* drop the rule name */
l = list_pop_front( l );

Index: jam.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/jam.c,v
retrieving revision 1.30
diff -u -r1.30 jam.c
--- jam.c	9 Sep 2004 11:57:09 -0000	1.30
+++ jam.c	15 Dec 2004 09:34:16 -0000
@@ -210,6 +210,9 @@
# ifdef OS_MAC
InitGraf(&qd.thePort);
# endif
+#ifdef HAVE_PYTHON
+ Py_Initialize();
+#endif

argc--, argv++;

@@ -463,5 +466,10 @@
if( globs.cmdout )
fclose( globs.cmdout );

+#ifdef HAVE_PYTHON
+ Py_Finalize();
+#endif
+
+
return status ? EXITBAD : EXITOK;
}
Index: jam.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/jam.h,v
retrieving revision 1.15
diff -u -r1.15 jam.h
--- jam.h	19 Oct 2004 15:12:58 -0000	1.15
+++ jam.h	15 Dec 2004 09:34:16 -0000
@@ -37,6 +37,11 @@

#ifndef JAM_H_VP_2003_08_01
#define JAM_H_VP_2003_08_01
+
+#ifdef HAVE_PYTHON
+#include <Python.h>
+#endif
+
/*
* VMS, OPENVMS
*/
Index: rules.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/rules.c,v
retrieving revision 1.19
diff -u -r1.19 rules.c
--- rules.c	1 Jun 2004 05:42:36 -0000	1.19
+++ rules.c	15 Dec 2004 09:34:16 -0000
@@ -82,6 +82,9 @@
r->arguments = 0;
r->exported = 0;
r->module = target_module;
+#ifdef HAVE_PYTHON
+ r->python_function = 0;
+#endif
}
return r;
}
Index: rules.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/rules.h,v
retrieving revision 1.25
diff -u -r1.25 rules.h
--- rules.h	18 Nov 2004 19:38:16 -0000	1.25
+++ rules.h	15 Dec 2004 09:34:16 -0000
@@ -87,6 +87,9 @@
* appear in the global module and be
* automatically imported into other modules
*/
+#ifdef HAVE_PYTHON
+ PyObject* python_function;
+#endif 
};

/* ACTIONS - a chain of ACTIONs */
 --Boundary-00=_SiAwBFVDSP8UA81--