[Boost-commit] svn:boost r46085 - branches/CMake/release/tools/build/CMake

From: troy_at_[hidden]
Date: 2008-06-03 15:56:05

Author: troy
Date: 2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
New Revision: 46085
URL: http://svn.boost.org/trac/boost/changeset/46085

Log:
More infrastructure for reporting build results via XML-RPC.  New make targets 'slave-start', 'slave-finish' and 'slave-info'.  Reporting of SVN url/rev, uname info, toolset.  To come: XML-RPC server that does sensible things with this on the other side.
Added:
   branches/CMake/release/tools/build/CMake/boost_build_slave.py.in   (contents, props changed)
   branches/CMake/release/tools/build/CMake/finish.py.in   (contents, props changed)
   branches/CMake/release/tools/build/CMake/info.py.in   (contents, props changed)
   branches/CMake/release/tools/build/CMake/marshal.py.in   (contents, props changed)
   branches/CMake/release/tools/build/CMake/start.py.in   (contents, props changed)
Removed:
   branches/CMake/release/tools/build/CMake/xmlize.py.in
Text files modified: 
   branches/CMake/release/tools/build/CMake/BoostBuildSlave.cmake |    56 +++++++++++++++++++++++++++++++++------ 
   branches/CMake/release/tools/build/CMake/BoostCore.cmake       |     4 --                                      
   branches/CMake/release/tools/build/CMake/post.py.in            |    32 +++++++++++++++-------                  
   3 files changed, 69 insertions(+), 23 deletions(-)

Modified: branches/CMake/release/tools/build/CMake/BoostBuildSlave.cmake
==============================================================================
--- branches/CMake/release/tools/build/CMake/BoostBuildSlave.cmake	(original)
+++ branches/CMake/release/tools/build/CMake/BoostBuildSlave.cmake	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -9,6 +9,11 @@
 ##########################################################################
 option(BOOST_BUILD_SLAVE "Be a build slave, report build/testing" OFF)
 
+set(BOOST_BUILD_SLAVE_SUBMIT_URL "http://localhost:8000" CACHE STRING
+  "URL to post regression testing results to.")
+
+set(BOOST_BUILD_SLAVE_PYTHONPATH "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}")
+
 message(STATUS "Configuring test/compile drivers")
   
 if(CMAKE_VERBOSE_MAKEFILE)
@@ -18,20 +23,32 @@
 endif(CMAKE_VERBOSE_MAKEFILE)
 
 # the program that does the posting when BOOST_BUILD_SLAVE is on
-configure_file(tools/build/CMake/post.py.in ${CMAKE_BINARY_DIR}/post.py @ONLY)
+
+foreach(PYFILE boost_build_slave passthru marshal start finish info post)
+  configure_file(tools/build/CMake/${PYFILE}.py.in 
+    ${BOOST_BUILD_SLAVE_PYTHONPATH}/${PYFILE}.py 
+    @ONLY
+    )
+endforeach()
+
+#configure_file(tools/build/CMake/boost_build_slave.py.in ${CMAKE_BINARY_DIR}/post.py @ONLY)
+
+
+#configure_file(tools/build/CMake/post.py.in ${CMAKE_BINARY_DIR}/post.py @ONLY)
 
 # the programs that sit between 'make' and the test binaries, one does xmlization
 # the other doesn't.  T
-configure_file(tools/build/CMake/passthru.py.in ${CMAKE_BINARY_DIR}/passthru.py @ONLY)
-configure_file(tools/build/CMake/xmlize.py.in ${CMAKE_BINARY_DIR}/xmlize.py @ONLY)
+#configure_file(tools/build/CMake/passthru.py.in ${CMAKE_BINARY_DIR}/passthru.py @ONLY)
+#configure_file(tools/build/CMake/xmlize.py.in ${CMAKE_BINARY_DIR}/xmlize.py @ONLY)
+#configure_file(tools/build/CMake/start.py.in ${CMAKE_BINARY_DIR}/start.py @ONLY)
 
 #
 # the test driver is either 
 #
 if(BOOST_BUILD_SLAVE)
-  set(BOOST_TEST_DRIVER ${CMAKE_BINARY_DIR}/xmlize.py)
+  set(BOOST_TEST_DRIVER ${BOOST_BUILD_SLAVE_PYTHONPATH}/marshal.py)
 else(BOOST_BUILD_SLAVE)
-  set(BOOST_TEST_DRIVER ${CMAKE_BINARY_DIR}/passthru.py)
+  set(BOOST_TEST_DRIVER ${BOOST_BUILD_SLAVE_PYTHONPATH}/passthru.py)
 endif(BOOST_BUILD_SLAVE)
 
 if(BOOST_BUILD_SLAVE)
@@ -51,17 +68,38 @@
   set(CMAKE_CXX_LINK_EXECUTABLE  
     "${BOOST_TEST_DRIVER} <CMAKE_CURRENT_BINARY_DIR> link_executable <TARGET> ${CMAKE_CXX_LINK_EXECUTABLE}") 
 
+
+  #
+  #  Custom targets for talking to the server via xmlrpc
+  #
+  add_custom_target(slave-start
+    COMMAND ${BOOST_BUILD_SLAVE_PYTHONPATH}/start.py
+    COMMENT "Slave starting build"
+    )
+
+  add_custom_target(slave-finish
+    COMMAND ${BOOST_BUILD_SLAVE_PYTHONPATH}/finish.py
+    COMMENT "Slave finishing build"
+    )
+
+  add_custom_target(slave-info
+    COMMAND ${BOOST_BUILD_SLAVE_PYTHONPATH}/info.py
+    COMMENT "Print slave info"
+    )
+
+
 endif(BOOST_BUILD_SLAVE)
 
-macro(boost_post_results PROJECT_NAME_ LIBRARY_TARGET WHAT LOGDIR)
 
+
+macro(boost_post_results PROJECT_NAME_ LIBRARY_TARGET WHAT LOGDIR)
   if(BOOST_BUILD_SLAVE)
     add_custom_command(TARGET ${LIBRARY_TARGET}
       POST_BUILD
-      COMMAND ${CMAKE_BINARY_DIR}/post.py ${PROJECT_NAME_} ${WHAT} ${LOGDIR}
-      COMMENT "${PROJECT_NAME_}: posting build results for ${WHAT} from ${LOGDIR}"
+      COMMAND ${BOOST_BUILD_SLAVE_PYTHONPATH}/post.py ${PROJECT_NAME_} ${WHAT} ${LOGDIR}
+      COMMENT "Submitting results for '${WHAT}' in  ${PROJECT_NAME_}"
       )
-    set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${LOGDIR}/Log.xml)
+    set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${LOGDIR}/Log.marshal)
 
   endif(BOOST_BUILD_SLAVE)
 endmacro(boost_post_results LIBRARY_TARGET)

Modified: branches/CMake/release/tools/build/CMake/BoostCore.cmake
==============================================================================
--- branches/CMake/release/tools/build/CMake/BoostCore.cmake	(original)
+++ branches/CMake/release/tools/build/CMake/BoostCore.cmake	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -282,9 +282,7 @@
       if(${BOOST_TEST_LIB_OPTION})
         add_custom_target(${PROJECT_NAME}-test)
 
-	add_dependencies(test
-	  ${PROJECT_NAME}-test
-	  )
+	add_dependencies(test ${PROJECT_NAME}-test)
 
         # the last argument here, the binary directory that the 
         # logs are in, has to match the binary directory

Added: branches/CMake/release/tools/build/CMake/boost_build_slave.py.in
==============================================================================
--- (empty file)
+++ branches/CMake/release/tools/build/CMake/boost_build_slave.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -0,0 +1,40 @@
+#!/usr/bin/python
+#
+#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
+#  
+#  Distributed under the Boost Software License, Version 1.0.
+#  See accompanying file LICENSE_1_0.txt or copy at
+#  http://www.boost.org/LICENSE_1_0.txt
+#
+
+#
+#  Utilities, variables, imports for build slave python
+#
+
+import os, os.path, marshal, xmlrpclib, pysvn, socket
+from pprint import pprint
+
+repo_path = "@CMAKE_SOURCE_DIR@"
+client = pysvn.Client()
+svn_entry = client.info(repo_path)
+hostname = socket.gethostname()
+uname = os.uname()
+toolset = "@BOOST_TOOLSET@"
+
+xmlrpc_url = "@BOOST_BUILD_SLAVE_SUBMIT_URL@"
+
+build_id_file = os.path.join("@BOOST_BUILD_SLAVE_PYTHONPATH@", "build_id.txt")
+
+try:
+    f = open(build_id_file)
+    build_id = int(f.read())
+except:
+    build_id = None
+
+def set_build_id(build_id):
+    print "Setting new build id %d locally" % build_id
+    f = open(build_id_file, "w")
+    f.write(str(build_id))
+    f.close()
+
+

Added: branches/CMake/release/tools/build/CMake/finish.py.in
==============================================================================
--- (empty file)
+++ branches/CMake/release/tools/build/CMake/finish.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -0,0 +1,23 @@
+#!/usr/bin/python
+#
+#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
+#  
+#  Distributed under the Boost Software License, Version 1.0.
+#  See accompanying file LICENSE_1_0.txt or copy at
+#  http://www.boost.org/LICENSE_1_0.txt
+#
+
+#
+#  Start a new build, notify server via xmlrpc
+#
+
+import sys
+sys.path.append("@BOOST_BUILD_SLAVE_PYTHONPATH@")
+from boost_build_slave import *
+
+print '\nFinishing build %d with %s via XML-RPC' % (build_id, xmlrpc_url)
+s = xmlrpclib.Server(xmlrpc_url)
+
+s.traash.finish_build(build_id)
+
+

Added: branches/CMake/release/tools/build/CMake/info.py.in
==============================================================================
--- (empty file)
+++ branches/CMake/release/tools/build/CMake/info.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+#
+#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
+#  
+#  Distributed under the Boost Software License, Version 1.0.
+#  See accompanying file LICENSE_1_0.txt or copy at
+#  http://www.boost.org/LICENSE_1_0.txt
+#
+
+#
+#  Start a new build, notify server via xmlrpc
+#
+
+import sys
+sys.path.append("@BOOST_BUILD_SLAVE_PYTHONPATH@")
+from boost_build_slave import *
+
+print 'Url:', svn_entry.url, "at rev", svn_entry.revision.number
+print 'Hostname:', hostname
+print 'Uname:', uname
+print 'Toolset:', toolset
+print 'Build ID:', build_id
+
+

Added: branches/CMake/release/tools/build/CMake/marshal.py.in
==============================================================================
--- (empty file)
+++ branches/CMake/release/tools/build/CMake/marshal.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+#
+#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
+#  
+#  Distributed under the Boost Software License, Version 1.0.
+#  See accompanying file LICENSE_1_0.txt or copy at
+#  http://www.boost.org/LICENSE_1_0.txt
+#
+#
+#  Compiler driver.  Takes a few arguments describing what is to be done
+#  (used to mark up the output) and executes compiler in a subshell, checking
+#  for errors and marshalling output to disk.
+#
+import sys, os.path, marshal
+from subprocess import Popen, PIPE
+import xml.etree.ElementTree as ET
+
+log = os.path.join(sys.argv[1], "Log.marshal")
+op = sys.argv[2]
+target = sys.argv[3]
+argv = sys.argv[4:]
+expect_fail = op.endswith("fail")
+
+print "***\n*** Executing op:" + op + "\n*** " + str(argv) + "\n*** log=" + log + "\n***"
+
+#
+#  execute subprocess
+#
+subproc = None
+returncode = None
+ex = None
+try:
+    subproc = Popen(argv, stdout=PIPE, stderr=PIPE)
+    returncode = subproc.wait()
+except EnvironmentError, e:
+    ex = e
+
+#
+#  Collect, possibly xmlize, and print subprocess return info
+#
+
+result = { 'expect_fail' : expect_fail }
+
+if not ex:
+    
+    result['returncode'] = returncode
+    result['stdout'] = subproc.stdout.read()
+    result['stderr'] = subproc.stderr.read()
+
+    if returncode != 0 and not expect_fail:
+        print "*** stdout:" + result['stdout']
+        print "*** stderr:" + result['stderr']
+    else:
+        print "*** OK"
+    
+else:
+    result['errno'] = ex.errno
+    result['filename'] = ex.filename
+    result['message'] = ex.message
+    result['strerror'] = ex.strerror
+    print "Errno:" + str(ex.errno) + ": " + ex.strerror
+
+result.update({'op' : op,
+               'target' : target,
+               'cmdline' : argv })
+
+f = open(log, "a")
+marshal.dump(result, f)
+
+if ex:
+    sys.exit(ex.errno)
+else:
+    if expect_fail:
+        if returncode != 0:
+            sys.exit(0)
+        else: 
+            print "UNEXPECTED SUCCESS"
+            sys.exit(1)  # we need an exit status for 'unexpected success'
+    else:
+        sys.exit(returncode)
+
+    
+

Modified: branches/CMake/release/tools/build/CMake/post.py.in
==============================================================================
--- branches/CMake/release/tools/build/CMake/post.py.in	(original)
+++ branches/CMake/release/tools/build/CMake/post.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -11,9 +11,11 @@
 #  HTTP POST the build log
 #
 
-import sys, os, os.path
-from subprocess import Popen, PIPE
-import xml.etree.ElementTree as ET
+import sys
+sys.path.append("@BOOST_BUILD_SLAVE_PYTHONPATH@")
+from boost_build_slave import *
+
+s = xmlrpclib.Server(xmlrpc_url)
 
 project_name = sys.argv[1]
 library_target = sys.argv[2]
@@ -22,15 +24,23 @@
 print "\n>>>\n>>> Project " + project_name \
       + "\n>>> POST build log for " + library_target \
       + "\n>>> from log dir" + logdir \
-      + "\n>>>\n"
-
-#
-# Here you would actually do an http post.  Just print for now.
-#
-p = os.path.join(logdir, "Log.xml")
+      + "\n>>> to " + xmlrpc_url \
+      + "\n>>> Server build ID: %d" % build_id \
+      + "\n>>>"
 
+p = os.path.join(logdir, "Log.marshal")
 f = open(p)
-print f.read() + "\n"
-f.close()
+
+while True:
+    try:
+        r = marshal.load(f)
+        r.update({ 'build_id' : build_id,
+                   'project' : project_name,
+                   'upper_target' : library_target})
+        pprint(r)
+        s.traash.step(r)
+    except EOFError, e:
+        break
+
 os.remove(p)
 

Added: branches/CMake/release/tools/build/CMake/start.py.in
==============================================================================
--- (empty file)
+++ branches/CMake/release/tools/build/CMake/start.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+#
+#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
+#  
+#  Distributed under the Boost Software License, Version 1.0.
+#  See accompanying file LICENSE_1_0.txt or copy at
+#  http://www.boost.org/LICENSE_1_0.txt
+#
+
+#
+#  Start a new build, notify server via xmlrpc
+#
+
+import sys
+sys.path.append("@BOOST_BUILD_SLAVE_PYTHONPATH@")
+from boost_build_slave import *
+
+print 'Url:', svn_entry.url, "at rev", svn_entry.revision.number
+print 'Hostname:', hostname
+print 'Uname:', uname
+print 'Toolset:', toolset
+
+print '\nNotifying %s of new build via XML-RPC' % xmlrpc_url
+s = xmlrpclib.Server(xmlrpc_url)
+
+build_id  = s.traash.start_build({ 'svn_url' : svn_entry.url,
+                                   'svn_rev' : svn_entry.revision.number,
+                                   'sysname' : uname[0],
+                                   'nodename' : uname[1],
+                                   'sys_release' : uname[2],
+                                   'sys_version' : uname[3],
+                                   'sys_machine' : uname[4],
+                                   'hostname' : hostname,
+                                   'toolset' : toolset})
+
+set_build_id(build_id)

Deleted: branches/CMake/release/tools/build/CMake/xmlize.py.in
==============================================================================
--- branches/CMake/release/tools/build/CMake/xmlize.py.in	2008-06-03 15:56:04 EDT (Tue, 03 Jun 2008)
+++ (empty file)
@@ -1,89 +0,0 @@
-#!/usr/bin/python
-#
-#  copyright (C) 2008  troy d. straszheim  <troy_at_[hidden]>
-#  
-#  Distributed under the Boost Software License, Version 1.0.
-#  See accompanying file LICENSE_1_0.txt or copy at
-#  http://www.boost.org/LICENSE_1_0.txt
-#
-
-#
-#  Compiler driver.  Takes a few arguments describing what is to be done
-#  (used to mark up the output) and executes compiler in a subshell, checking
-#  for errors and xmlizing any output.
-#
-import sys, os.path
-from subprocess import Popen, PIPE
-import xml.etree.ElementTree as ET
-
-log = os.path.join(sys.argv[1], "Log.xml")
-op = sys.argv[2]
-target = sys.argv[3]
-argv = sys.argv[4:]
-expect_fail = op.endswith("fail")
-passthru = op.startswith("passthru")
-
-if not passthru:
-    print "***\n*** Executing op:" + op + "\n*** " + str(argv) + "\n*** log=" + log + "\n***"
-
-#
-#  execute subprocess
-#
-subproc = None
-returncode = None
-ex = None
-try:
-    subproc = Popen(argv, stdout=PIPE, stderr=PIPE)
-    returncode = subproc.wait()
-except EnvironmentError, e:
-    ex = e
-
-#
-#  Collect, possibly xmlize, and print subprocess return info
-#
-
-if not passthru:
-    root = ET.Element(op)
-    ET.SubElement(root, "target").text = target
-
-    ET.SubElement(root, "commandline").text = " ".join(argv)
-
-    if not ex:
-        ET.SubElement(root, "returncode").text = str(returncode)
-        stdout = subproc.stdout.read()
-        stderr = subproc.stderr.read()
-        if returncode != 0 and not expect_fail:
-            print "*** stdout:" + stdout
-            print "*** stderr:" + stderr
-        else:
-            print "*** OK"
-            ET.SubElement(root, "stdout").text = stdout
-            ET.SubElement(root, "stderr").text = stderr
-    
-    else:
-        ET.SubElement(root, "errno").text = str(ex.errno)
-        ET.SubElement(root, "filename").text = ex.filename
-        ET.SubElement(root, "message").text = ex.message
-        ET.SubElement(root, "strerror").text = ex.strerror
-        print "Errno:" + str(ex.errno) + ": " + ex.strerror
-
-    f = open(log, "a")
-    f.write(ET.tostring(root) + "\n")
-else:
-    print subproc.stdout.read(),
-    print subproc.stderr.read(),
-
-if ex:
-    sys.exit(ex.errno)
-else:
-    if expect_fail:
-        if returncode != 0:
-            sys.exit(0)
-        else: 
-            print "UNEXPECTED SUCCESS"
-            sys.exit(1)  # we need an exit status for 'unexpected success'
-    else:
-        sys.exit(returncode)
-
-    
-