[BBv2] [bjam] Native response file support

From: Reece Dunn (msclrhd_at_[hidden])
Date: 2005-10-30 09:18:56

The results of the discussion on the "[BBv2] fix for response file 
regeneration" thread was to have builtin support for response files 
using the @(...) syntax based on Matt Armstrong's perforce Jam branch.

The bjam-response.diff patch adds @(...) syntax support for NT builds of 
BJam. This is based on Matt's code but was written from scratch and is 
improved from my initial submission. When you use @(...) the code (in 
variable.c) expands the variables in the '...' to a temporary file and 
then outputs the location of that file.

The response.diff patch is the BBv2 changes needed for native response 
file support. This removes the response file logic from common.jam and 
modifies msvc.jam and cw.jam to use the new syntax.

- Reece
 --------------040202080600080408010603 Content-Type: text/plain;
name="bjam-response.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="bjam-response.diff"

Index: build.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/build.jam,v
retrieving revision 1.74
diff -u -r1.74 build.jam
--- build.jam	26 Oct 2005 14:23:36 -0000	1.74
+++ build.jam	30 Oct 2005 11:57:23 -0000
@@ -412,6 +412,7 @@

if ( $(OS) = NT || $(NT) ) && ! NT in $(--defs)
{
+ --defs += OPT_RESPONSE ;
--defs += NT ;
}
if $(VMS)
Index: execnt.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/execnt.c,v
retrieving revision 1.22
diff -u -r1.22 execnt.c
--- execnt.c	10 Oct 2005 20:33:20 -0000	1.22
+++ execnt.c	30 Oct 2005 11:57:23 -0000
@@ -13,6 +13,7 @@
# include "jam.h"
# include "lists.h"
# include "execcmd.h"
+# include "pathsys.h"
# include "debug.h"
# include <errno.h>
# include <assert.h>
@@ -440,34 +441,6 @@
#endif 
}

-/* SVA - handle temp dirs with spaces in the path */
-static const char *getTempDir(void)
-{
- static char tempPath[_MAX_PATH];
- static char *pTempPath=NULL;
-
- if(pTempPath == NULL)
- {
- char *p;
-
- p = getenv("TEMP");
- if(p == NULL)
- {
- p = getenv("TMP");
- }
- if(p == NULL)
- {
- pTempPath = "\\temp";
- }
- else
- {
- GetShortPathName(p, tempPath, _MAX_PATH);
- pTempPath = tempPath;
- }
- }
- return pTempPath;
-}
-
/* 64-bit arithmetic helpers */

/* Compute the carry bit from the addition of two 32-bit unsigned numbers */
@@ -580,7 +553,7 @@
const char *tempdir;
DWORD procID;

- tempdir = getTempDir();
+ tempdir = path_to_temp();

/* SVA - allocate 64 other just to be safe */
cmdtab[ slot ].tempfile = malloc( strlen( tempdir ) + 64 );
@@ -589,7 +562,7 @@

procID = GetCurrentProcessId();

- sprintf( cmdtab[ slot ].tempfile, "%s\\jam%d-%02d.bat", 
+ sprintf( cmdtab[ slot ].tempfile, "%sjam%d-%02d.bat", 
tempdir, procID, slot );	
}

Index: pathsys.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/pathsys.h,v
retrieving revision 1.4
diff -u -r1.4 pathsys.h
--- pathsys.h	15 May 2003 13:07:09 -0000	1.4
+++ pathsys.h	30 Oct 2005 11:57:24 -0000
@@ -61,6 +61,8 @@
unaltered. */
char* short_path_to_long_path(char* short_path);

+const char* path_to_temp( void );
+
#endif

#endif
Index: pathunix.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/pathunix.c,v
retrieving revision 1.13
diff -u -r1.13 pathunix.c
--- pathunix.c	29 Oct 2004 09:25:33 -0000	1.13
+++ pathunix.c	30 Oct 2005 11:57:23 -0000
@@ -391,6 +391,34 @@
return newstr(short_path);
}

+const char* path_to_temp( void )
+{
+ static char tempPath[_MAX_PATH];
+ static char *pTempPath=NULL;
+
+ if(pTempPath == NULL)
+ {
+ char *p;
+
+ p = getenv("TEMP");
+ if(p == NULL)
+ {
+ p = getenv("TMP");
+ }
+ if(p == NULL)
+ {
+ pTempPath = "\\temp";
+ }
+ else
+ {
+ GetShortPathName(p, tempPath, _MAX_PATH);
+ pTempPath = tempPath;
+ }
+ strcat( pTempPath, "\\" );
+ }
+ return pTempPath;
+}
+
#endif

Index: variable.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/variable.c,v
retrieving revision 1.12
diff -u -r1.12 variable.c
--- variable.c	26 Sep 2005 05:26:14 -0000	1.12
+++ variable.c	30 Oct 2005 11:57:23 -0000
@@ -6,6 +6,7 @@

/* This file is ALSO:
* Copyright 2001-2004 David Abrahams.
+ * Copyright 2005 Reece H. Dunn.
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
*/
@@ -19,6 +20,8 @@
# include "filesys.h"
# include "newstr.h"
# include "strings.h"
+# include "pathsys.h"
+# include <limits.h>
# include <stdlib.h>

/*
@@ -28,6 +31,7 @@
*
*	var_defines() - load a bunch of variable=value settings
*	var_string() - expand a string with variables in it
+ *	var_string_file() - expand a string with variables in it to a file
*	var_get() - get value of a user defined symbol
*	var_set() - set a variable in jam's user defined symbol table
*	var_swap() - swap a variable's value with the given one
@@ -207,6 +211,64 @@

if( in[0] == '$' && in[1] == '(' )
dollar++;
+# if defined(OPT_RESPONSE)
+ else if( in[0] == '@' && in[1] == '(' )
+ {
+ int depth = 1;
+ char *ine = in + 2;
+
+ while( *ine && depth > 0 )
+ {
+ switch( *ine )
+ {
+ case '(':
+ ++depth;
+ break;
+ case ')':
+ --depth;
+ break;
+ }
+ ++ine;
+ }
+
+ if( depth == 0 )
+ {
+ static int rsp_count = 0;
+ char save = ine[ -1 ];
+ char rsp_name[ _MAX_PATH ];
+ int rsp_len;
+ FILE * rsp;
+
+ const char *tempdir = path_to_temp();
+
+ sprintf( rsp_name, "%srsp%06d.rsp", tempdir, rsp_count );
+ rsp_count++;
+ rsp_len = strlen( rsp_name );
+ rsp = fopen( rsp_name, "w" );
+
+ if (!rsp)
+ {
+ printf( "failed to write response file!\n" );
+ exit( EXITBAD );
+ }
+
+ ine[ -1 ] = '\0';
+
+ var_string_file( in + 2, rsp, lol );
+
+ ine[ -1 ] = save;
+
+ fclose( rsp );
+
+ if(( out + rsp_len ) >= oute )
+ return -1;
+
+ strcpy( out, rsp_name );
+ out += rsp_len;
+ }
+ in = ine;
+ }
+# endif

*out++ = *in++;
}
@@ -253,6 +315,76 @@
}

/*
+ * var_string_file() - expand a string with variables in it to a file
+ *
+ * Copies in to f; doesn't modify targets & sources.
+ */
+
+void
+var_string_file(
+	char	*in,
+	FILE *f,
+	LOL	*lol )
+{
+ char	buffer[ 1024 ];
+	char	*oute = buffer + sizeof(buffer) - 1;
+
+	while( *in )
+	{
+	char *out = buffer;
+	int	dollar = 0;
+
+	/* Skip white space */
+
+	while( isspace( *in ) )
+	{
+	in++;
+	}
+
+	/* Copy non-white space, watching for variables */
+
+	while( *in && !isspace( *in ) )
+	{
+	if( out >= oute )
+	{
+	printf( "variable name limit of %d exceeded!\n", sizeof(buffer));
+	exit( EXITBAD );
+	}
+
+	if( in[0] == '$' && in[1] == '(' )
+	{
+	dollar++;
+	}
+
+ *out++ = *in++;
+	}
+ 
+	/* If a variable encountered, expand it and and embed the */
+	/* space-separated members of the list in the output. */
+
+	if( dollar )
+	{
+	LIST	*l;
+
+	l = var_expand( L0, buffer, out, lol, 0 );
+
+	for( ; l; l = list_next( l ) )
+	{
+ fputs( l->string, f );
+ fputc( '\n', f );
+	}
+
+	list_free( l );
+	}
+	else
+	{
+ fputs( out, f );
+ fputc( '\n', f );
+	}
+	}
+}
+
+/*
* var_get() - get value of a user defined symbol
*
* Returns NULL if symbol unset.
Index: variable.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/variable.h,v
retrieving revision 1.5
diff -u -r1.5 variable.h
--- variable.h	30 May 2005 03:42:38 -0000	1.5
+++ variable.h	30 Oct 2005 11:57:23 -0000
@@ -8,10 +8,13 @@
* variable.h - handle jam multi-element variables
*/

+# include <stdio.h>
+
struct hash;

void var_defines( char* const *e, int preprocess );
int var_string( char *in, char *out, int outsize, LOL *lol );
+void var_string_file( char *in, FILE *rsp, LOL *lol );
LIST * var_get( char *symbol );
void var_set( char *symbol, LIST *value, int flag );
LIST * var_swap( char *symbol, LIST *value );
 --------------040202080600080408010603 Content-Type: text/plain;
name="response.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="response.diff"

? response.diff
? types
Index: common.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/common.jam,v
retrieving revision 1.41
diff -u -r1.41 common.jam
--- common.jam	22 Sep 2005 13:57:23 -0000	1.41
+++ common.jam	30 Oct 2005 14:02:36 -0000
@@ -389,11 +389,13 @@
{
RM = del /f ;
CP = copy ;
+ MV = move ;
}
else
{
RM = rm -f ;
CP = cp ;
+ MV = mv ;
}

nl = "
@@ -409,6 +411,11 @@
return $(CP) ;
}

+rule mv-command ( ) 
+{
+ return $(MV) ;
+}
+

# Returns the command needed to set the environment variable on the
# current platform. The variable setting persists through all
@@ -530,82 +537,6 @@
$(CP) "$(>)" "$(<)"
}

-# Cause creation of response file, containing the sources in 'sources'
-# All the targets in 'targets' will depend on response file, and response
-# file will be created before the targets are built.
-rule response-file ( targets + : sources * : the-response-file ? : properties * )
-{
- # TODO: now 'the-response-file' is just ignored. Need to remove
- # the argument altother and adjust callers.
- 
- # Create a target for response file. Note that we add 'rsp' to the target
- # name (without stripping suffix), so that response file names for c.exe
- # and c.obj are different.
- local rsp = $(targets[1]).rsp ;
- RSP on $(targets) = $(rsp) ;
- LOCATE on $(rsp) = [ on $(targets[1]) return $(LOCATE) ] ;
- DEPENDS $(targets) : $(rsp) ;
- # In theory, we don't need dependecy from response file on sources
- # because response file only needs the names of the sources.
- # In practice, bjam won't recreated TEMPORARY target without dependencies.
- DEPENDS $(rsp) : $(sources) ;
- # Make sure the directory is created before the response file. The target
- # itself depends on directory, but response file does not, yet.
- DEPENDS $(rsp) : [ on $(targets[1]) return $(LOCATE) ] ;
- 
- TEMPORARY $(rsp) ;
- 
- # Add libraries from <library> property to the list of sources.
- local libraries ;
- for local p in $(properties)
- {
- if $(p:G) = <library-file> && 
- ! [ type.is-derived [ $(p:G=).type ] SHARED_LIB ] 
- {
- libraries += $(p:G=) ;
- } 
- }
- # Get real jam targets
- local xlibraries ;
- for local l in $(libraries)
- {
- xlibraries += [ $(l).actualize ] ;
- }
- 
- sources += $(xlibraries) ; 
-
- response-file-1 $(rsp) : $(sources[1]) ;
- if $(sources[2-])
- {
- response-file-2 $(rsp) : $(sources[2-]) ;
- }
- 
- print.output $(rsp) ;
- print.text [ utility.apply-default-suffix .lib :
- [ on $(targets[1])
- return "$(LIBRARY_OPTION)$(FINDLIBS_ST)"
- "$(LIBRARY_OPTION)$(FINDLIBS_SA)"
- ] ] ;
-
- print.text
- [ on $(targets[1])
- return -D$(DEFINES) -I\"$(INCLUDES)\"
- ] ;
-}
-
-# response-file generation is broken up into two phases, the first of
-# which overwrites any existing file and the second of which appends
-# to the file, piecemeal, so that no command-line is too long.
-actions quietly response-file-1
-{
- echo "$(>)" > "$(<)"
-}
-
-actions quietly piecemeal response-file-2
-{
- echo "$(>)" >> "$(<)"
-}
-
rule __test__ ( ) {

import assert ;
Index: cw.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/cw.jam,v
retrieving revision 1.6
diff -u -r1.6 cw.jam
--- cw.jam	21 Sep 2005 13:44:06 -0000	1.6
+++ cw.jam	30 Oct 2005 14:02:48 -0000
@@ -23,6 +23,8 @@

feature.extend toolset : cw ;

+MV = [ common.mv-command ] ;
+
rule init ( version ? : command * : options * )
{
# TODO: fix the $(command[1]) = $(compiler) issue
@@ -147,11 +149,13 @@

actions compile.c
{
- $(.CC) -c -cwd include -lang c -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(USER_CFLAGS) -I- -I"$(INCLUDES)" -o "$(<)" "$(>)"
+ $(MV) @("$(>)" -D$(DEFINES) "-I$(INCLUDES)") "$(<[1]:W).rsp"
+ $(.CC) -c -cwd include -lang c -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"$(<[1]:W).rsp"
}
actions compile.c++
{
- $(.CC) -c -cwd include -lang c++ -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(USER_CFLAGS) -I- -I"$(INCLUDES)" -o "$(<)" "$(>)"
+ $(MV) @("$(>)" -D$(DEFINES) "-I$(INCLUDES)") "$(<[1]:W).rsp"
+ $(.CC) -c -cwd include -lang c++ -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"$(<[1]:W).rsp"
}

## linking phase
@@ -173,70 +177,35 @@

if [ os.name ] in NT
{
- rule link ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- rule link.dll ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
- }
-
- rule archive ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- actions archive bind RSP
+ actions archive bind
{
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
if exist "$(<[1])" DEL "$(<[1])"
- $(.LD) -library -o "$(<[1])" @"$(RSP:W)"
+ $(.LD) -library -o "$(<[1])" @"$(<[1]:W).rsp"
}
}
else # cygwin
{
- rule link ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- rule link.dll ( targets + : sources + : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- .cygpath = "cygpath -d " ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
- }
-
- rule archive ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- actions archive bind RSP
+ actions archive bind
{
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
_bbv2_out_="$(<)"
if test -f "$_bbv2_out_" ; then
_bbv2_existing_="$(<:W)"
fi
- $(.LD) -library -o "$(<:W)" $_bbv2_existing_ @"$(RSP:W)"
+ $(.LD) -library -o "$(<:W)" $_bbv2_existing_ @"$(<[1]:W).rsp"
}
}

-actions link bind DEF_FILE RSP
+actions link bind DEF_FILE
{
- $(.LD) -o "$(<[1]:W)" -L"$(LINKPATH)" $(LINKFLAGS) $(USER_LINKFLAGS) @"$(RSP:W)"
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
+ $(.LD) -o "$(<[1]:W)" -L"$(LINKPATH)" $(LINKFLAGS) $(USER_LINKFLAGS) @"$(<[1]:W).rsp"
}

-actions link.dll bind DEF_FILE RSP
+actions link.dll bind DEF_FILE
{
- $(.LD) -o "$(<[1]:W)" -implib "$(<[2]:W)" -L"$(LINKPATH)" $(LINKFLAGS) -f "$(DEF_FILE)" $(USER_LINKFLAGS) @"$(RSP:W)"
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
+ $(.LD) -o "$(<[1]:W)" -implib "$(<[2]:W)" -L"$(LINKPATH)" $(LINKFLAGS) -f "$(DEF_FILE)" $(USER_LINKFLAGS) @"$(<[1]:W).rsp"
}

Index: msvc.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/msvc.jam,v
retrieving revision 1.61
diff -u -r1.61 msvc.jam
--- msvc.jam	26 Oct 2005 06:40:14 -0000	1.61
+++ msvc.jam	30 Oct 2005 14:03:04 -0000
@@ -37,6 +37,7 @@
.versions = [ new configurations ] ;

RM = [ common.rm-command ] ;
+MV = [ common.mv-command ] ;

# Initialize the toolset for a specific version. As the result, path to
@@ -536,13 +537,15 @@
flags msvc.compile INCLUDES <include> ;

# The actions differ only by explicit selection of input language
-actions compile.c bind RSP
+actions compile.c bind
{
- $(.CC) /Zm800 -nologo -TC -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) @"$(RSP:W)" -c -Fo"$(<[1]:W)" && $(RM) "$(RSP)"
+ $(MV) @("$(>)" -D$(DEFINES) "-I$(INCLUDES)") "$(<[1]:W).rsp"
+ $(.CC) /Zm800 -nologo -TC -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) @"$(<[1]:W).rsp" -c -Fo"$(<[1]:W)"
}
-actions compile.c++ bind RSP
+actions compile.c++ bind
{
- $(.CC) /Zm800 -nologo -TP -U$(UNDEFS) $(CFLAGS) $(C++FLAGS) $(USER_CFLAGS) @"$(RSP:W)" -c -Fo"$(<[1]:W)" && $(RM) "$(RSP)" 
+ $(MV) @("$(>)" -D$(DEFINES) "-I$(INCLUDES)") "$(<[1]:W).rsp"
+ $(.CC) /Zm800 -nologo -TP -U$(UNDEFS) $(CFLAGS) $(C++FLAGS) $(USER_CFLAGS) @"$(<[1]:W).rsp" -c -Fo"$(<[1]:W)"
}

actions compile.rc
@@ -592,23 +595,6 @@
flags msvc.link LIBRARY_OPTION <toolset>msvc : "" : unchecked ;

-rule archive ( targets + : sources * : properties * ) 
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) : $(properties) ; 
-}
-
-rule link ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) 
- : $(properties) ;
-}
-
-rule link.dll ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[3]) : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
-} 
-
# Declare action for creating static libraries
# If library exists, remove it before adding files. See
# http://article.gmane.org/gmane.comp.lib.boost.build/4241 
@@ -617,18 +603,20 @@
{ 
# The 'DEL' command would issue a message to stdout
# if the file does not exist, so need a check.
- actions archive bind RSP
+ actions archive bind
{ 
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
if exist "$(<[1])" DEL "$(<[1])" 
- $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(RSP)" && $(RM) "$(RSP)"
+ $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(<[1]:W).rsp"
}
}
else
{
- actions archive bind RSP
+ actions archive bind
{ 
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
$(RM) "$(<[1])"
- $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(RSP)" && $(RM) "$(RSP)"
+ $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(<[1]:W).rsp"
}
}

@@ -644,28 +632,20 @@
# manifests are embedded as resourses and are useful in 
# any PE targets (both DLL and EXE)

-actions link bind DEF_FILE RSP
+actions link bind DEF_FILE
{
- $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /LIBPATH:"$(LINKPATH:W)" $(USER_LINKFLAGS) @"$(RSP:W)" && $(RM) "$(RSP)"
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
+ $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /LIBPATH:"$(LINKPATH:W)" $(USER_LINKFLAGS) @"$(<[1]:W).rsp"
$(MANIFEST)$(<[1]).manifest $(OUTPUTRESOURCE)$(<[1]);#2
}

-actions link.dll bind DEF_FILE RSP
+actions link.dll bind DEF_FILE
{
- $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:$(DEF_FILE) $(USER_LINKFLAGS) @"$(RSP:W)" && $(RM) "$(RSP)"
+ $(MV) @("$(>)") "$(<[1]:W).rsp"
+ $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:$(DEF_FILE) $(USER_LINKFLAGS) @"$(<[1]:W).rsp"
$(MANIFEST)$(<[1]).manifest $(OUTPUTRESOURCE)$(<[1]);#2
}

-rule compile.c++ ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(response-file) : $(properties) ;
-}
-
-rule compile.c ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) : $(properties) ;
-}
-

#
# Autodetection code
 --------------040202080600080408010603--