Re: [BBv2] [bjam] Native response file support

From: Reece Dunn (msclrhd_at_[hidden])
Date: 2005-11-01 04:08:28

Rene Rivera wrote:
> Reece Dunn wrote:
> 
>>Now, we have @(FILE:E="EXPR") as the syntax. The response file code can get 
>>the FILE section and expand it to produce the name of the file (with STDOUT 
>>and STDERR having predefined behaviours). It can then pass the EXPR string 
>>to var_string_file to expand the string content to the file we have 
>>retrieved from FILE.
> 
> Yea, that sounds like a reasonable way to implement it. If you don't 
> have the time you can postpone implementing the STDOUT and STDERR 
> special cases.

STDOUT and STDERR are easy: just compare the string for $(STDOUT) and 
$(STDERR), then associate the output with the corresponding FILE *.

The harder ones to expand would be $(TMPPATH), $(TMPNAME) and $(TMPFILE) 
as these would require hooking into the expand.c logic in order to add 
the path_tmpdir() and tmpnam() outputs to the string expansion. I am 
leaving those for the moment, unless you can write them :).

> Yes, I forgot that one might just want a specific file in the tempdir :-)

:)

> And also:
> 
> @("$(TMPDIR)/myfile.txt":E="In the temp dir.") - write to a 
> specific file in the temp directory.
> 
> Right?

Yup.

>>NOTE: Jam.html says that the :E=value syntax assigns value to the variable 
>>if it's not been set. According to this:
>>
>> @($(STDOUT):E="Hi") @($(STDOUT):E=" There")
>>
>>would only output "Hi" according to this and we would have the same problem 
>>w.r.t. response files! The behaviour of :E= inside @() would be to always 
>>override (if a file) or append (if STDOUT/ERR).
> 
> Yea, that makes sense. And it's OK for the semantics to be slightly 
> different as it's intuitive why they are different in this case.

Ok. I thought it was best to clarify this.

> Cool... Thanks for working on this :-)

No problem :) I have attached the updated patches.

Note that there is a problem with the whitespace expansion in 
var_string_file that I can't figure out.

- Reece
 --------------030101070404090609090503 Content-Type: text/plain;
name="bjam-response.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="bjam-response.diff"

Index: build.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/build.jam,v
retrieving revision 1.74
diff -u -r1.74 build.jam
--- build.jam	26 Oct 2005 14:23:36 -0000	1.74
+++ build.jam	1 Nov 2005 08:46:05 -0000
@@ -412,6 +412,7 @@

if ( $(OS) = NT || $(NT) ) && ! NT in $(--defs)
{
+ --defs += OPT_RESPONSE ;
--defs += NT ;
}
if $(VMS)
Index: variable.c
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/variable.c,v
retrieving revision 1.12
diff -u -r1.12 variable.c
--- variable.c	26 Sep 2005 05:26:14 -0000	1.12
+++ variable.c	1 Nov 2005 08:46:04 -0000
@@ -6,6 +6,7 @@

/* This file is ALSO:
* Copyright 2001-2004 David Abrahams.
+ * Copyright 2005 Reece H. Dunn.
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
*/
@@ -19,6 +20,8 @@
# include "filesys.h"
# include "newstr.h"
# include "strings.h"
+# include "pathsys.h"
+# include <limits.h>
# include <stdlib.h>

/*
@@ -28,6 +31,7 @@
*
*	var_defines() - load a bunch of variable=value settings
*	var_string() - expand a string with variables in it
+ *	var_string_file() - expand a string with variables in it to a file
*	var_get() - get value of a user defined symbol
*	var_set() - set a variable in jam's user defined symbol table
*	var_swap() - swap a variable's value with the given one
@@ -207,6 +211,132 @@

if( in[0] == '$' && in[1] == '(' )
dollar++;
+# if defined(OPT_RESPONSE)
+ else if( in[0] == '@' && in[1] == '(' )
+ {
+ int depth = 1;
+ char *ine = in + 2;
+ char *split = 0;
+
+ /* Scan the content of the response file @() section. */
+
+ while( *ine && depth > 0 )
+ {
+ switch( *ine )
+ {
+ case '(':
+ ++depth;
+ break;
+ case ')':
+ --depth;
+ break;
+ case ':':
+ if( depth == 1 && ine[1] == 'E' && ine[2] == '=' )
+ {
+ split = ine;
+ }
+ break;
+ }
+ ++ine;
+ }
+
+ if (!split)
+ {
+ printf( "no response file specified!\n" );
+ exit( EXITBAD );
+ }
+
+ if( depth == 0 )
+ {
+ char filename[_MAX_PATH];
+ char* rsp_name = in + 2;
+ int rsp_len = split - rsp_name;
+ FILE * rsp = 0;
+
+ char save1 = rsp_name[rsp_len];
+ char save2 = ine[-1];
+
+ rsp_name[rsp_len] = ine[-1] = '\0';
+
+ if( rsp_len == 9 )
+ {
+ /* Handle the STDOUT and STDERR special cases. */
+
+ if( strcmp( rsp_name, "$(STDOUT)" ) == 0 )
+ {
+ rsp = stdout;
+ }
+ else if( strcmp( rsp_name, "$(STDERR)" ) == 0 )
+ {
+ rsp = stderr;
+ }
+ }
+
+ if( !rsp )
+ {
+ /* Expand the filename string. */
+
+ if( var_string( rsp_name, filename, _MAX_PATH, lol ) == -1 )
+ {
+ printf( "response file '%s' is too long!\n", rsp_name );
+ exit( EXITBAD );
+ }
+
+ rsp_name = filename;
+ while( isspace( *rsp_name ) ) /* Skip leading whitespace. */
+ {
+ ++rsp_name;
+ }
+
+ rsp_len = strlen( rsp_name );
+ while( isspace( rsp_name[rsp_len - 1] ) ) /* Skip trailing whitespace. */
+ {
+ --rsp_len;
+ }
+
+ rsp_name[rsp_len] = '\0';
+
+ /* Handle "path to file" filenames. */
+
+ if( rsp_name[0] == '"' && rsp_name[rsp_len - 1] == '"' )
+ {
+ rsp_name++;
+ rsp_len = rsp_len - 2;
+ save1 = '"';
+ }
+
+ rsp = fopen( rsp_name, "w" );
+ }
+
+ if (!rsp)
+ {
+ printf( "failed to write response file '%s'!\n", rsp_name );
+ exit( EXITBAD );
+ }
+
+ var_string_file( split + 3, rsp, lol );
+
+ if( rsp != stdout && rsp != stderr )
+ {
+ fflush( rsp );
+ fclose( rsp );
+
+ /* Write the name of the file to the output if it is. */
+ /* not a standard stream. */
+
+ if(( out + rsp_len ) >= oute )
+ return -1;
+
+ strcpy( out, rsp_name );
+ out += rsp_len;
+ }
+
+ rsp_name[rsp_len] = save1;
+ ine[-1] = save2;
+ }
+ in = ine;
+ }
+# endif

*out++ = *in++;
}
@@ -253,6 +383,85 @@
}

/*
+ * var_string_file() - expand a string with variables in it to a file
+ *
+ * Copies in to f; doesn't modify targets & sources.
+ */
+
+void
+var_string_file(
+	char	*in,
+	FILE *f,
+	LOL	*lol )
+{
+ char	buffer[ 1024 ];
+	char	*oute = buffer + sizeof(buffer) - 1;
+
+	while( *in )
+	{
+	char *lastword;
+	char *out = buffer;
+	int	dollar = 0;
+
+	/* Copy white space */
+
+	while( isspace( *in ) )
+	{
+	if( out >= oute )
+	{
+	printf( "variable name limit of %d exceeded!\n", sizeof(buffer));
+	exit( EXITBAD );
+	}
+
+	*out++ = *in++;
+	}
+
+ lastword = out;
+
+	/* Copy non-white space, watching for variables */
+
+	while( *in && !isspace( *in ) )
+	{
+	if( out >= oute )
+	{
+	printf( "variable name limit of %d exceeded!\n", sizeof(buffer));
+	exit( EXITBAD );
+	}
+
+	if( in[0] == '$' && in[1] == '(' )
+	{
+	dollar++;
+	}
+
+ *out++ = *in++;
+	}
+ 
+	/* If a variable encountered, expand it and and embed the */
+	/* space-separated members of the list in the output. */
+
+	if( dollar )
+	{
+	LIST	*l;
+
+	l = var_expand( L0, lastword, out, lol, 0 );
+
+	for( ; l; l = list_next( l ) )
+	{
+ fputs( l->string, f );
+ fputc( ' ', f );
+	}
+
+	list_free( l );
+	}
+	else
+	{
+ fputs( out, f );
+ fputc( ' ', f );
+	}
+	}
+}
+
+/*
* var_get() - get value of a user defined symbol
*
* Returns NULL if symbol unset.
Index: variable.h
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/jam_src/variable.h,v
retrieving revision 1.5
diff -u -r1.5 variable.h
--- variable.h	30 May 2005 03:42:38 -0000	1.5
+++ variable.h	1 Nov 2005 08:46:04 -0000
@@ -8,10 +8,13 @@
* variable.h - handle jam multi-element variables
*/

+# include <stdio.h>
+
struct hash;

void var_defines( char* const *e, int preprocess );
int var_string( char *in, char *out, int outsize, LOL *lol );
+void var_string_file( char *in, FILE *rsp, LOL *lol );
LIST * var_get( char *symbol );
void var_set( char *symbol, LIST *value, int flag );
LIST * var_swap( char *symbol, LIST *value );
 --------------030101070404090609090503 Content-Type: text/plain;
name="response.diff"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="response.diff"

Index: common.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/common.jam,v
retrieving revision 1.41
diff -u -r1.41 common.jam
--- common.jam	22 Sep 2005 13:57:23 -0000	1.41
+++ common.jam	1 Nov 2005 09:04:38 -0000
@@ -530,82 +530,6 @@
$(CP) "$(>)" "$(<)"
}

-# Cause creation of response file, containing the sources in 'sources'
-# All the targets in 'targets' will depend on response file, and response
-# file will be created before the targets are built.
-rule response-file ( targets + : sources * : the-response-file ? : properties * )
-{
- # TODO: now 'the-response-file' is just ignored. Need to remove
- # the argument altother and adjust callers.
- 
- # Create a target for response file. Note that we add 'rsp' to the target
- # name (without stripping suffix), so that response file names for c.exe
- # and c.obj are different.
- local rsp = $(targets[1]).rsp ;
- RSP on $(targets) = $(rsp) ;
- LOCATE on $(rsp) = [ on $(targets[1]) return $(LOCATE) ] ;
- DEPENDS $(targets) : $(rsp) ;
- # In theory, we don't need dependecy from response file on sources
- # because response file only needs the names of the sources.
- # In practice, bjam won't recreated TEMPORARY target without dependencies.
- DEPENDS $(rsp) : $(sources) ;
- # Make sure the directory is created before the response file. The target
- # itself depends on directory, but response file does not, yet.
- DEPENDS $(rsp) : [ on $(targets[1]) return $(LOCATE) ] ;
- 
- TEMPORARY $(rsp) ;
- 
- # Add libraries from <library> property to the list of sources.
- local libraries ;
- for local p in $(properties)
- {
- if $(p:G) = <library-file> && 
- ! [ type.is-derived [ $(p:G=).type ] SHARED_LIB ] 
- {
- libraries += $(p:G=) ;
- } 
- }
- # Get real jam targets
- local xlibraries ;
- for local l in $(libraries)
- {
- xlibraries += [ $(l).actualize ] ;
- }
- 
- sources += $(xlibraries) ; 
-
- response-file-1 $(rsp) : $(sources[1]) ;
- if $(sources[2-])
- {
- response-file-2 $(rsp) : $(sources[2-]) ;
- }
- 
- print.output $(rsp) ;
- print.text [ utility.apply-default-suffix .lib :
- [ on $(targets[1])
- return "$(LIBRARY_OPTION)$(FINDLIBS_ST)"
- "$(LIBRARY_OPTION)$(FINDLIBS_SA)"
- ] ] ;
-
- print.text
- [ on $(targets[1])
- return -D$(DEFINES) -I\"$(INCLUDES)\"
- ] ;
-}
-
-# response-file generation is broken up into two phases, the first of
-# which overwrites any existing file and the second of which appends
-# to the file, piecemeal, so that no command-line is too long.
-actions quietly response-file-1
-{
- echo "$(>)" > "$(<)"
-}
-
-actions quietly piecemeal response-file-2
-{
- echo "$(>)" >> "$(<)"
-}
-
rule __test__ ( ) {

import assert ;
Index: cw.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/cw.jam,v
retrieving revision 1.6
diff -u -r1.6 cw.jam
--- cw.jam	21 Sep 2005 13:44:06 -0000	1.6
+++ cw.jam	1 Nov 2005 08:54:27 -0000
@@ -23,6 +23,9 @@

feature.extend toolset : cw ;

+nl = "
+" ;
+
rule init ( version ? : command * : options * )
{
# TODO: fix the $(command[1]) = $(compiler) issue
@@ -147,11 +150,11 @@

actions compile.c
{
- $(.CC) -c -cwd include -lang c -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(USER_CFLAGS) -I- -I"$(INCLUDES)" -o "$(<)" "$(>)"
+ $(.CC) -c -cwd include -lang c -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")"
}
actions compile.c++
{
- $(.CC) -c -cwd include -lang c++ -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(USER_CFLAGS) -I- -I"$(INCLUDES)" -o "$(<)" "$(>)"
+ $(.CC) -c -cwd include -lang c++ -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")"
}

## linking phase
@@ -173,70 +176,31 @@

if [ os.name ] in NT
{
- rule link ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- rule link.dll ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
- }
-
- rule archive ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- actions archive bind RSP
+ actions archive bind
{
if exist "$(<[1])" DEL "$(<[1])"
- $(.LD) -library -o "$(<[1])" @"$(RSP:W)"
+ $(.LD) -library -o "$(<[1])" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}
}
else # cygwin
{
- rule link ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- rule link.dll ( targets + : sources + : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- .cygpath = "cygpath -d " ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
- }
-
- rule archive ( targets + : sources * : properties * )
- {
- common.response-file $(targets) : $(sources) : 
- : $(properties) ;
- }
-
- actions archive bind RSP
+ actions archive bind
{
_bbv2_out_="$(<)"
if test -f "$_bbv2_out_" ; then
_bbv2_existing_="$(<:W)"
fi
- $(.LD) -library -o "$(<:W)" $_bbv2_existing_ @"$(RSP:W)"
+ $(.LD) -library -o "$(<:W)" $_bbv2_existing_ @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}
}

-actions link bind DEF_FILE RSP
+actions link bind DEF_FILE
{
- $(.LD) -o "$(<[1]:W)" -L"$(LINKPATH)" $(LINKFLAGS) $(USER_LINKFLAGS) @"$(RSP:W)"
+ $(.LD) -o "$(<[1]:W)" -L"$(LINKPATH)" $(LINKFLAGS) $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}

-actions link.dll bind DEF_FILE RSP
+actions link.dll bind DEF_FILE
{
- $(.LD) -o "$(<[1]:W)" -implib "$(<[2]:W)" -L"$(LINKPATH)" $(LINKFLAGS) -f "$(DEF_FILE)" $(USER_LINKFLAGS) @"$(RSP:W)"
+ $(.LD) -o "$(<[1]:W)" -implib "$(<[2]:W)" -L"$(LINKPATH)" $(LINKFLAGS) -f "$(DEF_FILE)" $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}

Index: msvc.jam
===================================================================
RCS file: /cvsroot/boost/boost/tools/build/v2/tools/msvc.jam,v
retrieving revision 1.61
diff -u -r1.61 msvc.jam
--- msvc.jam	26 Oct 2005 06:40:14 -0000	1.61
+++ msvc.jam	1 Nov 2005 09:02:16 -0000
@@ -37,6 +37,8 @@
.versions = [ new configurations ] ;

RM = [ common.rm-command ] ;
+nl = "
+" ;

# Initialize the toolset for a specific version. As the result, path to
@@ -536,13 +538,14 @@
flags msvc.compile INCLUDES <include> ;

# The actions differ only by explicit selection of input language
-actions compile.c bind RSP
+actions compile.c bind
{
- $(.CC) /Zm800 -nologo -TC -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) @"$(RSP:W)" -c -Fo"$(<[1]:W)" && $(RM) "$(RSP)"
+ $(.CC) /Zm800 -nologo -TC -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")" -c -Fo"$(<[1]:W)"
}
-actions compile.c++ bind RSP
+
+actions compile.c++ bind
{
- $(.CC) /Zm800 -nologo -TP -U$(UNDEFS) $(CFLAGS) $(C++FLAGS) $(USER_CFLAGS) @"$(RSP:W)" -c -Fo"$(<[1]:W)" && $(RM) "$(RSP)" 
+ $(.CC) /Zm800 -nologo -TP -U$(UNDEFS) $(CFLAGS) $(C++FLAGS) $(USER_CFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")" -c -Fo"$(<[1]:W)"
}

actions compile.rc
@@ -592,23 +595,6 @@
flags msvc.link LIBRARY_OPTION <toolset>msvc : "" : unchecked ;

-rule archive ( targets + : sources * : properties * ) 
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) : $(properties) ; 
-}
-
-rule link ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) 
- : $(properties) ;
-}
-
-rule link.dll ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[3]) : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
-} 
-
# Declare action for creating static libraries
# If library exists, remove it before adding files. See
# http://article.gmane.org/gmane.comp.lib.boost.build/4241 
@@ -617,18 +603,18 @@
{ 
# The 'DEL' command would issue a message to stdout
# if the file does not exist, so need a check.
- actions archive bind RSP
+ actions archive bind
{ 
if exist "$(<[1])" DEL "$(<[1])" 
- $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(RSP)" && $(RM) "$(RSP)"
+ $(.LD) /lib /NOLOGO /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}
}
else
{
- actions archive bind RSP
+ actions archive bind
{ 
$(RM) "$(<[1])"
- $(.LD) /lib /NOLOGO /out:"$(<[1])" @"$(RSP)" && $(RM) "$(RSP)"
+ $(.LD) /lib /NOLOGO /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
}
}

@@ -644,28 +630,18 @@
# manifests are embedded as resourses and are useful in 
# any PE targets (both DLL and EXE)

-actions link bind DEF_FILE RSP
+actions link bind DEF_FILE
{
- $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /LIBPATH:"$(LINKPATH:W)" $(USER_LINKFLAGS) @"$(RSP:W)" && $(RM) "$(RSP)"
+ $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /LIBPATH:"$(LINKPATH:W)" $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
$(MANIFEST)$(<[1]).manifest $(OUTPUTRESOURCE)$(<[1]);#2
}

-actions link.dll bind DEF_FILE RSP
+actions link.dll bind DEF_FILE
{
- $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:$(DEF_FILE) $(USER_LINKFLAGS) @"$(RSP:W)" && $(RM) "$(RSP)"
+ $(.LD) /NOLOGO $(LINKFLAGS) /out:"$(<[1]:W)" /INCREMENTAL:NO /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:$(DEF_FILE) $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
$(MANIFEST)$(<[1]).manifest $(OUTPUTRESOURCE)$(<[1]);#2
}

-rule compile.c++ ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(response-file) : $(properties) ;
-}
-
-rule compile.c ( targets + : sources * : properties * )
-{
- common.response-file $(targets) : $(sources) : $(targets[2]) : $(properties) ;
-}
-

#
# Autodetection code
 --------------030101070404090609090503--