Index: engine/command.c =================================================================== --- engine/command.c (revision 86425) +++ engine/command.c (working copy) @@ -24,6 +24,37 @@ /* + * cmdlist_append_cmd + */ +CMDLIST * cmdlist_append_cmd( CMDLIST * l, CMD * cmd ) +{ + CMDLIST * result = (CMDLIST *)BJAM_MALLOC( sizeof( CMDLIST ) ); + result->iscmd = 1; + result->next = l; + result->cmd = cmd; + return result; +} + +CMDLIST * cmdlist_append_target( CMDLIST * l, TARGET * t ) +{ + CMDLIST * result = (CMDLIST *)BJAM_MALLOC( sizeof( CMDLIST ) ); + result->iscmd = 0; + result->next = l; + result->t = t; + return result; +} + +void cmdlist_free( CMDLIST * l ) +{ + while ( l ) + { + CMDLIST * tmp = l->next; + BJAM_FREE( l ); + l = tmp; + } +} + +/* * cmd_new() - return a new CMD. */ @@ -37,6 +68,10 @@ cmd->shell = shell; cmd->next = 0; cmd->noop = 0; + cmd->asynccnt = 1; + cmd->status = 0; + cmd->lock = NULL; + cmd->unlock = NULL; lol_init( &cmd->args ); lol_add( &cmd->args, targets ); @@ -62,9 +97,11 @@ void cmd_free( CMD * cmd ) { + cmdlist_free( cmd->next ); lol_free( &cmd->args ); list_free( cmd->shell ); string_free( cmd->buf ); + freetargets( cmd->unlock ); BJAM_FREE( (void *)cmd ); } Index: engine/command.h =================================================================== --- engine/command.h (revision 86425) +++ engine/command.h (working copy) @@ -44,16 +44,42 @@ #include "rules.h" #include "strings.h" +typedef struct _cmd CMD; -typedef struct _cmd CMD; +/* + * A list whose elements are either TARGETS or CMDS. + * CMDLIST is used only by CMD. A TARGET means that + * the CMD is the last updating action required to + * build the target. A CMD is the next CMD required + * to build the same target. (Note that a single action + * can update more than one target, so the CMDs form + * a DAG, not a straight linear list.) + */ +typedef struct _cmdlist { + struct _cmdlist * next; + union { + CMD * cmd; + TARGET * t; + }; + char iscmd; +} CMDLIST; + +CMDLIST * cmdlist_append_cmd( CMDLIST *, CMD * ); +CMDLIST * cmdlist_append_target( CMDLIST *, TARGET * ); +void cmd_list_free( CMDLIST * ); + struct _cmd { - CMD * next; + CMDLIST * next; RULE * rule; /* rule->actions contains shell script */ LIST * shell; /* $(JAMSHELL) value */ LOL args; /* LISTs for $(<), $(>) */ string buf[ 1 ]; /* actual commands */ int noop; /* no-op commands should be faked instead of executed */ + int asynccnt; /* number of outstanding dependencies */ + TARGETS * lock; /* semaphores that are required by this cmd. */ + TARGETS * unlock; /* semaphores that are released when this cmd finishes. */ + char status; /* the command status */ }; CMD * cmd_new Index: engine/rules.h =================================================================== --- engine/rules.h (revision 86425) +++ engine/rules.h (working copy) @@ -92,8 +92,13 @@ #define A_INIT 0 #define A_RUNNING_NOEXEC 1 #define A_RUNNING 2 - char status; /* see TARGET status */ int refs; + + /* WARNING: These variables are used to pass state required by make1cmds and + * are not valid anywhere else. + */ + void * first_cmd; /* Pointer to the first CMD created by this action */ + void * last_cmd; /* Pointer to the last CMD created by this action */ }; /* SETTINGS - variables to set when executing a TARGET's ACTIONS. */ Index: engine/make.c =================================================================== --- engine/make.c (revision 86425) +++ engine/make.c (working copy) @@ -161,6 +161,8 @@ * make0() to be updated. */ +static void force_rebuilds( TARGET * t ); + static void update_dependants( TARGET * t ) { TARGETS * q; @@ -190,6 +192,8 @@ update_dependants( p ); } } + /* Make sure that rebuilds can be chained. */ + force_rebuilds( t ); } @@ -676,12 +680,36 @@ else fate = t->fate; - /* Step 4g: If this target needs to be built, force rebuild everything in + /* + * Step 4g: If this target needs to be built, make0 all targets + * that are updated by the same actions used to update this target. + * These have already been marked as REBUILDS, and make1 has + * special handling for them. We just need to make sure that + * they get make0ed. + */ + if ( ( fate >= T_FATE_BUILD ) && ( fate < T_FATE_BROKEN ) ) + { + ACTIONS * a; + TARGETS * c; + for ( a = t->actions; a; a = a->next ) + { + for ( c = a->action->targets; c; c = c->next ) + { + if ( c->target->fate == T_FATE_INIT ) + { + make0( c->target, ptime, depth + 1, counts, anyhow, rescanning ); + } + } + } + } + + /* Step 4h: If this target needs to be built, force rebuild everything in * its rebuilds list. */ if ( ( fate >= T_FATE_BUILD ) && ( fate < T_FATE_BROKEN ) ) force_rebuilds( t ); + /* * Step 5: Sort dependencies by their update time. */ Index: engine/compile.c =================================================================== --- engine/compile.c (revision 86425) +++ engine/compile.c (working copy) @@ -116,6 +116,7 @@ action->sources = targetlist( (TARGETS *)0, lol_get( frame->args, 1 ) ); action->refs = 1; + /* FIXME: outdated comment */ /* If we have a group of targets all being built using the same action * then we must not allow any of them to be used as sources unless they * are all up to date and their action does not need to be run or their @@ -165,8 +166,8 @@ TARGET * const t0 = action->targets->target; for ( t = action->targets->next; t; t = t->next ) { - target_include( t->target, t0 ); - target_include( t0, t->target ); + t->target->rebuilds = targetentry( t->target->rebuilds, t0 ); + t0->rebuilds = targetentry( t0->rebuilds, t->target ); } } Index: engine/make1.c =================================================================== --- engine/make1.c (revision 86425) +++ engine/make1.c (working copy) @@ -63,7 +63,14 @@ static void make1bind ( TARGET * ); static TARGET * make1findcycle ( TARGET * ); static void make1breakcycle( TARGET *, TARGET * cycle_root ); +static void push_cmds( CMDLIST * cmds, int status ); +static int cmd_sem_lock( TARGET * t ); +static void cmd_sem_unlock( TARGET * t ); +static int targets_contains( TARGETS * l, TARGET * t ); +static int targets_subset( TARGETS * l1, TARGETS * l2 ); +static int targets_setequal( TARGETS * l1, TARGETS * l2 ); + /* Ugly static - it is too hard to carry it through the callbacks. */ static struct @@ -372,33 +379,16 @@ TARGET * failed = 0; char const * failed_name = "dependencies"; + pop_state( &state_stack ); + /* If any dependencies are still outstanding, wait until they signal their * completion by pushing this same state for their parent targets. */ if ( --t->asynccnt ) { - pop_state( &state_stack ); return; } - /* Try to aquire a semaphore. If it is locked, wait until the target that - * locked it is built and signals completition. - */ -#ifdef OPT_SEMAPHORE - if ( t->semaphore && t->semaphore->asynccnt ) - { - /* Append 't' to the list of targets waiting on semaphore. */ - t->semaphore->parents = targetentry( t->semaphore->parents, t ); - t->asynccnt++; - - if ( DEBUG_EXECCMD ) - printf( "SEM: %s is busy, delaying launch of %s\n", - object_str( t->semaphore->name ), object_str( t->name ) ); - pop_state( &state_stack ); - return; - } -#endif - /* Now ready to build target 't', if dependencies built OK. */ /* Collect status from dependencies. If -n was passed then act as though all @@ -492,28 +482,14 @@ abort(); } -#ifdef OPT_SEMAPHORE - /* If there is a semaphore, indicate that it is in use. */ - if ( t->semaphore ) - { - ++t->semaphore->asynccnt; - if ( DEBUG_EXECCMD ) - printf( "SEM: %s now used by %s\n", object_str( t->semaphore->name - ), object_str( t->name ) ); - } -#endif - /* Proceed to MAKE1C to begin executing the chain of commands prepared for * building the target. If we are not going to build the target (e.g. due to * dependency failures or no commands needing to be run) the chain will be * empty and MAKE1C processing will directly signal the target's completion. */ - /* Implementation note: - * Morfing the current state on the stack instead of popping it and - * pushing a new one is a slight optimization with no side-effects since we - * pushed no other states while processing this one. - */ - pState->curstate = T_STATE_MAKE1C; + + if ( t->cmds == NULL || --( ( CMD * )t->cmds )->asynccnt == 0 ) + push_state( &state_stack, t, NULL, T_STATE_MAKE1C ); } @@ -534,7 +510,7 @@ TARGET * const t = pState->t; CMD * const cmd = (CMD *)t->cmds; - if ( cmd && t->status == EXEC_CMD_OK ) + if ( cmd ) { /* Pop state first in case something below (e.g. exec_cmd(), exec_wait() * or make1c_closure()) pushes a new state. Note that we must not access @@ -543,6 +519,21 @@ */ pop_state( &state_stack ); + if ( cmd->status != EXEC_CMD_OK ) + { + t->cmds = NULL; + push_cmds( cmd->next, cmd->status ); + cmd_free( cmd ); + return; + } + +#ifdef OPT_SEMAPHORE + if ( ! cmd_sem_lock( t ) ) + { + return; + } +#endif + /* Increment the jobs running counter. */ ++cmdsrunning; @@ -575,14 +566,6 @@ { ACTIONS * actions; - /* Collect status from actions, and distribute it as well. */ - for ( actions = t->actions; actions; actions = actions->next ) - if ( actions->action->status > t->status ) - t->status = actions->action->status; - for ( actions = t->actions; actions; actions = actions->next ) - if ( t->status > actions->action->status ) - actions->action->status = t->status; - /* Tally success/failure for those we tried to update. */ if ( t->progress == T_MAKE_RUNNING ) switch ( t->status ) @@ -677,38 +660,6 @@ push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B ); } -#ifdef OPT_SEMAPHORE - /* If there is a semaphore, it is now free. */ - if ( t->semaphore ) - { - assert( t->semaphore->asynccnt == 1 ); - --t->semaphore->asynccnt; - - if ( DEBUG_EXECCMD ) - printf( "SEM: %s is now free\n", object_str( - t->semaphore->name ) ); - - /* If anything is waiting, notify the next target. There is no - * point in notifying all waiting targets, since they will be - * notified again. - */ - if ( t->semaphore->parents ) - { - TARGETS * first = t->semaphore->parents; - t->semaphore->parents = first->next; - if ( first->next ) - first->next->tail = first->tail; - - if ( DEBUG_EXECCMD ) - printf( "SEM: placing %s on stack\n", object_str( - first->target->name ) ); - push_state( &temp_stack, first->target, NULL, T_STATE_MAKE1B - ); - BJAM_FREE( first ); - } - } -#endif - /* Must pop state before pushing any more. */ pop_state( &state_stack ); @@ -945,15 +896,55 @@ } } +#ifdef OPT_SEMAPHORE + /* Release any semaphores used by this action. */ + cmd_sem_unlock( t ); +#endif + /* Free this command and push the MAKE1C state to execute the next one * scheduled for building this same target. */ - t->cmds = (char *)cmd_next( cmd ); + t->cmds = NULL; + push_cmds( cmd->next, t->status ); cmd_free( cmd ); - push_state( &state_stack, t, NULL, T_STATE_MAKE1C ); } +/* push the next MAKE1C state after a command is run. */ +static void push_cmds( CMDLIST * cmds, int status ) +{ + CMDLIST * cmd_iter; + for( cmd_iter = cmds; cmd_iter; cmd_iter = cmd_iter->next ) + { + if ( cmd_iter->iscmd ) + { + CMD * next_cmd = cmd_iter->cmd; + /* Propagate the command status. */ + if ( next_cmd->status < status ) + next_cmd->status = status; + if ( --next_cmd->asynccnt == 0 ) + { + /* Select the first target associated with the action. + * This is safe because sibling CMDs cannot have targets + * in common. + */ + TARGET * first_target = bindtarget( list_front( lol_get( &next_cmd->args, 0 ) ) ); + first_target->cmds = (char *)next_cmd; + push_state( &state_stack, first_target, NULL, T_STATE_MAKE1C ); + } + } + else + { + /* This is a target that we're finished updating */ + TARGET * updated_target = cmd_iter->t; + if ( updated_target->status < status ) + updated_target->status = status; + updated_target->cmds = NULL; + push_state( &state_stack, updated_target, NULL, T_STATE_MAKE1C ); + } + } +} + /* * swap_settings() - replace the settings from the current module and target * with those from the new module and target @@ -995,13 +986,14 @@ static CMD * make1cmds( TARGET * t ) { CMD * cmds = 0; - CMD * * cmds_next = &cmds; + CMD * last_cmd; LIST * shell = L0; module_t * settings_module = 0; TARGET * settings_target = 0; ACTIONS * a0; int const running_flag = globs.noexec ? A_RUNNING_NOEXEC : A_RUNNING; + /* FIXME: outdated comment */ /* Step through actions. Actions may be shared with other targets or grouped * using RULE_TOGETHER, so actions already seen are skipped. */ @@ -1017,9 +1009,33 @@ /* Only do rules with commands to execute. If this action has already * been executed, use saved status. */ - if ( !actions || a0->action->running >= running_flag ) + if ( !actions ) continue; + if ( a0->action->running >= running_flag ) + { + CMD * first; + /* FIXME: Forbid multiple targets with RULE_TOGETHER, or + * figure out exactly what it should mean. + */ + if ( actions->flags & RULE_TOGETHER ) + continue; + /* This action has already been processed for another target. + * Just set up the dependency graph correctly and move on. + */ + first = a0->action->first_cmd; + if( cmds ) + { + last_cmd->next = cmdlist_append_cmd( last_cmd->next, first ); + } + else + { + cmds = first; + } + last_cmd = a0->action->last_cmd; + continue; + } + a0->action->running = running_flag; /* Make LISTS of targets and sources. If `execute together` has been @@ -1076,8 +1092,11 @@ int const length = list_length( ns ); int start = 0; int chunk = length; + int cmd_count = 0; LIST * cmd_targets = L0; LIST * cmd_shell = L0; + TARGETS * semaphores = NULL; + TARGETS * targets_iter; do { CMD * cmd; @@ -1138,9 +1157,21 @@ if ( accept_command ) { /* Chain it up. */ - *cmds_next = cmd; - cmds_next = &cmd->next; + if ( cmds ) + { + last_cmd->next = cmdlist_append_cmd( last_cmd->next, cmd ); + last_cmd = cmd; + } + else + { + cmds = last_cmd = cmd; + } + if ( cmd_count++ == 0 ) + { + a0->action->first_cmd = cmd; + } + /* Mark lists we need recreated for the next command since * they got consumed by the cmd object. */ @@ -1160,6 +1191,35 @@ start += chunk; } while ( start < length ); + + /* Record the end of the actions cmds */ + a0->action->last_cmd = last_cmd; + /* We need to wait until all the targets agree that + * it's okay to run this action. + */ + ( ( CMD * )a0->action->first_cmd )->asynccnt = list_length( nt ); + + /* Add all targets produced by the action to the update list. */ + for ( targets_iter = a0->action->targets; targets_iter; targets_iter = targets_iter->next ) + { + push_state( &state_stack, targets_iter->target, NULL, T_STATE_MAKE1A ); + } + +#if OPT_SEMAPHORE + /* Collect semaphores */ + for ( targets_iter = a0->action->targets; targets_iter; targets_iter = targets_iter->next ) + { + TARGET * sem = targets_iter->target->semaphore; + if ( sem ) + { + TARGETS * semiter; + if ( ! targets_contains( semaphores, sem ) ) + semaphores = targetentry( semaphores, sem ); + } + } + ( ( CMD * )a0->action->first_cmd )->lock = semaphores; + ( ( CMD * )a0->action->last_cmd )->unlock = semaphores; +#endif } /* These were always copied when used. */ @@ -1171,6 +1231,11 @@ freesettings( boundvars ); } + if ( cmds ) + { + last_cmd->next = cmdlist_append_target( last_cmd->next, t ); + } + swap_settings( &settings_module, &settings_target, 0, 0 ); return cmds; } @@ -1281,3 +1346,109 @@ t->binding = timestamp_empty( &t->time ) ? T_BIND_MISSING : T_BIND_EXISTS; popsettings( root_module(), t->settings ); } + + +static int targets_contains( TARGETS * l, TARGET * t ) +{ + for ( ; l; l = l->next ) + { + if ( t == l->target ) + { + return 1; + } + } + return 0; +} + +static int targets_subset( TARGETS * l1, TARGETS * l2 ) +{ + TARGETS * iter; + for ( iter = l1; iter; iter = iter->next ) + { + if ( !targets_contains( l2, iter->target ) ) + { + return 0; + } + } + return 1; +} + +static int targets_setequal( TARGETS * l1, TARGETS * l2 ) +{ + return targets_subset( l1, l2 ) && targets_subset( l2, l1 ); +} + + +#ifdef OPT_SEMAPHORE + +static int cmd_sem_lock( TARGET * t ) +{ + CMD * cmd = (CMD *)t->cmds; + TARGETS * iter; + /* Check whether all the semaphores required for updating + * this target are free. + */ + for ( iter = cmd->lock; iter; iter = iter->next ) + { + if ( iter->target->asynccnt > 0 ) + { + if ( DEBUG_EXECCMD ) + printf( "SEM: %s is busy, delaying launch of %s\n", + object_str( iter->target->name ), object_str( t->name ) ); + iter->target->parents = targetentry( iter->target->parents, t ); + return 0; + } + } + /* Lock the semaphores. */ + for ( iter = cmd->lock; iter; iter = iter->next ) + { + ++iter->target->asynccnt; + if ( DEBUG_EXECCMD ) + printf( "SEM: %s now used by %s\n", object_str( iter->target->name + ), object_str( t->name ) ); + } + /* A cmd only needs to be locked around its execution. + * clearing cmd->lock here makes it safe to call cmd_sem_lock + * twice. + */ + cmd->lock = NULL; + return 1; +} + +static void cmd_sem_unlock( TARGET * t ) +{ + CMD * cmd = ( CMD * )t->cmds; + TARGETS * iter; + /* Release the semaphores. */ + for ( iter = cmd->unlock; iter; iter = iter->next ) + { + if ( DEBUG_EXECCMD ) + printf( "SEM: %s is now free\n", object_str( + iter->target->name ) ); + --iter->target->asynccnt; + assert( iter->target->asynccnt <= 0 ); + } + for ( iter = cmd->unlock; iter; iter = iter->next ) + { + /* Find a waiting target that's ready */ + while ( iter->target->parents ) + { + TARGETS * first = iter->target->parents; + TARGET * t1 = first->target; + + /* Pop the first waiting CMD */ + if ( first->next ) + first->next->tail = first->tail; + iter->target->parents = first->next; + BJAM_FREE( first ); + + if ( cmd_sem_lock( t1 ) ) + { + push_state( &state_stack, t1, NULL, T_STATE_MAKE1C ); + break; + } + } + } +} + +#endif