[Boost-users] Boost.ProgramOptions issues porting from 1.56 to 1.64

Subject: [Boost-users] Boost.ProgramOptions issues porting from 1.56 to 1.64
From: Dominique Devienne (ddevienne_at_[hidden])
Date: 2017-07-24 14:33:23

I have an SVN-like "multi-command" single executable, i.e.
exe [global-options] command [command-options]

It uses two levels of option parsing. First level:

typedef std::vector<std::string> UnparsedArgs;
inline boost::program_options::typed_value<UnparsedArgs>* anyArgs() {
    return boost::program_options::value<UnparsedArgs>();
}

    options_description global_all_desc("Global Options", columns);
    global_all_desc.add_options()
("command",         txtArg(),   "Command to execute")
("cmdargs",         anyArgs(),  "Arguments for command")
...;

    // from http://stackoverflow.com/questions/15541498
    // see also https://gist.github.com/randomphrase/10801888
    po::positional_options_description pos;
    pos .add("command", 1)
        .add("cmdargs", -1);

        variables_map global_args;
        parsed_options parsed = po::command_line_parser(argc, argv)
            .options(global_all_desc)
            .positional(pos) // for implicit command and cmdargs above
            .allow_unregistered() // for command-specific arguments.
            .run();

        po::store(parsed, global_args);

    std::vector<std::string> cmd_args = po::collect_unrecognized(
        parsed.options, po::include_positional
    );

Second level

        parsed_options reparsed = po::command_line_parser(cmd_args)
            .options(cmd_all_desc)
            .run();
        po::store(reparsed, args_);

        // JIRA FOO-426
        for (const auto& option : reparsed.options) {
            if (option.position_key >= 0) {
                std::ostringstream oss;
                for (const auto& token : option.original_tokens) {
                    oss << token << ' ';
                }
                auto s = oss.str();
                s.resize(s.size() - 1);

                err() << "Error: Too many commands: " << command() << ' '
<< s << endl;
                return 1;
            }
        }

in this particular case, the option that fails is

 ( "new", txtArgOpt(), "Creates a new group" ),

with txtArgOpt() as below:

inline boost::program_options::typed_value<std::string>* txtArg() {
    return boost::program_options::value<std::string>();
}

inline boost::program_options::typed_value<std::string>* txtArgOpt(const
std::string& implicit_value = "") {
    return txtArg()->implicit_value(implicit_value);
}

in the case that fails, the first parsing works fine, and remains ["--new",
"groupA"] as cmd_args.

in 1.56, reparsed.options contains 1 element:

- [0] {string_key="new" position_key=-1 value={ size=1 } ...}
boost::program_options::basic_option<char>
+ string_key "new"
std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
position_key -1 int
- value { size=1 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[size] 1 __int64
[capacity] 1 __int64
+ [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [Raw View] 0x0000000009fae6c0 {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > > *
- original_tokens { size=2 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[size] 2 __int64
[capacity] 2 __int64
+ [0] "--new" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [1] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [Raw View] 0x0000000009fae6e0 {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > > *
unregistered false bool
case_insensitive false bool

but in 1.64, it contains 2 elements:

- [0] {string_key="new" position_key=-1 value={ size=0 } ...}
boost::program_options::basic_option<char>
+ string_key "new"
std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
position_key -1 int
- value { size=0 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[capacity] 0 __int64
+ [allocator] allocator
std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> >
>,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >,1>
+ [Raw View] {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
- original_tokens { size=1 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[capacity] 1 __int64
+ [allocator] allocator
std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> >
>,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >,1>
+ [0] "--new" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [Raw View] {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
unregistered false bool
case_insensitive false bool
- [1] {string_key="" position_key=0 value={ size=1 } ...}
boost::program_options::basic_option<char>
+ string_key ""
std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
position_key 0 int
- value { size=1 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[capacity] 1 __int64
+ [allocator] allocator
std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> >
>,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >,1>
+ [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [Raw View] {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
- original_tokens { size=1 }
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
[capacity] 1 __int64
+ [allocator] allocator
std::_Compressed_pair<std::_Wrap_alloc<std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> >
>,std::_Vector_val<std::_Simple_types<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >,1>
+ [0] "GroupA" std::basic_string<char,std::char_traits<char>,std::allocator<char>
>
+ [Raw View] {...}
std::vector<std::basic_string<char,std::char_traits<char>,std::allocator<char>
>,std::allocator<std::basic_string<char,std::char_traits<char>,std::allocator<char>
> > >
unregistered false bool
case_insensitive false bool

This is copy/pasted from VS, sorry, a little hard to read, but basically
1.56 has

#0
string_key = "new"
position_key = -1
value = ["groupA"]
original_tokens = ["--new", "groupA"]
unregistered = false
case_insensitive = false

while 1.64 has 2 items:

#0
string_key = "new"
position_key = -1
value = []
original_tokens = ["--new"]
unregistered = false
case_insensitive = false

#1
string_key = ""
position_key = 0
value = ["groupA"]
original_tokens = ["groupA"]
unregistered = false
case_insensitive = false

as-if the arg to --new was now a positional parameter, instead of an
optional arg to --new.

And this triggers the work-around code that errors out I had put in place
to catch cases when the old 1.56 code wouldn't complain about extra
arguments (e.g. "exe ... cmd1 cmd1_args cmd2" would be silently accepted,
despite the extra cmd2, or "exe ... cmd --arg1 read-only" was accepted,
when "exe ... cmd --arg1 --read-only" was expected), and this even though
the second parsing doesn't have .allow_unregistered().

I'm trying to understand what's going on;
obviously there's some kind of semantic change with optional argument to
switches.

Could anyone shed some light on this please? I'm not sure how to proceed
now. TIA, -DD