diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index c17b88b0b..027156d83 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -1,208 +1,295 @@ // Copyright (c) 2011-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include +#include #include #include #include +inline bool operator==(const util::SettingsValue &a, + const util::SettingsValue &b) { + return a.write() == b.write(); +} + +inline std::ostream &operator<<(std::ostream &os, + const util::SettingsValue &value) { + os << value.write(); + return os; +} + +inline std::ostream & +operator<<(std::ostream &os, + const std::pair &kv) { + util::SettingsValue out(util::SettingsValue::VOBJ); + out.__pushKV(kv.first, kv.second); + os << out.write(); + return os; +} + +inline void WriteText(const fs::path &path, const std::string &text) { + fsbridge::ofstream file; + file.open(path); + file << text; +} + BOOST_FIXTURE_TEST_SUITE(settings_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(ReadWrite) { + fs::path path = GetDataDir() / "settings.json"; + + WriteText(path, R"({ + "string": "string", + "num": 5, + "bool": true, + "null": null + })"); + + std::map expected{ + {"string", "string"}, + {"num", 5}, + {"bool", true}, + {"null", {}}, + }; + + // Check file read. + std::map values; + std::vector errors; + BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), + expected.begin(), expected.end()); + BOOST_CHECK(errors.empty()); + + // Check no errors if file doesn't exist. + fs::remove(path); + BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(values.empty()); + BOOST_CHECK(errors.empty()); + + // Check duplicate keys not allowed + WriteText(path, R"({ + "dupe": "string", + "dupe": "dupe" + })"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector dup_keys = {strprintf( + "Found duplicate key dupe in settings file %s", path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), + dup_keys.begin(), dup_keys.end()); + + // Check non-kv json files not allowed + WriteText(path, R"("non-kv")"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector non_kv = { + strprintf("Found non-object value \"non-kv\" in settings file %s", + path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), + non_kv.end()); + + // Check invalid json not allowed + WriteText(path, R"(invalid json)"); + BOOST_CHECK(!util::ReadSettings(path, values, errors)); + std::vector fail_parse = { + strprintf("Unable to parse settings file %s", path.string())}; + BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), + fail_parse.begin(), fail_parse.end()); +} + //! Check settings struct contents against expected json strings. static void CheckValues(const util::Settings &settings, const std::string &single_val, const std::string &list_val) { util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false); util::SettingsValue list_value(util::SettingsValue::VARR); for (const auto &item : GetSettingsList(settings, "section", "name", false)) { list_value.push_back(item); } BOOST_CHECK_EQUAL(single_value.write().c_str(), single_val); BOOST_CHECK_EQUAL(list_value.write().c_str(), list_val); }; // Simple settings merge test case. BOOST_AUTO_TEST_CASE(Simple) { util::Settings settings; settings.command_line_options["name"].push_back("val1"); settings.command_line_options["name"].push_back("val2"); settings.ro_config["section"]["name"].push_back(2); // The last given arg takes precedence when specified via commandline. CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); util::Settings settings2; settings2.ro_config["section"]["name"].push_back("val2"); settings2.ro_config["section"]["name"].push_back("val3"); // The first given arg takes precedence when specified via config file. CheckValues(settings2, R"("val2")", R"(["val2","val3"])"); } // Confirm that a high priority setting overrides a lower priority setting even // if the high priority setting is null. This behavior is useful for a high // priority setting source to be able to effectively reset any setting back to // its default value. BOOST_AUTO_TEST_CASE(NullOverride) { util::Settings settings; settings.command_line_options["name"].push_back("value"); BOOST_CHECK_EQUAL( R"("value")", GetSetting(settings, "section", "name", false, false).write().c_str()); settings.forced_settings["name"] = {}; BOOST_CHECK_EQUAL( R"(null)", GetSetting(settings, "section", "name", false, false).write().c_str()); } // Test different ways settings can be merged, and verify results. This test can // be used to confirm that updates to settings code don't change behavior // unintentionally. struct MergeTestingSetup : public BasicTestingSetup { //! Max number of actions to sequence together. Can decrease this when //! debugging to make test results easier to understand. static constexpr int MAX_ACTIONS = 3; enum Action { END, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; using ActionList = Action[MAX_ACTIONS]; //! Enumerate all possible test configurations. template void ForEachMergeSetup(Fn &&fn) { ActionList arg_actions = {}; // command_line_options do not have sections. Only iterate over SET and // NEGATE ForEachNoDup(arg_actions, SET, NEGATE, [&] { ActionList conf_actions = {}; ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&] { for (bool force_set : {false, true}) { for (bool ignore_default_section_config : {false, true}) { fn(arg_actions, conf_actions, force_set, ignore_default_section_config); } } }); }); } }; // Regression test covering different ways config settings can be merged. The // test parses and merges settings, representing the results as strings that get // compared against an expected hash. To debug, the result strings can be dumped // to a file (see comments below). BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) { CHash256 out_sha; FILE *out_file = nullptr; if (const char *out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { out_file = fsbridge::fopen(out_path, "w"); if (!out_file) { throw std::system_error(errno, std::generic_category(), "fopen failed"); } } const std::string &network = CBaseChainParams::MAIN; ForEachMergeSetup([&](const ActionList &arg_actions, const ActionList &conf_actions, bool force_set, bool ignore_default_section_config) { std::string desc; int value_suffix = 0; util::Settings settings; const std::string &name = ignore_default_section_config ? "wallet" : "server"; auto push_values = [&](Action action, const char *value_prefix, const std::string &name_prefix, std::vector &dest) { if (action == SET || action == SECTION_SET) { for (int i = 0; i < 2; ++i) { dest.push_back(value_prefix + std::to_string(++value_suffix)); desc += " " + name_prefix + name + "=" + dest.back().get_str(); } } else if (action == NEGATE || action == SECTION_NEGATE) { dest.push_back(false); desc += " " + name_prefix + "no" + name; } }; if (force_set) { settings.forced_settings[name] = "forced"; desc += " " + name + "=forced"; } for (Action arg_action : arg_actions) { push_values(arg_action, "a", "-", settings.command_line_options[name]); } for (Action conf_action : conf_actions) { bool use_section = conf_action == SECTION_SET || conf_action == SECTION_NEGATE; push_values(conf_action, "c", use_section ? network + "." : "", settings.ro_config[use_section ? network : ""][name]); } desc += " || "; desc += GetSetting(settings, network, name, ignore_default_section_config, /* get_chain_name= */ false) .write(); desc += " |"; for (const auto &s : GetSettingsList(settings, network, name, ignore_default_section_config)) { desc += " "; desc += s.write(); } desc += " |"; if (OnlyHasDefaultSectionSetting(settings, network, name)) { desc += " ignored"; } desc += "\n"; out_sha.Write((const uint8_t *)desc.data(), desc.size()); if (out_file) { BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); } }); if (out_file) { if (fclose(out_file)) { throw std::system_error(errno, std::generic_category(), "fclose failed"); } out_file = nullptr; } uint8_t out_sha_bytes[CSHA256::OUTPUT_SIZE]; out_sha.Finalize(out_sha_bytes); std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); // If check below fails, should manually dump the results with: // // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin // --run_test=settings_tests/Merge // // And verify diff against previous results to make sure the changes are // expected. // // Results file is formatted like: // // || GetSetting() | GetSettingsList() | // OnlyHasDefaultSectionSetting() BOOST_CHECK_EQUAL( out_sha_hex, "79db02d74e3e193196541b67c068b40ebd0c124a24b3ecbe9cbf7e85b1c4ba7a"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/settings.cpp b/src/util/settings.cpp index e9311743a..f71b75ab9 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -1,213 +1,283 @@ // Copyright (c) 2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include namespace util { namespace { enum class Source { FORCED, COMMAND_LINE, CONFIG_FILE_NETWORK_SECTION, CONFIG_FILE_DEFAULT_SECTION }; //! Merge settings from multiple sources in precedence order: //! Forced config > command line > config file network-specific section > //! config file default section //! //! This function is provided with a callback function fn that contains //! specific logic for how to merge the sources. template static void MergeSettings(const Settings &settings, const std::string §ion, const std::string &name, Fn &&fn) { // Merge in the forced settings if (auto *value = FindKey(settings.forced_settings, name)) { fn(SettingsSpan(*value), Source::FORCED); } // Merge in the command-line options if (auto *values = FindKey(settings.command_line_options, name)) { fn(SettingsSpan(*values), Source::COMMAND_LINE); } // Merge in the network-specific section of the config file if (!section.empty()) { if (auto *map = FindKey(settings.ro_config, section)) { if (auto *values = FindKey(*map, name)) { fn(SettingsSpan(*values), Source::CONFIG_FILE_NETWORK_SECTION); } } } // Merge in the default section of the config file if (auto *map = FindKey(settings.ro_config, "")) { if (auto *values = FindKey(*map, name)) { fn(SettingsSpan(*values), Source::CONFIG_FILE_DEFAULT_SECTION); } } } } // namespace +bool ReadSettings(const fs::path &path, + std::map &values, + std::vector &errors) { + values.clear(); + errors.clear(); + + fsbridge::ifstream file; + file.open(path); + if (!file.is_open()) { + // Ok for file not to exist. + return true; + } + + SettingsValue in; + if (!in.read(std::string{std::istreambuf_iterator(file), + std::istreambuf_iterator()})) { + errors.emplace_back( + strprintf("Unable to parse settings file %s", path.string())); + return false; + } + + if (file.fail()) { + errors.emplace_back( + strprintf("Failed reading settings file %s", path.string())); + return false; + } + // Done with file descriptor. Release while copying data. + file.close(); + + if (!in.isObject()) { + errors.emplace_back( + strprintf("Found non-object value %s in settings file %s", + in.write(), path.string())); + return false; + } + + const std::vector &in_keys = in.getKeys(); + const std::vector &in_values = in.getValues(); + for (size_t i = 0; i < in_keys.size(); ++i) { + auto inserted = values.emplace(in_keys[i], in_values[i]); + if (!inserted.second) { + errors.emplace_back( + strprintf("Found duplicate key %s in settings file %s", + in_keys[i], path.string())); + } + } + return errors.empty(); +} + +bool WriteSettings(const fs::path &path, + const std::map &values, + std::vector &errors) { + SettingsValue out(SettingsValue::VOBJ); + for (const auto &value : values) { + out.__pushKV(value.first, value.second); + } + fsbridge::ofstream file; + file.open(path); + if (file.fail()) { + errors.emplace_back( + strprintf("Error: Unable to open settings file %s for writing", + path.string())); + return false; + } + file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl; + file.close(); + return true; +} + SettingsValue GetSetting(const Settings &settings, const std::string §ion, const std::string &name, bool ignore_default_section_config, bool get_chain_name) { SettingsValue result; // Done merging any more settings sources. bool done = false; MergeSettings( settings, section, name, [&](SettingsSpan span, Source source) { // Weird behavior preserved for backwards compatibility: Apply // negated setting even if non-negated setting would be ignored. A // negated value in the default section is applied to network // specific options, even though normal non-negated values there // would be ignored. const bool never_ignore_negated_setting = span.last_negated(); // Weird behavior preserved for backwards compatibility: Take first // assigned value instead of last. In general, later settings take // precedence over early settings, but for backwards compatibility // in the config file the precedence is reversed for all settings // except chain name settings. const bool reverse_precedence = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !get_chain_name; // Weird behavior preserved for backwards compatibility: Negated // -regtest and -testnet arguments which you would expect to // override values set in the configuration file are currently // accepted but silently ignored. It would be better to apply these // just like other negated values, or at least warn they are // ignored. const bool skip_negated_command_line = get_chain_name; if (done) { return; } // Ignore settings in default config section if requested. if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION && !never_ignore_negated_setting) { return; } // Skip negated command line settings. if (skip_negated_command_line && span.last_negated()) { return; } if (!span.empty()) { result = reverse_precedence ? span.begin()[0] : span.end()[-1]; done = true; } else if (span.last_negated()) { result = false; done = true; } }); return result; } std::vector GetSettingsList(const Settings &settings, const std::string §ion, const std::string &name, bool ignore_default_section_config) { std::vector result; // Done merging any more settings sources. bool done = false; bool prev_negated_empty = false; MergeSettings( settings, section, name, [&](SettingsSpan span, Source source) { // Weird behavior preserved for backwards compatibility: Apply // config file settings even if negated on command line. Negating a // setting on command line will ignore earlier settings on the // command line and ignore settings in the config file, unless the // negated command line value is followed by non-negated value, in // which case config file settings will be brought back from the // dead (but earlier command line settings will still be ignored). const bool add_zombie_config_values = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && !prev_negated_empty; // Ignore settings in default config section if requested. if (ignore_default_section_config && source == Source::CONFIG_FILE_DEFAULT_SECTION) { return; } // Add new settings to the result if isn't already complete, or if // the values are zombies. if (!done || add_zombie_config_values) { for (const auto &value : span) { if (value.isArray()) { result.insert(result.end(), value.getValues().begin(), value.getValues().end()); } else { result.push_back(value); } } } // If a setting was negated, or if a setting was forced, set done to // true to ignore any later lower priority settings. done |= span.negated() > 0 || source == Source::FORCED; // Update the negated and empty state used for the zombie values // check. prev_negated_empty |= span.last_negated() && result.empty(); }); return result; } bool OnlyHasDefaultSectionSetting(const Settings &settings, const std::string §ion, const std::string &name) { bool has_default_section_setting = false; bool has_other_setting = false; MergeSettings( settings, section, name, [&](SettingsSpan span, Source source) { if (span.empty()) { return; } else if (source == Source::CONFIG_FILE_DEFAULT_SECTION) { has_default_section_setting = true; } else { has_other_setting = true; } }); // If a value is set in the default section and not explicitly overwritten // by the user on the command line or in a different section, then we want // to enable warnings about the value being ignored. return has_default_section_setting && !has_other_setting; } SettingsSpan::SettingsSpan(const std::vector &vec) noexcept : SettingsSpan(vec.data(), vec.size()) {} const SettingsValue *SettingsSpan::begin() const { return data + negated(); } const SettingsValue *SettingsSpan::end() const { return data + size; } bool SettingsSpan::empty() const { return size == 0 || last_negated(); } bool SettingsSpan::last_negated() const { return size > 0 && data[size - 1].isFalse(); } size_t SettingsSpan::negated() const { for (size_t i = size; i > 0; --i) { if (data[i - 1].isFalse()) { // Return number of negated values (position of last false value) return i; } } return 0; } } // namespace util diff --git a/src/util/settings.h b/src/util/settings.h index 52e32f62a..6c60125d4 100644 --- a/src/util/settings.h +++ b/src/util/settings.h @@ -1,98 +1,110 @@ // Copyright (c) 2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UTIL_SETTINGS_H #define BITCOIN_UTIL_SETTINGS_H +#include + #include #include #include class UniValue; namespace util { //! Settings value type (string/integer/boolean/null variant). //! //! @note UniValue is used here for convenience and because it can be easily //! serialized in a readable format. But any other variant type that can //! be assigned strings, int64_t, and bool values and has get_str(), //! get_int64(), get_bool(), isNum(), isBool(), isFalse(), isTrue() and //! isNull() methods can be substituted if there's a need to move away //! from UniValue. (An implementation with boost::variant was posted at //! https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812) using SettingsValue = UniValue; //! Stored bitcoin settings. This struct combines settings from the command line //! and a read-only configuration file. struct Settings { //! Map of setting name to forced setting value. std::map forced_settings; //! Map of setting name to list of command line values. std::map> command_line_options; //! Map of config section name and setting name to list of config file //! values. std::map>> ro_config; }; +//! Read settings file. +bool ReadSettings(const fs::path &path, + std::map &values, + std::vector &errors); + +//! Write settings file. +bool WriteSettings(const fs::path &path, + const std::map &values, + std::vector &errors); + //! Get settings value from combined sources: forced settings, command line //! arguments and the read-only config file. //! //! @param ignore_default_section_config - ignore values in the default section //! of the config file (part before any //! [section] keywords) //! @param get_chain_name - enable special backwards compatible behavior //! for GetChainName SettingsValue GetSetting(const Settings &settings, const std::string §ion, const std::string &name, bool ignore_default_section_config, bool get_chain_name); //! Get combined setting value similar to GetSetting(), except if setting was //! specified multiple times, return a list of all the values specified. std::vector GetSettingsList(const Settings &settings, const std::string §ion, const std::string &name, bool ignore_default_section_config); //! Return true if a setting is set in the default config file section, and not //! overridden by a higher priority command-line or network section value. //! //! This is used to provide user warnings about values that might be getting //! ignored unintentionally. bool OnlyHasDefaultSectionSetting(const Settings &settings, const std::string §ion, const std::string &name); //! Accessor for list of settings that skips negated values when iterated over. //! The last boolean `false` value in the list and all earlier values are //! considered negated. struct SettingsSpan { explicit SettingsSpan() = default; explicit SettingsSpan(const SettingsValue &value) noexcept : SettingsSpan(&value, 1) {} explicit SettingsSpan(const SettingsValue *dataIn, size_t sizeIn) noexcept : data(dataIn), size(sizeIn) {} explicit SettingsSpan(const std::vector &vec) noexcept; const SettingsValue *begin() const; // auto FindKey(Map &&map, Key &&key) -> decltype(&map.at(key)) { auto it = map.find(key); return it == map.end() ? nullptr : &it->second; } } // namespace util #endif // BITCOIN_UTIL_SETTINGS_H