diff --git a/.arclint b/.arclint index ea7e00ee8..bfb2b6087 100644 --- a/.arclint +++ b/.arclint @@ -1,308 +1,308 @@ { "linters": { "generated": { "type": "generated" }, "clang-format": { "type": "clang-format", "version": ">=12.0", "bin": ["clang-format-12", "clang-format"], "include": "(^(src|chronik)/.*\\.(h|c|cpp|mm)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "autopep8": { "type": "autopep8", "version": ">=1.3.4", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)" ], "flags": [ "--aggressive", "--ignore=W503,W504", "--max-line-length=88" ] }, "flake8": { "type": "flake8", "version": ">=3.0", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)" ], "flags": [ "--ignore=E303,E305,E501,E704,W503,W504" ] }, "lint-format-strings": { "type": "lint-format-strings", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)", "(^src/test/fuzz/strprintf.cpp$)" ] }, "check-doc": { "type": "check-doc", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)" }, "lint-tests": { "type": "lint-tests", "include": "(^src/(seeder/|rpc/|wallet/)?test/.*\\.(cpp)$)" }, - "lint-python-format": { - "type": "lint-python-format", - "include": "(\\.py$)", - "exclude": [ - "(^test/lint/lint-python-format\\.py$)", - "(^contrib/gitian-builder/)", - "(^contrib/apple-sdk-tools/)" - ] - }, "phpcs": { "type": "phpcs", "include": "(\\.php$)", "exclude": [ "(^arcanist/__phutil_library_.+\\.php$)" ], "phpcs.standard": "arcanist/phpcs.xml" }, "lint-locale-dependence": { "type": "lint-locale-dependence", "include": "(^(src|chronik)/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/))", "(^src/bench/nanobench.h$)" ] }, "lint-cheader": { "type": "lint-cheader", "include": "(^(src|chronik)/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "spelling": { "type": "spelling", "exclude": [ "(^build-aux/m4/)", "(^depends/)", "(^doc/release-notes/)", "(^contrib/gitian-builder/)", "(^src/(qt/locale|secp256k1|univalue|leveldb)/)", "(^test/lint/dictionary/)", "(package-lock.json)" ], "spelling.dictionaries": [ "test/lint/dictionary/english.json" ] }, "lint-assert-with-side-effects": { "type": "lint-assert-with-side-effects", "include": "(^(src|chronik)/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "lint-include-quotes": { "type": "lint-include-quotes", "include": "(^(src|chronik)/.*\\.(h|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "lint-include-guard": { "type": "lint-include-guard", "include": "(^(src|chronik)/.*\\.h$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)", "(^src/tinyformat.h$)" ] }, "lint-include-source": { "type": "lint-include-source", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "lint-std-chrono": { "type": "lint-std-chrono", "include": "(^(src|chronik)/.*\\.(h|cpp)$)" }, "lint-stdint": { "type": "lint-stdint", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)", "(^src/compat/assumptions.h$)" ] }, "lint-source-filename": { "type": "lint-source-filename", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "lint-boost-dependencies": { "type": "lint-boost-dependencies", "include": "(^(src|chronik)/.*\\.(h|cpp)$)" }, "lint-python-encoding": { "type": "lint-python-encoding", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)" ] }, "lint-python-shebang": { "type": "lint-python-shebang", "include": "(\\.py$)", "exclude": [ "(__init__\\.py$)", "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)" ] }, "lint-bash-shebang": { "type": "lint-bash-shebang", "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)" ] }, "shellcheck": { "type": "shellcheck", "version": ">=0.7.0", "flags": [ "--external-sources", "--source-path=SCRIPTDIR" ], "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue)/)" ] }, "lint-shell-locale": { "type": "lint-shell-locale", "include": "(\\.sh$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue)/)", "(^cmake/utils/log-and-print-on-failure.sh)" ] }, "lint-cpp-void-parameters": { "type": "lint-cpp-void-parameters", "include": "(^(src|chronik)/.*\\.(h|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)", "(^src/compat/glibc_compat.cpp$)" ] }, "lint-logs": { "type": "lint-logs", "include": "(^(src|chronik)/.*\\.(h|cpp)$)" }, "lint-qt": { "type": "lint-qt", "include": "(^src/qt/.*\\.(h|cpp)$)", "exclude": [ "(^src/qt/(locale|forms|res)/)" ] }, "lint-doxygen": { "type": "lint-doxygen", "include": "(^(src|chronik)/.*\\.(h|c|cpp)$)", "exclude": [ "(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "lint-whitespace": { "type": "lint-whitespace", "include": "(\\.(ac|am|cmake|conf|in|include|json|m4|md|openrc|php|pl|rs|sh|txt|yml)$)", "exclude": [ "(^contrib/gitian-builder/)", "(^src/(secp256k1|univalue|leveldb)/)", "(^src/bench/nanobench.h$)" ] }, "yamllint": { "type": "yamllint", "include": "(\\.(yml|yaml)$)", "exclude": "(^src/(secp256k1|univalue|leveldb)/)" }, "lint-check-nonfatal": { "type": "lint-check-nonfatal", "include": [ "(^src/rpc/.*\\.(h|c|cpp)$)", "(^src/wallet/rpc*.*\\.(h|c|cpp)$)" ], "exclude": "(^src/rpc/server.cpp)" }, "lint-markdown": { "type": "lint-markdown", "include": [ "(\\.md$)" ], "exclude": "(^contrib/gitian-builder/)" }, "lint-python-mypy": { "type": "lint-python-mypy", "version": ">=0.780", "include": "(\\.py$)", "exclude": "(^contrib/)", "flags": [ "--ignore-missing-imports" ] }, "lint-python-mutable-default": { "type": "lint-python-mutable-default", "include": "(\\.py$)", "exclude": [ "(^contrib/gitian-builder/)", "(^contrib/apple-sdk-tools/)" ] }, "prettier": { "type": "prettier", "version":">=2.4.1", "include": "(^web/.*\\.(css|html|js|json|jsx|md|scss|ts|tsx)$)", "exclude": "(^web/.*/translations/.*\\.json$)" }, "lint-python-isort": { "type": "lint-python-isort", "version": ">=5.6.4", "include": "(\\.py$)", "exclude": "(^contrib/)" }, "rustfmt": { "type": "rustfmt", "version": ">=1.5.1", "include": "(\\.rs$)" }, "eslint": { "type": "eslint", "version": ">=8.0.0", "include": "(web/cashtab/.*\\.js$)" + }, + "lint-python-flynt": { + "type": "lint-python-flynt", + "version": ">=0.78", + "include": "(\\.py$)", + "exclude": "(^contrib/)", + "flags": [ + "--transform-concats" + ] } } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a8ceeafc..472296573 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,275 +1,275 @@ Contributing to Bitcoin ABC =========================== The Bitcoin ABC project welcomes contributors! This guide is intended to help developers contribute effectively to Bitcoin ABC. Communicating with Developers ----------------------------- To get in contact with Bitcoin ABC developers, you can join the [eCash Development Telegram group](https://t.me/eCashDevelopment). The intent of this group is to facilitate development of Bitcoin ABC and other eCash node implementations. We welcome people who wish to participate. Acceptable use of this group includes the following: * Introducing yourself to other eCash developers. * Getting help with your development environment. * Discussing how to complete a patch. It is not for: * Market discussion * Non-constructive criticism Bitcoin ABC Development Philosophy ---------------------------------- Bitcoin ABC aims for fast iteration and continuous integration. This means that there should be quick turnaround for patches to be proposed, reviewed, and committed. Changes should not sit in a queue for long. Here are some tips to help keep the development working as intended. These are guidelines for the normal and expected development process. Developers can use their judgement to deviate from these guidelines when they have a good reason to do so. - Keep each change small and self-contained. - Reach out for a 1-on-1 review so things move quickly. - Land the Diff quickly after it is accepted. - Don't amend changes after the Diff accepted, new Diff for another fix. - Review Diffs from other developers as quickly as possible. - Large changes should be broken into logical chunks that are easy to review, and keep the code in a functional state. - Do not mix moving stuff around with changing stuff. Do changes with renames on their own. - Sometimes you want to replace one subsystem by another implementation, in which case it is not possible to do things incrementally. In such cases, you keep both implementations in the codebase for a while, as described [here](http://sevangelatos.com/john-carmack-on-parallel-implementations/) - There are no "development" branches, all Diffs apply to the master branch, and should always improve it (no regressions). - Don't break the build, it is important to keep master green as much as possible. If a Diff is landed, and breaks the build, fix it quickly. If it cannot be fixed quickly, it should be reverted, and re-applied later when it no longer breaks the build. - As soon as you see a bug, you fix it. Do not continue on. Fixing the bug becomes the top priority, more important than completing other tasks. - Automate as much as possible, and spend time on things only humans can do. Here are some handy links for development practices aligned with Bitcoin ABC: - [Developer Notes](doc/developer-notes.md) - [Statement of Bitcoin ABC Values and Visions](https://archive.md/ulgFI) - [How to Make Your Code Reviewer Fall in Love with You](https://mtlynch.io/code-review-love/) - [Large Diffs Are Hurting Your Ability To Ship](https://medium.com/@kurtisnusbaum/large-diffs-are-hurting-your-ability-to-ship-e0b2b41e8acf) - [Stacked Diffs: Keeping Phabricator Diffs Small](https://medium.com/@kurtisnusbaum/stacked-diffs-keeping-phabricator-diffs-small-d9964f4dcfa6) - [Parallel Implementations](http://sevangelatos.com/john-carmack-on-parallel-implementations/) - [The Pragmatic Programmer: From Journeyman to Master](https://www.amazon.com/Pragmatic-Programmer-Journeyman-Master/dp/020161622X) - [Monorepo: Advantages of monolithic version control](https://danluu.com/monorepo/) - [Monorepo: Why Google Stores Billions of Lines of Code in a Single Repository](https://www.youtube.com/watch?v=W71BTkUbdqE) - [The importance of fixing bugs immediately](https://youtu.be/E2MIpi8pIvY?t=16m0s) - [Slow Deployment Causes Meetings](https://www.facebook.com/notes/kent-beck/slow-deployment-causes-meetings/1055427371156793/) - [Good Work, Great Work, and Right Work](https://forum.dlang.org/post/q7u6g1$94p$1@digitalmars.com) - [Accelerate: The Science of Lean Software and DevOps](https://www.amazon.com/Accelerate-Software-Performing-Technology-Organizations/dp/1942788339) - [Facebook Engineering Process with Kent Beck](https://softwareengineeringdaily.com/2019/08/28/facebook-engineering-process-with-kent-beck/) - [Trunk Based Development](https://trunkbaseddevelopment.com/) - [Step-by-step: Programming incrementally](https://ourmachinery.com/post/step-by-step-programming-incrementally/) - [Semantic Compression](https://caseymuratori.com/blog_0015) Getting set up with the Bitcoin ABC Repository ---------------------------------------------- 1. Create an account at [reviews.bitcoinabc.org](https://reviews.bitcoinabc.org/) 2. Install Git and Arcanist on your machine Git documentation can be found at [git-scm.com](https://git-scm.com/). For Arcanist documentation, you can read [Arcanist Quick Start](https://secure.phabricator.com/book/phabricator/article/arcanist_quick_start/) and the [Arcanist User Guide](https://secure.phabricator.com/book/phabricator/article/arcanist/). To install these packages on Debian or Ubuntu, type: `sudo apt-get install git arcanist` 3. If you do not already have an SSH key set up, follow these steps: Type: `ssh-keygen -t ed25519 -C "your_email@example.com"` Enter a file in which to save the key (/home/*username*/.ssh/id_ed25519): [Press enter] 4. Upload your SSH public key to - Go to: `https://reviews.bitcoinabc.org/settings/user/*username*/page/ssh/` - Under "SSH Key Actions", Select "Upload Public Key" Paste contents from: `/home/*username*/.ssh/id_ed25519.pub` 5. Clone the repository and install Arcanist certificate: ``` git clone ssh://vcs@reviews.bitcoinabc.org:2221/source/bitcoin-abc.git cd bitcoin-abc arc install-certificate ``` Note: Arcanist tooling will tend to fail if your remote origin is set to something other than the above. A common mistake is to clone from Github and then forget to update your remotes. Follow instructions provided by `arc install-certificate` to provide your API token. Contributing to the node software --------------------------------- During submission of patches, arcanist will automatically run `arc lint` to enforce Bitcoin ABC code formatting standards, and often suggests changes. If code formatting tools do not install automatically on your system, you will have to install the following: Install all the code formatting tools on Debian Bullseye (11) or Ubuntu 20.04: ``` sudo apt-get install python3-autopep8 python3-pip flake8 php-codesniffer shellcheck yamllint -pip3 install isort>=5.6.4 mypy>=0.780 +pip3 install isort>=5.6.4 mypy>=0.780 flynt>=0.78 echo "export PATH=\"`python3 -m site --user-base`/bin:\$PATH\"" >> ~/.bashrc source ~/.bashrc ``` If not available in the distribution, `clang-format-12` and `clang-tidy-12` can be installed from or . For example, for macOS: ``` curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz | tar -xJv ln -s $PWD/clang+llvm-12.0.0-x86_64-apple-darwin/bin/clang-format /usr/local/bin/clang-format ln -s $PWD/clang+llvm-12.0.0-x86_64-apple-darwin/bin/clang-tidy /usr/local/bin/clang-tidy ``` If you are modifying a shell script, you will need to install the `shellcheck` linter. A recent version is required and may not be packaged for your distribution. Standalone binaries are available for download on [the project's github release page](https://github.com/koalaman/shellcheck/releases). **Note**: In order for arcanist to detect the `shellcheck` executable, you need to make it available in your `PATH`; if another version is already installed, make sure the recent one is found first. Arcanist will tell you what version is expected and what is found when running `arc lint` against a shell script. If you are running Debian 10, it is also available in the backports repository: ``` sudo apt-get -t buster-backports install shellcheck ``` If you are modifying Rust files, you will need to install a stable rust version, plus a nightly toolchain called "abc-nightly" for formatting: ```bash # Install latest stable Rust version curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s source ~/.cargo/env rustup install nightly-2023-02-17 rustup component add rustfmt --toolchain nightly-2023-02-17 # Name the nightly toolchain "abc-nightly" rustup toolchain link abc-nightly "$(rustc +nightly-2023-02-17 --print sysroot)" ``` Contributing to the web projects -------------------------------- To contribute to web projects, you will need `nodejs` > 16 and `npm` > 8.5.0. Follow these [installation instructions](https://github.com/nvm-sh/nvm#installing-and-updating) to install `nodejs` with node version manager. Then: ``` cd bitcoin-abc [sudo] nvm install 16 [sudo] npm install -g npm@latest [sudo] npm install -g prettier [sudo] npm install -g eslint ``` To work on the extension, you will need `browserify` ``` [sudo] npm install -g browserify ``` Working with The Bitcoin ABC Repository --------------------------------------- A typical workflow would be: - Create a topic branch in Git for your changes git checkout -b 'my-topic-branch' - Make your changes, and commit them git commit -a -m 'my-commit' - Create a differential with Arcanist arc diff You should add suggested reviewers and a test plan to the commit message. Note that Arcanist is set up to look only at the most-recent commit message, So all you changes for this Diff should be in one Git commit. - For large changes, break them into several Diffs, as described in this [guide](https://medium.com/@kurtisnusbaum/stacked-diffs-keeping-phabricator-diffs-small-d9964f4dcfa6). You must also include "Depends on Dxxx" in the Arcanist summary to indicate dependence on other Diffs. Note: the `arc land` procedure described in the guide above is obsolete. With the most recent version of arcanist, you may `arc land` the latest commit of your stacked diff after all parts are approved. - Log into Phabricator to see review and feedback. - Make changes as suggested by the reviewers. You can simply edit the files with my-topic-branch checked out, and then type `arc diff`. Arcanist will give you the option to add uncommited changes. Or, alternatively, you can commit the changes using `git commit -a --am` to add them to the last commit, or squash multiple commits by typing `git rebase -i master`. If you squash, make sure the commit message has the information needed for arcanist (such as the Diff number, reviewers, etc.). - Update your Diff by typing `arc diff` again. - When reviewers approve your Diff, it should be listed as "ready to Land" in Phabricator. When you want to commit your diff to the repository, check out type my-topic-branch in git, then type `arc land`. You have now successfully committed a change to the Bitcoin ABC repository. - When reviewing a Diff, apply the changeset on your local by using `arc patch D{NNNN}` - You will likely be re-writing git histories multiple times, which causes timestamp changes that require re-building a significant number of files. It's highly recommended to install `ccache` (re-run cmake if you install it later), as this will help cut your re-build times from several minutes to under a minute, in many cases. What to work on --------------- If you are looking for a useful task to contribute to the project, a good place to start is the list of tasks at . You could also try [backporting](doc/backporting.md) some code from Bitcoin Core. Copyright --------- By contributing to this repository, you agree to license your work under the MIT license unless specified otherwise in `contrib/debian/copyright` or at the top of the file itself. Any work contributed where you are not the original author must contain its license header with the original author(s) and source. Disclosure Policy ----------------- See [DISCLOSURE_POLICY](DISCLOSURE_POLICY.md). diff --git a/arcanist/__phutil_library_map__.php b/arcanist/__phutil_library_map__.php index 9d9725428..e0b5b7231 100644 --- a/arcanist/__phutil_library_map__.php +++ b/arcanist/__phutil_library_map__.php @@ -1,100 +1,100 @@ 2, 'class' => array( 'AbstractGlobalExternalLinter' => 'linter/AbstractGlobalExternalLinter.php', 'ArcanistBitcoinABCConfiguration' => 'configuration/ArcanistBitcoinABCConfiguration.php', 'ArcanistLandBotWorkflow' => 'workflow/ArcanistLandBotWorkflow.php', 'AssertWithSideEffectsLinter' => 'linter/AssertWithSideEffectsLinter.php', 'AutoPEP8FormatLinter' => 'linter/AutoPEP8Linter.php', 'BashShebangLinter' => 'linter/BashShebangLinter.php', 'BoostDependenciesLinter' => 'linter/BoostDependenciesLinter.php', 'CHeaderLinter' => 'linter/CHeaderLinter.php', 'CheckDocLinter' => 'linter/CheckDocLinter.php', 'CheckNonFatalOverAssertInRpc' => 'linter/CheckNonFatalOverAssertInRpc.php', 'ClangFormatLinter' => 'linter/ClangFormatLinter.php', 'CppVoidParameterLinter' => 'linter/CppVoidParameterLinter.php', 'DoxygenLinter' => 'linter/DoxygenLinter.php', 'ESLinter' => 'linter/ESLinter.php', 'ExtendedConfigurationDrivenLintEngine' => 'engine/ExtendedConfigurationDrivenLintEngine.php', 'FileNameLinter' => 'linter/FileNameLinter.php', + 'FlyntFormattLinter' => 'linter/FlyntLinter.php', 'FormatStringLinter' => 'linter/FormatStringLinter.php', 'ISortFormatLinter' => 'linter/ISortLinter.php', 'IncludeGuardLinter' => 'linter/IncludeGuardLinter.php', 'IncludeQuotesLinter' => 'linter/IncludeQuotesLinter.php', 'IncludeSourceLinter' => 'linter/IncludeSourceLinter.php', 'LintOnceInterface' => 'linter/LintOnceInterface.php', 'LocaleDependenceLinter' => 'linter/LocaleDependenceLinter.php', 'LogLinter' => 'linter/LogLinter.php', 'MarkdownLinter' => 'linter/MarkdownLinter.php', 'MyPyLinter' => 'linter/MyPyLinter.php', 'PrettierLinter' => 'linter/PrettierLinter.php', 'PythonFileEncodingLinter' => 'linter/PythonFileEncodingLinter.php', - 'PythonFormatLinter' => 'linter/PythonFormatLinter.php', 'PythonMutableDefaultLinter' => 'linter/PythonMutableDefaultLinter.php', 'PythonShebangLinter' => 'linter/PythonShebangLinter.php', 'Qt5Linter' => 'linter/Qt5Linter.php', 'RustfmtLinter' => 'linter/RustfmtLinter.php', 'ShellCheckLinter' => 'linter/ShellCheckLinter.php', 'ShellLocaleLinter' => 'linter/ShellLocaleLinter.php', 'StdChronoLinter' => 'linter/StdChronoLinter.php', 'StdintLinter' => 'linter/StdintLinter.php', 'TestsLinter' => 'linter/TestsLinter.php', 'WhitespaceLinter' => 'linter/WhitespaceLinter.php', 'YamllintLinter' => 'linter/YamllintLinter.php', ), 'function' => array( 'startsWith' => 'linter/MarkdownLinter.php', ), 'xmap' => array( 'AbstractGlobalExternalLinter' => array( 'ArcanistExternalLinter', 'LintOnceInterface', ), 'ArcanistBitcoinABCConfiguration' => 'ArcanistConfiguration', 'ArcanistLandBotWorkflow' => 'ArcanistWorkflow', 'AssertWithSideEffectsLinter' => 'ArcanistLinter', 'AutoPEP8FormatLinter' => 'ArcanistExternalLinter', 'BashShebangLinter' => 'ArcanistLinter', 'BoostDependenciesLinter' => 'AbstractGlobalExternalLinter', 'CHeaderLinter' => 'ArcanistLinter', 'CheckDocLinter' => 'AbstractGlobalExternalLinter', 'CheckNonFatalOverAssertInRpc' => 'ArcanistLinter', 'ClangFormatLinter' => 'ArcanistExternalLinter', 'CppVoidParameterLinter' => 'ArcanistLinter', 'DoxygenLinter' => 'ArcanistLinter', 'ESLinter' => 'ArcanistExternalLinter', 'ExtendedConfigurationDrivenLintEngine' => 'ArcanistLintEngine', 'FileNameLinter' => 'ArcanistLinter', + 'FlyntFormattLinter' => 'ArcanistExternalLinter', 'FormatStringLinter' => 'ArcanistExternalLinter', 'ISortFormatLinter' => 'ArcanistExternalLinter', 'IncludeGuardLinter' => 'ArcanistLinter', 'IncludeQuotesLinter' => 'ArcanistLinter', 'IncludeSourceLinter' => 'ArcanistLinter', 'LocaleDependenceLinter' => 'ArcanistLinter', 'LogLinter' => 'ArcanistLinter', 'MarkdownLinter' => 'ArcanistLinter', 'MyPyLinter' => 'ArcanistExternalLinter', 'PrettierLinter' => 'ArcanistExternalLinter', 'PythonFileEncodingLinter' => 'ArcanistLinter', - 'PythonFormatLinter' => 'ArcanistExternalLinter', 'PythonMutableDefaultLinter' => 'ArcanistLinter', 'PythonShebangLinter' => 'ArcanistLinter', 'Qt5Linter' => 'ArcanistLinter', 'RustfmtLinter' => 'ArcanistExternalLinter', 'ShellCheckLinter' => 'ArcanistExternalLinter', 'ShellLocaleLinter' => 'ArcanistLinter', 'StdChronoLinter' => 'ArcanistLinter', 'StdintLinter' => 'ArcanistLinter', 'TestsLinter' => 'ArcanistExternalLinter', 'WhitespaceLinter' => 'ArcanistLinter', 'YamllintLinter' => 'ArcanistExternalLinter', ), )); diff --git a/arcanist/linter/FlyntLinter.php b/arcanist/linter/FlyntLinter.php new file mode 100644 index 000000000..1297584c9 --- /dev/null +++ b/arcanist/linter/FlyntLinter.php @@ -0,0 +1,113 @@ +getProjectRoot(); + $path = Filesystem::resolvePath($path, $root); + $orig = file_get_contents($path); + + // remove extra newline added to stdout by flynt + // (bug present at least in versions 0.77 and 0.78) + $stdout = rtrim($stdout) . "\n"; + + if ($orig == $stdout) { + return array(); + } + + $message = id(new ArcanistLintMessage()) + ->setPath($path) + ->setLine(1) + ->setChar(1) + ->setGranularity(ArcanistLinter::GRANULARITY_FILE) + ->setCode('FLYNT') + ->setSeverity(ArcanistLintSeverity::SEVERITY_AUTOFIX) + ->setName('Converting into f-strings') + ->setDescription("'$path' has strings to be converted into f-strings.") + ->setOriginalText($orig) + ->setReplacementText($stdout); + + return array($message); + } +} diff --git a/arcanist/linter/PythonFormatLinter.php b/arcanist/linter/PythonFormatLinter.php deleted file mode 100644 index e928543ca..000000000 --- a/arcanist/linter/PythonFormatLinter.php +++ /dev/null @@ -1,94 +0,0 @@ -getProjectRoot()); - } - - public function shouldUseInterpreter() { - return true; - } - - public function getDefaultInterpreter() { - return "python3"; - } - - public function getInstallInstructions() { - return pht('The test/lint/lint-python-format.py script is part of the '. - 'bitcoin-abc project'); - } - - public function shouldExpectCommandErrors() { - return false; - } - - protected function getMandatoryFlags() { - return array(); - } - - protected function parseLinterOutput($path, $err, $stdout, $stderr) { - $pattern = '/\((\d+)\) ([\s\S]+?)=> (.+)/'; - $found = preg_match_all($pattern, $stdout, $snippets, - $flags = PREG_SET_ORDER); - - /* - * Matched snippets $snippets are organized like this: - * [0] The complete mask - * [1] The line number - * [2] The original snippet - * [3] The replacement snippet - */ - - if (!$found) { - return array(); - } - - $messages = []; - foreach($snippets as $snippet) { - $messages[] = id(new ArcanistLintMessage()) - ->setPath($path) - ->setLine($snippet[1]) - ->setChar(1) - ->setGranularity(ArcanistLinter::GRANULARITY_FILE) - ->setCode('PYFMT') - ->setSeverity(ArcanistLintSeverity::SEVERITY_AUTOFIX) - ->setName('Old string format notation') - ->setDescription("'$path' uses old style string formatting.") - ->setOriginalText(rtrim($snippet[2])) - ->setReplacementText($snippet[3]); - } - - return $messages; - } -} diff --git a/contrib/utils/install-dependencies-bullseye.sh b/contrib/utils/install-dependencies-bullseye.sh index d289ed89a..de5ec31da 100755 --- a/contrib/utils/install-dependencies-bullseye.sh +++ b/contrib/utils/install-dependencies-bullseye.sh @@ -1,179 +1,179 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail dpkg --add-architecture i386 PACKAGES=( arcanist automake autotools-dev binutils bsdmainutils build-essential ccache cmake curl default-jdk devscripts doxygen dput flake8 g++-9 g++-9-aarch64-linux-gnu g++-9-arm-linux-gnueabihf g++-9-multilib g++-mingw-w64 gcc-9 gcc-9-aarch64-linux-gnu gcc-9-arm-linux-gnueabihf gcc-9-multilib gettext-base git golang gnupg graphviz gperf help2man imagemagick jq lcov less lib32stdc++-10-dev libboost-all-dev libbz2-dev libc6-dev:i386 libcap-dev libdb++-dev libdb-dev libevent-dev libjemalloc-dev libminiupnpc-dev libnatpmp-dev libprotobuf-dev libqrencode-dev libqt5core5a libqt5dbus5 libqt5gui5 librsvg2-bin libsqlite3-dev libssl-dev libtiff-tools libtinfo5 libtool libzmq3-dev lld make ninja-build nsis pandoc php-codesniffer pkg-config protobuf-compiler python3 python3-autopep8 python3-pip python3-setuptools python3-yaml python3-zmq qemu-user-static qttools5-dev qttools5-dev-tools shellcheck software-properties-common tar wget xvfb yamllint wine ) function join_by() { local IFS="$1" shift echo "$*" } apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}") # Install llvm and clang apt-key add "$(dirname "$0")"/llvm.pub add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main" apt-get update LLVM_PACKAGES=( clang-12 clang-format-12 clang-tidy-12 clang-tools-12 ) DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}") # Make sure our specific llvm and clang versions have highest priority update-alternatives --install /usr/bin/clang clang "$(command -v clang-12)" 100 update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-12)" 100 update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "$(command -v llvm-symbolizer-12)" 100 # Use gcc-9/g++-9 by default so it uses libstdc++-9. This prevents from pulling # the new pthread_cond_clockwait symbol from GLIBC_30 and ensure we are testing # under the same condition our release it built. update-alternatives --install /usr/bin/gcc gcc "$(command -v gcc-9)" 100 update-alternatives --install /usr/bin/g++ g++ "$(command -v g++-9)" 100 update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc "$(command -v aarch64-linux-gnu-gcc-9)" 100 update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ "$(command -v aarch64-linux-gnu-g++-9)" 100 update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc "$(command -v arm-linux-gnueabihf-gcc-9)" 100 update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ "$(command -v arm-linux-gnueabihf-g++-9)" 100 # Use the mingw posix variant update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix) update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix) # Python library for merging nested structures pip3 install deepmerge # For running Python test suites pip3 install pytest # For en/-decoding protobuf messages # This version is compatible with Debian's "protobuf-compiler" package pip3 install "protobuf<=3.20" -# Up-to-date mypy and isort packages are required python linters -pip3 install isort==5.6.4 mypy==0.780 +# Up-to-date mypy, isort and flynt packages are required python linters +pip3 install isort==5.6.4 mypy==0.780 flynt==0.78 echo "export PATH=\"$(python3 -m site --user-base)/bin:\$PATH\"" >> ~/.bashrc # shellcheck source=/dev/null source ~/.bashrc # Install npm v7.x and nodejs v15.x curl -sL https://deb.nodesource.com/setup_15.x | bash - apt-get install -y nodejs # Install Rust stable 1.67.1 and nightly from the 2023-02-17 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.67.1 RUST_HOME="${HOME}/.cargo/bin" RUST_NIGHTLY_DATE=2023-02-17 "${RUST_HOME}/rustup" install nightly-${RUST_NIGHTLY_DATE} "${RUST_HOME}/rustup" component add rustfmt --toolchain nightly-${RUST_NIGHTLY_DATE} # Name the nightly toolchain "abc-nightly" "${RUST_HOME}/rustup" toolchain link abc-nightly "$(${RUST_HOME}/rustc +nightly-${RUST_NIGHTLY_DATE} --print sysroot)" # Install required compile platform targets on stable "${RUST_HOME}/rustup" target add "i686-unknown-linux-gnu" \ "x86_64-unknown-linux-gnu" \ "aarch64-unknown-linux-gnu" \ "arm-unknown-linux-gnueabihf" \ "x86_64-apple-darwin" \ "x86_64-pc-windows-gnu" # Install corrosion from Github wget https://api.github.com/repos/corrosion-rs/corrosion/tarball/v0.3.0 -O corrosion.tar.gz echo "3b9a48737264add649983df26c83f3285ce17e20d86194c7756689a0d8470267 corrosion.tar.gz" | sha256sum -c tar xzf corrosion.tar.gz CORROSION_SRC_FOLDER=corrosion-rs-corrosion-b764a9f CORROSION_BUILD_FOLDER=${CORROSION_SRC_FOLDER}-build cmake -S${CORROSION_SRC_FOLDER} -B${CORROSION_BUILD_FOLDER} -DCMAKE_BUILD_TYPE=Release cmake --build ${CORROSION_BUILD_FOLDER} --config Release cmake --install ${CORROSION_BUILD_FOLDER} --config Release diff --git a/test/lint/lint-python-format-tests.txt b/test/lint/lint-python-format-tests.txt deleted file mode 100644 index 58c87c79d..000000000 --- a/test/lint/lint-python-format-tests.txt +++ /dev/null @@ -1,80 +0,0 @@ -# This file contains python code with % string formatters to test the -# lint-python-format.py script. - -# Single line -"test %s" % "string" -"pi %.2f" % 3.1415 - -# Multiple lines -"test %s" % - "string" -"test %s" % \ - "string" -"test %s" \ - % "string" -"test %s %s %s" \ - % ("1", "2", "3") -"test %s %s %s" % \ - ("1", "2", "3") -"test %s %s %s" \ - % ("1", - "2", "3") -"test %s %s %s" \ - % ("0" \ - + "1", - "2", "3") - -# In a comment -# "test %s" % "string" - -# Nested comment -"test %s %s %s" \ - % ("1", - #"4", - "2", "3") - -# Inlined comments are not supported -# "test %s %s %s" \ -# % ("1", #4, -# "2", "3") - -# Nested format inside a list, dict or function -["test %s" % "string"] -{"key1":"%s" % "value1", "key2":"value2"} -f("%d" % len("string"), argument2) -f("%d %s" % (len("string"), "argument1"), argument2) -["test %s %s" % - ("string1", "string2")] - -# Multiple formats on a single line -("%s" % "string1", "%s" % "string2") -("%s" % "string1", "%s %s" % ("string2", "string3") -("%s %s" % ("string1", "string2"), "%s %s" % ("string3", "string4")) - -# Combo ! -["test %05i %% %s" % - (len("string1"), - "%d %-10s %%" % (len("string2"), - "string2"))] - -# Code with % but not using it for formatting should not throw any linter error -"test %100" -"test %\n" -"test %%" -"test %d" 42 -"test % 42" -"test %\ - 1" -"test % \ - 1" -a = 10 % 2 - -# An array -"test %s" % an_array[0] -# A matrix -"test %s" % an_array[0][0] -# An array in an array -["test %s" % an_array[0]] -# An matrix in an array in a dict -{"test":" ["test %s" % an_array[0][0]]} - diff --git a/test/lint/lint-python-format.py b/test/lint/lint-python-format.py deleted file mode 100755 index 2997eda87..000000000 --- a/test/lint/lint-python-format.py +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2019 The Bitcoin developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint python format : This program checks that the old python fomatting method -# is not being used (formatting with "string %s" % content). -# The new "{}".format(content) or f"{} content" method should be used instead. -# Usage of the % formatter is expected to be deprecated by python in the -# future. - -import re -import sys -from doctest import testmod - - -def is_complete(snippet): - r"""Check if a code snippet is complete. - - >>> is_complete("a = [1, 2, 3]") - True - >>> is_complete("a = [1,") - False - >>> is_complete("a = [1, (") - False - >>> is_complete("a = [") - False - >>> is_complete("a = [1, {") - False - >>> is_complete('a = [1, 2, "%d" % \\') - False - >>> is_complete('a = [1, 2, \"\"\"') - False - >>> is_complete('"%s" %') - False - """ - can_continue = [',', '(', '[', '{', '\\', '"""', '%'] - return not any(snippet.strip().endswith(end) for end in can_continue) - - -def build_replacement(error): - r"""Replace a snippet using the % formatter with a version using .format(). - - >>> build_replacement('"test %s" % "1"') - '"test {}".format("1")' - >>> build_replacement('"test %s" % ("1")') - '"test {}".format("1")' - >>> build_replacement('"test %.2f" % 3.1415') - '"test {:.2f}".format(3.1415)' - >>> build_replacement('"test %s" \\\n% "1"') - '"test {}".format("1")' - >>> build_replacement('"test %s" %\\\n"1"') - '"test {}".format("1")' - >>> build_replacement('"test %s %s %s" % ("1", "2", "3")') - '"test {} {} {}".format("1", "2", "3")' - >>> build_replacement('"test %s %s %s" % \\\n("1", "2", "3")') - '"test {} {} {}".format("1", "2", "3")' - >>> build_replacement('"test %s %s %s" % \\\n("1",\n"2", "3")') - '"test {} {} {}".format("1", "2", "3")' - >>> build_replacement('"test %d" % 1') - '"test {}".format(1)' - >>> build_replacement('"test %i" % 1') - '"test {}".format(1)' - >>> build_replacement('"test %r" % "1"') - '"test {!r}".format("1")' - >>> build_replacement('"test %-10s" % "1"') - '"test {:10s}".format("1")' - >>> build_replacement('"test %s.%s" % ("1", "2")') - '"test {}.{}".format("1", "2")' - >>> build_replacement('"test %% %s %s" % ("1", "2")') - '"test % {} {}".format("1", "2")' - >>> build_replacement('"test %s %% %s" % ("1", "2")') - '"test {} % {}".format("1", "2")' - >>> build_replacement('"test %s %s %%" % ("1", "2")') - '"test {} {} %".format("1", "2")' - >>> build_replacement('"test %%%s%%%s%%" % ("1", "2")') - '"test %{}%{}%".format("1", "2")' - """ - # Inline the error snippet. - # Replace line continuation ('\'), line breaks and their surrounding - # spaces and indentation to a single space character - replacement = re.sub(r"\s*\\\s+", " ", error, re.MULTILINE) - replacement = re.sub(r"\s*(?:\r?\n|\r(?!\n))\s*", - " ", replacement, re.MULTILINE) - - # Escape the %% in 2 passes to avoid the % character to mess up with the - # regexes - # First change %% to \xec\xec then back \xec\xec to % (\xec is the infinity - # symbol in extended ascii, it is unlikely to encounter it twice) - replacement = re.sub(r"%%", "\xec\xec", replacement, re.MULTILINE) - - # Replace the specifiers, retaining their content. - # E.g. %.2f => {:.2f} - def specifier_sub(match): - # There are some special cases to handle: - # - {:s} only works with strings, but %s worked with almost anything. - # To avoid type errors, just use {} - # - {:i} does not exists, it should be {:d} or better {}. - # - {:r} is invalid, the new syntax is {!r} - # - The left alignement marker (e.g. %-5s) is now the default, remove it - specifier = match.group(1) - specifier_converts_to_empty_brackets = ["s", "i", "d"] - if specifier in specifier_converts_to_empty_brackets: - return "{}" - elif specifier == "r": - return "{!r}" - elif specifier.startswith("-"): - return f"{{:{specifier[1:]}}}" - specifier = specifier.replace("i", "d") - return f"{{:{specifier}}}" - - (replacement, count) = re.subn( - r"%([.-]?[0-9]*[a-zA-Z])", specifier_sub, replacement, flags=re.MULTILINE) - - # Replace the qualifier. - # E.g % 42 => .format(42) - # E.g. % (42, "my_string") => .format(42, "my_string") - def single_qualifier_sub(match): - qualifier = f".format({match.group(1).strip()}" - # Where to close the parenthesis if there is a single specifier ? - # It is whether at the end or before the first ',', ']', '}' (if - # enclosed in a function call, a list or a dictionary). - # - # There is a special case to be handled when the qualifier is an array. - # In this case, ensure there is one more ']' than '['. - close_before = [",", "]", "}"] - opening_count = 0 - for i, c in enumerate(qualifier): - if c == "[": - opening_count += 1 - if c in close_before: - if c == "]" and opening_count > 0: - opening_count -= 1 - continue - return f"{qualifier[:i]}){qualifier[i:]}" - return f"{qualifier})" - - def multi_qualifier_sub(match): - # The closing parenthesis is already there as we are replacing a tuple - qualifier = f".format({match.group(1).strip()}" - return qualifier - - # There are 2 possible way to write the qualifier: - # - If there is a single qualifier, it can be set directly. - # E.g.: "%s" % "string" - # - It can always be set as a tuple: - # E.g.: "%s" % ("string") - # E.g.: "%s %s" % ("string1", "string2") - # - # Solution: try to find the pattern with the opening parenthesis first, then - # fall back to no parenthesis. - replacement = re.sub(r"\s*(?) - => - - >>> main("test/lint/lint-python-format-tests.txt") - (5) "test %s" % "string" - => "test {}".format("string") - (6) "pi %.2f" % 3.1415 - => "pi {:.2f}".format(3.1415) - (9) "test %s" % - "string" - => "test {}".format("string") - (11) "test %s" % \ - "string" - => "test {}".format("string") - (13) "test %s" \ - % "string" - => "test {}".format("string") - (15) "test %s %s %s" \ - % ("1", "2", "3") - => "test {} {} {}".format("1", "2", "3") - (17) "test %s %s %s" % \ - ("1", "2", "3") - => "test {} {} {}".format("1", "2", "3") - (19) "test %s %s %s" \ - % ("1", - "2", "3") - => "test {} {} {}".format("1", "2", "3") - (22) "test %s %s %s" \ - % ("0" \ - + "1", - "2", "3") - => "test {} {} {}".format("0" + "1", "2", "3") - (31) "test %s %s %s" \ - % ("1", - "2", "3") - => "test {} {} {}".format("1", "2", "3") - (42) ["test %s" % "string"] - => ["test {}".format("string")] - (43) {"key1":"%s" % "value1", "key2":"value2"} - => {"key1":"{}".format("value1"), "key2":"value2"} - (44) f("%d" % len("string"), argument2) - => f("{}".format(len("string")), argument2) - (45) f("%d %s" % (len("string"), "argument1"), argument2) - => f("{} {}".format(len("string"), "argument1"), argument2) - (46) ["test %s %s" % - ("string1", "string2")] - => ["test {} {}".format("string1", "string2")] - (50) ("%s" % "string1", "%s" % "string2") - => ("{}".format("string1"), "{}".format("string2")) - (51) ("%s" % "string1", "%s %s" % ("string2", "string3") - => ("{}".format("string1"), "{} {}".format("string2", "string3") - (52) ("%s %s" % ("string1", "string2"), "%s %s" % ("string3", "string4")) - => ("{} {}".format("string1", "string2"), "{} {}".format("string3", "string4")) - (55) ["test %05i %% %s" % - (len("string1"), - "%d %-10s %%" % (len("string2"), - "string2"))] - => ["test {:05d} % {}".format(len("string1"), "{} {:10s} %".format(len("string2"), "string2"))] - (73) "test %s" % an_array[0] - => "test {}".format(an_array[0]) - (75) "test %s" % an_array[0][0] - => "test {}".format(an_array[0][0]) - (77) ["test %s" % an_array[0]] - => ["test {}".format(an_array[0])] - (79) {"test":" ["test %s" % an_array[0][0]]} - => {"test":" ["test {}".format(an_array[0][0])]} - """ - errors = find_errors(file) - # Python dictionnaries do not guarantee ordering, sort by line number - for line_number, error in sorted(errors.items(), - key=lambda pair: int(pair[0])): - replacement = build_replacement(error) - print(f"({line_number}) {error.rstrip()}") - print(f"=> {replacement}") - - -if __name__ == "__main__": - if len(sys.argv) != 2: - sys.exit(testmod()[1]) - else: - main(sys.argv[1])