diff --git a/.arclint b/.arclint
index 5e2af14b7..f3c9f5683 100644
--- a/.arclint
+++ b/.arclint
@@ -1,215 +1,216 @@
{
"linters": {
"generated": {
"type": "generated"
},
"clang-format": {
"type": "clang-format",
"version": ">=8.0",
"bin": ["clang-format-8", "clang-format"],
"include": "(^src/.*\\.(h|c|cpp|mm)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"autopep8": {
"type": "autopep8",
"version": ">=1.3.4",
"include": "(\\.py$)",
"flags": [
"--aggressive",
"--ignore=W503,W504"
]
},
"flake8": {
"type": "flake8",
"version": ">=3.0",
"include": "(\\.py$)",
"flags": [
"--ignore=E501,E704,W503,W504"
]
},
"lint-format-strings": {
"type": "lint-format-strings",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"check-doc": {
"type": "check-doc",
"include": "(^src/.*\\.(h|c|cpp)$)"
},
"lint-tests": {
"type": "lint-tests",
"include": "(^src/(seeder/|rpc/|wallet/)?test/.*\\.(cpp)$)"
},
"lint-python-format": {
"type": "lint-python-format",
"include": "(\\.py$)",
"exclude": [
"(^test/lint/lint-python-format\\.py$)"
]
},
"phpcs": {
"type": "phpcs",
"include": "(\\.php$)",
"exclude": [
"(^arcanist/__phutil_library_.+\\.php$)"
],
"phpcs.standard": "arcanist/phpcs.xml"
},
"lint-locale-dependence": {
"type": "lint-locale-dependence",
"include": "(^src/.*\\.(h|cpp)$)",
"exclude": [
"(^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/))"
]
},
"lint-cheader": {
"type": "lint-cheader",
"include": "(^src/.*\\.(h|cpp)$)",
"exclude": [
"(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)"
]
},
"spelling": {
"type": "spelling",
"exclude": [
"(^build-aux/m4/)",
"(^depends/)",
"(^doc/release-notes/)",
"(^src/(qt/locale|secp256k1|univalue|leveldb)/)",
"(^test/lint/dictionary/)"
],
"spelling.dictionaries": [
"test/lint/dictionary/english.json"
]
},
"lint-assert-with-side-effects": {
"type": "lint-assert-with-side-effects",
"include": "(^src/.*\\.(h|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"lint-include-quotes": {
"type": "lint-include-quotes",
"include": "(^src/.*\\.(h|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"lint-include-guard": {
"type": "lint-include-guard",
"include": "(^src/.*\\.h$)",
"exclude": [
"(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)",
"(^src/tinyformat.h$)"
]
},
"lint-include-source": {
"type": "lint-include-source",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"lint-stdint": {
"type": "lint-stdint",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)",
"(^src/compat/assumptions.h$)"
]
},
"lint-source-filename": {
"type": "lint-source-filename",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"lint-boost-dependencies": {
"type": "lint-boost-dependencies",
"include": "(^src/.*\\.(h|cpp)$)"
},
"check-rpc-mappings": {
"type": "check-rpc-mappings",
"include": "(^src/(rpc/|wallet/rpc).*\\.cpp$)"
},
"lint-python-encoding": {
"type": "lint-python-encoding",
"include": "(\\.py$)"
},
"lint-python-shebang": {
"type": "lint-python-shebang",
"include": "(\\.py$)",
"exclude": [
"(__init__\\.py$)"
]
},
"lint-bash-shebang": {
"type": "lint-bash-shebang",
"include": "(\\.sh$)"
},
"shellcheck": {
"type": "shellcheck",
+ "version": ">=0.7.0",
"flags": [
"--external-sources",
"--source-path=SCRIPTDIR"
],
"include": "(\\.sh$)",
"exclude": [
"(^src/(secp256k1|univalue)/)"
]
},
"lint-shell-locale": {
"type": "lint-shell-locale",
"include": "(\\.sh$)",
"exclude": [
"(^src/(secp256k1|univalue)/)"
]
},
"lint-cpp-void-parameters": {
"type": "lint-cpp-void-parameters",
"include": "(^src/.*\\.(h|cpp)$)",
"exclude": [
"(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)",
"(^src/compat/glibc_compat.cpp$)"
]
},
"lint-logs": {
"type": "lint-logs",
"include": "(^src/.*\\.(h|cpp)$)"
},
"lint-qt": {
"type": "lint-qt",
"include": "(^src/qt/.*\\.(h|cpp)$)",
"exclude": [
"(^src/qt/(locale|forms|res)/)"
]
},
"lint-doxygen": {
"type": "lint-doxygen",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)"
]
},
"lint-whitespace": {
"type": "lint-whitespace",
"include": "(\\.(ac|am|cmake|conf|in|include|json|m4|md|openrc|php|pl|sh|txt|yml)$)",
"exclude": [
"(^src/(secp256k1|univalue|leveldb)/)"
]
},
"lint-cppcheck": {
"type": "lint-cppcheck",
"include": "(^src/.*\\.(h|c|cpp)$)",
"exclude": [
"(^src/(crypto/ctaes|secp256k1|univalue|leveldb)/)"
]
}
}
}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 70da7296a..b1006e3a1 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,218 +1,231 @@
Contributing to Bitcoin ABC
===========================
The Bitcoin ABC project welcomes contributors!
This guide is intended to help developers contribute effectively to Bitcoin ABC.
Communicating with Developers
-----------------------------
To get in contact with ABC developers, we monitor a telegram supergroup. The
intent of this group is specifically to facilitate development of Bitcoin-ABC,
and to welcome people who wish to participate.
https://t.me/joinchat/HCYr50mxRWjA2uLqii-psw
Acceptable use of this supergroup includes the following:
* Introducing yourself to other ABC developers.
* Getting help with your development environment.
* Discussing how to complete a patch.
It is not for:
* Market discussion
* Non-constructive criticism
Bitcoin ABC Development Philosophy
----------------------------------
Bitcoin ABC aims for fast iteration and continuous integration.
This means that there should be quick turnaround for patches to be proposed,
reviewed, and committed. Changes should not sit in a queue for long.
Here are some tips to help keep the development working as intended. These
are guidelines for the normal and expected development process. Developers
can use their judgement to deviate from these guidelines when they have a
good reason to do so.
- Keep each change small and self-contained.
- Reach out for a 1-on-1 review so things move quickly.
- Land the Diff quickly after it is accepted.
- Don't amend changes after the Diff accepted, new Diff for another fix.
- Review Diffs from other developers as quickly as possible.
- Large changes should be broken into logical chunks that are easy to review,
and keep the code in a functional state.
- Do not mix moving stuff around with changing stuff. Do changes with renames
on their own.
- Sometimes you want to replace one subsystem by another implementation,
in which case it is not possible to do things incrementally. In such cases,
you keep both implementations in the codebase for a while, as described
[here](https://www.gamasutra.com/view/news/128325/Opinion_Parallel_Implementations.php)
- There are no "development" branches, all Diffs apply to the master
branch, and should always improve it (no regressions).
- Don't break the build, it is important to keep master green as much as possible.
If a Diff is landed, and breaks the build, fix it quickly. If it cannot be fixed
quickly, it should be reverted, and re-applied later when it no longer breaks the build.
- As soon as you see a bug, you fix it. Do not continue on. Fixing the bug becomes the
top priority, more important than completing other tasks.
- Automate as much as possible, and spend time on things only humans can do.
Here are some handy links for development practices aligned with Bitcoin ABC:
- [Developer Notes](doc/developer-notes.md)
- [Statement of Bitcoin ABC Values and Visions](https://www.yours.org/content/bitcoin-abc---our-values-and-vision-a282afaade7c)
- [How to Do Code Reviews Like a Human - Part 1](https://mtlynch.io/human-code-reviews-1/)
- [How to Do Code Reviews Like a Human - Part 2](https://mtlynch.io/human-code-reviews-2/)
- [Large Diffs Are Hurting Your Ability To Ship](https://medium.com/@kurtisnusbaum/large-diffs-are-hurting-your-ability-to-ship-e0b2b41e8acf)
- [Stacked Diffs: Keeping Phabricator Diffs Small](https://medium.com/@kurtisnusbaum/stacked-diffs-keeping-phabricator-diffs-small-d9964f4dcfa6)
- [Parallel Implementations](https://www.gamasutra.com/view/news/128325/Opinion_Parallel_Implementations.php)
- [The Pragmatic Programmer: From Journeyman to Master](https://www.amazon.com/Pragmatic-Programmer-Journeyman-Master/dp/020161622X)
- [Advantages of monolithic version control](https://danluu.com/monorepo/)
- [The importance of fixing bugs immediately](https://youtu.be/E2MIpi8pIvY?t=16m0s)
- [Slow Deployment Causes Meetings](https://www.facebook.com/notes/kent-beck/slow-deployment-causes-meetings/1055427371156793/)
- [Good Work, Great Work, and Right Work](https://forum.dlang.org/post/q7u6g1$94p$1@digitalmars.com)
- [Accelerate: The Science of Lean Software and DevOps](https://www.amazon.com/Accelerate-Software-Performing-Technology-Organizations/dp/1942788339)
Getting set up with the Bitcoin ABC Repository
----------------------------------------------
1. Create an account at https://reviews.bitcoinabc.org/
2. Install Git and Arcanist on your machine
Git documentation can be found at: https://git-scm.com/
Arcanist documentation can be found at:
https://secure.phabricator.com/book/phabricator/article/arcanist_quick_start/
And: https://secure.phabricator.com/book/phabricator/article/arcanist/
To install these packages on Debian or Ubuntu, type: `sudo apt-get install git arcanist`
3. If you do not already have an SSH key set up, follow these steps:
Type: `ssh-keygen -t rsa -b 4096 -C "your_email@example.com"`
Enter a file in which to save the key (/home/*username*/.ssh/id_rsa): [Press enter]
4. Upload your SSH public key to reviews.bitcoinabc.org
- Go to: `https://reviews.bitcoinabc.org/settings/user/*username*/page/ssh/`
- Under "SSH Key Actions", Select "Upload Public Key"
Paste contents from: `/home/*username*/.ssh/id_rsa.pub`
5. Clone the repository and install Arcanist certificate:
```
git clone ssh://vcs@reviews.bitcoinabc.org:2221/source/bitcoin-abc.git
cd bitcoin-abc
arc install-certificate
```
Note: Arcanist tooling will tend to fail if your remote origin is set to something other
than the above. A common mistake is to clone from Github and then forget to update
your remotes.
Follow instructions provided by `arc install-certificate` to provide your API token.
6. Code formatting tools
During submission of patches, arcanist will automatically run `arc lint` to
enforce Bitcoin ABC code formatting standards, and often suggests changes.
If code formatting tools do not install automatically on your system, you
will have to install the following:
On Ubuntu (>= 18.04+updates) or Debian (>= 10):
```
-sudo apt-get install clang-format-8 clang-tidy-8 clang-tools-8 cppcheck python-autopep8 flake8 php-codesniffer shellcheck
+sudo apt-get install clang-format-8 clang-tidy-8 clang-tools-8 cppcheck python-autopep8 flake8 php-codesniffer
```
If not available in the distribution, `clang-format-8` and `clang-tidy` can be
installed from https://releases.llvm.org/download.html or https://apt.llvm.org.
For example, for macOS:
```
curl http://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-apple-darwin.tar.xz | tar -xJv
ln -s $PWD/clang+llvm-8.0.0-x86_64-apple-darwin/bin/clang-format /usr/local/bin/clang-format
ln -s $PWD/clang+llvm-8.0.0-x86_64-apple-darwin/bin/clang-tidy /usr/local/bin/clang-tidy
```
+If you are modifying a shell script, you will need to install the `shellcheck` linter.
+A recent version is required and may not be packaged for your distribution.
+Standalone binaries are available for download on the project's github release page:
+https://github.com/koalaman/shellcheck/releases
+
+If you are running Debian 10, it is also available in the backports repository:
+```
+echo "deb http://deb.debian.org/debian buster-backports main" | sudo tee -a /etc/apt/sources.list
+sudo apt-get update
+sudo apt-get -t buster-backports install shellcheck
+```
+
+
Working with The Bitcoin ABC Repository
---------------------------------------
A typical workflow would be:
- Create a topic branch in Git for your changes
git checkout -b 'my-topic-branch'
- Make your changes, and commit them
git commit -a -m 'my-commit'
- Create a differential with Arcanist
arc diff
You should add suggested reviewers and a test plan to the commit message.
Note that Arcanist is set up to look only at the most-recent commit message,
So all you changes for this Diff should be in one Git commit.
- For large changes, break them into several Diffs, as described in this
[guide](https://medium.com/@kurtisnusbaum/stacked-diffs-keeping-phabricator-diffs-small-d9964f4dcfa6).
You can also include "Depends on Dxxx" in the Arcanist message to indicate
dependence on other Diffs.
- Log into Phabricator to see review and feedback.
- Make changes as suggested by the reviewers. You can simply edit the files
with my-topic-branch checked out, and then type `arc diff`. Arcanist will
give you the option to add uncommited changes. Or, alternatively, you can
commit the changes using `git commit -a --am` to add them to the last commit,
or squash multiple commits by typing `git rebase -i master`. If you squash,
make sure the commit message has the information needed for arcanist (such
as the Diff number, reviewers, etc.).
- Update your Diff by typing `arc diff` again.
- When reviewers approve your Diff, it should be listed as "ready to Land"
in Phabricator. When you want to commit your diff to the repository, check out
type my-topic-branch in git, then type `arc land`. You have now successfully
committed a change to the Bitcoin ABC repository.
- When reviewing a Diff, apply the changeset on your local by using
`arc patch D{NNNN}`
- You will likely be re-writing git histories multiple times, which causes
timestamp changes that require re-building a significant number of files. It's
highly recommended to install `ccache` (re-run cmake if you install it
later), as this will help cut your re-build times from several minutes to under
a minute, in many cases.
What to work on
---------------
If you are looking for a useful task to contribute to the project, a good place
to start is the list of tasks at https://reviews.bitcoinabc.org/maniphest/
You could also try [backporting](doc/backporting.md) some code from Bitcoin Core.
Copyright
---------
By contributing to this repository, you agree to license your work under the
MIT license unless specified otherwise in `contrib/debian/copyright` or at
the top of the file itself. Any work contributed where you are not the original
author must contain its license header with the original author(s) and source.
Disclosure Policy
-----------------
See [DISCLOSURE_POLICY](DISCLOSURE_POLICY.md).
diff --git a/arcanist/linter/ShellCheckLinter.php b/arcanist/linter/ShellCheckLinter.php
index 8f34d735d..aa25bba7f 100644
--- a/arcanist/linter/ShellCheckLinter.php
+++ b/arcanist/linter/ShellCheckLinter.php
@@ -1,87 +1,98 @@
<?php
/**
* Uses the ShellCheck tool to run static analysis on shell scripts
*/
final class ShellCheckLinter extends ArcanistExternalLinter {
const SHELLCHECK_EXCLUDED_RULES = array(
// phpcs:disable Generic.Files.LineLength.TooLong
// phpcs:disable Generic.Files.LineLength.MaxExceeded
'SC1117', // Backslash is literal in "\.". Prefer explicit escaping: "\\.".
'SC2046', // Quote this to prevent word splitting.
'SC2086', // Double quote to prevent globbing and word splitting.
'SC2162', // read without -r will mangle backslashes.
// phpcs:enable
);
const SHELLCHECK_SEVERITY_MAP = array(
'note' => ArcanistLintSeverity::SEVERITY_ADVICE,
'warning' => ArcanistLintSeverity::SEVERITY_WARNING,
'error' => ArcanistLintSeverity::SEVERITY_ERROR,
);
public function getInfoName() {
return 'shellcheck';
}
public function getInfoDescription() {
return pht('Use shellcheck for processing specified files.');
}
public function getLinterName() {
return 'SHELLCHECK';
}
public function getLinterConfigurationName() {
return 'shellcheck';
}
public function getDefaultBinary() {
return 'shellcheck';
}
public function getInstallInstructions() {
return pht('Make sure shellcheck is in directory specified by $PATH');
}
public function shouldExpectCommandErrors() {
return true;
}
+ public function getVersion() {
+ list($stdout) = execx('%C --version', $this->getExecutableCommand());
+
+ $regex = '/^version: (?P<version>\d+\.\d+\.\d+)/m';
+ if (preg_match($regex, $stdout, $matches)) {
+ return $matches['version'];
+ }
+
+ return false;
+ }
+
protected function getMandatoryFlags() {
return array(
'--format=gcc',
'--exclude='.implode(',', self::SHELLCHECK_EXCLUDED_RULES));
}
private function getSeverity($severity) {
if (array_key_exists($severity, self::SHELLCHECK_SEVERITY_MAP)) {
return self::SHELLCHECK_SEVERITY_MAP[$severity];
}
return ArcanistLintSeverity::SEVERITY_ERROR;
}
protected function parseLinterOutput($path, $err, $stdout, $stderr) {
$messages = array();
$pattern = '/(.+):(\d+):(\d+): (.+): (.+) \[(SC\d+)\]/';
if (preg_match_all($pattern, $stdout, $errors, PREG_SET_ORDER)) {
foreach ($errors as $error) {
list(, $file, $line, $char, $severity, $message, $code) = $error;
$messages[] = id(new ArcanistLintMessage())
->setGranularity(ArcanistLinter::GRANULARITY_FILE)
->setPath($path)
->setLine($line)
->setChar($char)
->setCode($code)
->setSeverity($this->getSeverity($severity))
->setName('ShellCheck found an issue:')
->setDescription($message);
}
}
return $messages;
}
}
diff --git a/contrib/teamcity/setup-debian-buster.sh b/contrib/teamcity/setup-debian-buster.sh
index 58393f89c..662585734 100755
--- a/contrib/teamcity/setup-debian-buster.sh
+++ b/contrib/teamcity/setup-debian-buster.sh
@@ -1,109 +1,116 @@
#!/usr/bin/env bash
export LC_ALL=C.UTF-8
set -euxo pipefail
dpkg --add-architecture i386
PACKAGES=(
arcanist
automake
autotools-dev
binutils
bsdmainutils
build-essential
ccache
cmake
cppcheck
curl
flake8
g++-aarch64-linux-gnu
g++-arm-linux-gnueabihf
git
g++-mingw-w64
gnupg
gperf
imagemagick
jq
lcov
less
lib32stdc++-8-dev
libboost-all-dev
libbz2-dev
libc6-dev:i386
libcap-dev
libdb++-dev
libdb-dev
libevent-dev
libjemalloc-dev
libminiupnpc-dev
libprotobuf-dev
libqrencode-dev
libqt5core5a
libqt5dbus5
libqt5gui5
librsvg2-bin
libssl-dev
libtiff-tools
libtinfo5
libtool
libzmq3-dev
make
ninja-build
nsis
php-codesniffer
pkg-config
protobuf-compiler
python3
python3-autopep8
python3-setuptools
python3-zmq
qemu-user-static
qttools5-dev
qttools5-dev-tools
- shellcheck
software-properties-common
tar
wget
wine
)
function join_by() {
local IFS="$1"
shift
echo "$*"
}
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}")
+BACKPORTS=(
+ shellcheck
+)
+
+echo "deb http://deb.debian.org/debian buster-backports main" | tee -a /etc/apt/sources.list
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get -t buster-backports install -y $(join_by ' ' "${BACKPORTS[@]}")
+
TEAMCITY_DIR=$(dirname "$0")
# FIXME this should no longer be needed starting with Teamcity 2020.1, which
# supports Java 11.
"${TEAMCITY_DIR}/install_openjdk8.sh"
# Install llvm-8 and clang-10
apt-key add "${TEAMCITY_DIR}"/llvm.pub
add-apt-repository "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-8 main"
add-apt-repository "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-10 main"
apt-get update
LLVM_PACKAGES=(
clang-8
clang-10
clang-format-8
clang-tidy-8
clang-tools-8
)
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}")
update-alternatives --install /usr/bin/clang clang "$(command -v clang-8)" 100
update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-8)" 100
# Use a lower priority to keep clang-8 the default
update-alternatives --install /usr/bin/clang clang "$(command -v clang-10)" 50
update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-10)" 50
# Use the mingw posix variant
update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix)
update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix)