diff --git a/CVE-2020-7921.patch b/CVE-2020-7921.patch deleted file mode 100644 index 8bb520d513c3abd793311e4d6f8156fe1cb7bf09..0000000000000000000000000000000000000000 --- a/CVE-2020-7921.patch +++ /dev/null @@ -1,159 +0,0 @@ -From fb87cc88ecb5d300f14cda7bc238d7d5132118f5 Mon Sep 17 00:00:00 2001 -From: Spencer Jackson -Date: Wed, 15 Jan 2020 16:30:37 +0000 -Subject: [PATCH] SERVER-45472 Ensure RoleGraph can serialize authentication - restrictions to BSON - -(cherry picked from commit 521e56b407ac72bc69a97a24d1253f51a5b6e81b) -(cherry picked from commit a10d0a22d5d009d27664967181042933ec1bef36) ---- - .../auth/authentication_restrictions_role.js | 8 +++++ - src/mongo/db/auth/role_graph.cpp | 9 +++++ - src/mongo/db/auth/role_graph_test.cpp | 36 ++++++++++++++++++- - 3 files changed, 52 insertions(+), 1 deletion(-) - -diff --git a/jstests/auth/authentication_restrictions_role.js b/jstests/auth/authentication_restrictions_role.js -index 3f23cfdcb921..691491a0765c 100644 ---- a/jstests/auth/authentication_restrictions_role.js -+++ b/jstests/auth/authentication_restrictions_role.js -@@ -42,6 +42,14 @@ - assert.commandWorked(admin.runCommand({createRole: "role3", roles: [], privileges: []})); - - print("=== Role creation tests"); -+ print("When a role is updated, it retains authenticationRestrictions"); -+ assert.commandWorked(admin.runCommand({updateRole: "role2", roles: ["root"]})); -+ const role2Info = assert.commandWorked( -+ admin.runCommand({rolesInfo: "role2", showAuthenticationRestrictions: true})); -+ printjson(role2Info); -+ assert.eq(JSON.stringify([[{clientSource: ["127.0.0.1/32"]}]]), -+ JSON.stringify(role2Info.roles[0].authenticationRestrictions)); -+ - print( - "When a client creates roles with empty authenticationRestrictions, the operation succeeds, though it has no effect"); - assert.commandWorked(admin.runCommand( -diff --git a/src/mongo/db/auth/role_graph.cpp b/src/mongo/db/auth/role_graph.cpp -index fc55b6a1d43b..03776361e3ca 100644 ---- a/src/mongo/db/auth/role_graph.cpp -+++ b/src/mongo/db/auth/role_graph.cpp -@@ -590,6 +590,15 @@ Status RoleGraph::getBSONForRole(RoleGraph* graph, - uassertStatusOK(rolesArrayElement.pushBack(roleObj)); - } - -+ // Build authentication restrictions -+ auto restrictions = graph->getDirectAuthenticationRestrictions(roleName); -+ mutablebson::Element authenticationRestrictionsElement = -+ result.getDocument().makeElementArray("authenticationRestrictions"); -+ uassertStatusOK(result.pushBack(authenticationRestrictionsElement)); -+ if (restrictions) { -+ uassertStatusOK(authenticationRestrictionsElement.setValueArray(restrictions->toBSON())); -+ } -+ - return Status::OK(); - } catch (...) { - return exceptionToStatus(); -diff --git a/src/mongo/db/auth/role_graph_test.cpp b/src/mongo/db/auth/role_graph_test.cpp -index 0e22892f4ebf..e763b0929f99 100644 ---- a/src/mongo/db/auth/role_graph_test.cpp -+++ b/src/mongo/db/auth/role_graph_test.cpp -@@ -35,6 +35,7 @@ - #include - - #include "mongo/bson/mutable/document.h" -+#include "mongo/db/auth/address_restriction.h" - #include "mongo/db/auth/role_graph.h" - #include "mongo/unittest/unittest.h" - #include "mongo/util/mongoutils/str.h" -@@ -48,16 +49,21 @@ TEST(RoleParsingTest, BuildRoleBSON) { - RoleName roleA("roleA", "dbA"); - RoleName roleB("roleB", "dbB"); - RoleName roleC("roleC", "dbC"); -+ RoleName roleD("roleD", "dbD"); - ActionSet actions; - actions.addAction(ActionType::find); - actions.addAction(ActionType::insert); -+ SharedRestrictionDocument restrictions = uassertStatusOK(parseAuthenticationRestriction( -+ BSON_ARRAY(BSON("clientSource" << BSON_ARRAY("127.0.0.1"))))); - - ASSERT_OK(graph.createRole(roleA)); - ASSERT_OK(graph.createRole(roleB)); - ASSERT_OK(graph.createRole(roleC)); -+ ASSERT_OK(graph.createRole(roleD)); - - ASSERT_OK(graph.addRoleToRole(roleA, roleC)); - ASSERT_OK(graph.addRoleToRole(roleA, roleB)); -+ ASSERT_OK(graph.addRoleToRole(roleA, roleD)); - ASSERT_OK(graph.addRoleToRole(roleB, roleC)); - - ASSERT_OK(graph.addPrivilegeToRole( -@@ -66,6 +72,7 @@ TEST(RoleParsingTest, BuildRoleBSON) { - roleB, Privilege(ResourcePattern::forExactNamespace(NamespaceString("dbB.foo")), actions))); - ASSERT_OK( - graph.addPrivilegeToRole(roleC, Privilege(ResourcePattern::forClusterResource(), actions))); -+ ASSERT_OK(graph.replaceRestrictionsForRole(roleD, restrictions)); - ASSERT_OK(graph.recomputePrivilegeData()); - - -@@ -78,6 +85,8 @@ TEST(RoleParsingTest, BuildRoleBSON) { - ASSERT_EQUALS("roleA", roleDoc["role"].String()); - ASSERT_EQUALS("dbA", roleDoc["db"].String()); - -+ ASSERT_TRUE(roleDoc["authenticationRestrictions"].Array().empty()); -+ - std::vector privs = roleDoc["privileges"].Array(); - ASSERT_EQUALS(1U, privs.size()); - ASSERT_EQUALS("", privs[0].Obj()["resource"].Obj()["db"].String()); -@@ -89,7 +98,7 @@ TEST(RoleParsingTest, BuildRoleBSON) { - ASSERT_EQUALS("insert", actionElements[1].String()); - - std::vector roles = roleDoc["roles"].Array(); -- ASSERT_EQUALS(2U, roles.size()); -+ ASSERT_EQUALS(3U, roles.size()); - ASSERT_EQUALS("roleC", roles[0].Obj()["role"].String()); - ASSERT_EQUALS("dbC", roles[0].Obj()["db"].String()); - ASSERT_EQUALS("roleB", roles[1].Obj()["role"].String()); -@@ -104,6 +113,8 @@ TEST(RoleParsingTest, BuildRoleBSON) { - ASSERT_EQUALS("roleB", roleDoc["role"].String()); - ASSERT_EQUALS("dbB", roleDoc["db"].String()); - -+ ASSERT_TRUE(roleDoc["authenticationRestrictions"].Array().empty()); -+ - privs = roleDoc["privileges"].Array(); - ASSERT_EQUALS(1U, privs.size()); - ASSERT_EQUALS("dbB", privs[0].Obj()["resource"].Obj()["db"].String()); -@@ -128,6 +139,8 @@ TEST(RoleParsingTest, BuildRoleBSON) { - ASSERT_EQUALS("roleC", roleDoc["role"].String()); - ASSERT_EQUALS("dbC", roleDoc["db"].String()); - -+ ASSERT_TRUE(roleDoc["authenticationRestrictions"].Array().empty()); -+ - privs = roleDoc["privileges"].Array(); - ASSERT_EQUALS(1U, privs.size()); - ASSERT(privs[0].Obj()["resource"].Obj()["cluster"].Bool()); -@@ -140,6 +153,27 @@ TEST(RoleParsingTest, BuildRoleBSON) { - - roles = roleDoc["roles"].Array(); - ASSERT_EQUALS(0U, roles.size()); -+ -+ // Role D -+ doc.reset(); -+ ASSERT_OK(RoleGraph::getBSONForRole(&graph, roleD, doc.root())); -+ roleDoc = doc.getObject(); -+ -+ ASSERT_EQUALS("dbD.roleD", roleDoc["_id"].String()); -+ ASSERT_EQUALS("roleD", roleDoc["role"].String()); -+ ASSERT_EQUALS("dbD", roleDoc["db"].String()); -+ -+ ASSERT_FALSE(roleDoc["authenticationRestrictions"].Array().empty()); -+ auto restrictionObj = BSONArray(roleDoc["authenticationRestrictions"].Obj()); -+ SharedRestrictionDocument parsedRestrictions = -+ uassertStatusOK(parseAuthenticationRestriction(restrictionObj)); -+ ASSERT_EQ(restrictions->toString(), parsedRestrictions->toString()); -+ -+ privs = roleDoc["privileges"].Array(); -+ ASSERT_TRUE(privs.empty()); -+ -+ roles = roleDoc["roles"].Array(); -+ ASSERT_EQUALS(0U, roles.size()); - } - - // Tests adding and removing roles from other roles, the RoleNameIterator, and the diff --git a/inconsistent-tabs.patch b/inconsistent-tabs.patch deleted file mode 100644 index 2d86825814ef0ee93d2dab21b19da1f24d00e72b..0000000000000000000000000000000000000000 --- a/inconsistent-tabs.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -uprN mongo-r4.0.1_raw/SConstruct mongo-r4.0.1/SConstruct ---- mongo-r4.0.1_raw/SConstruct 2018-07-28 02:14:20.000000000 +0800 -+++ mongo-r4.0.1/SConstruct 2020-01-10 17:33:08.800735290 +0800 -@@ -3320,7 +3320,7 @@ def doConfigure(myenv): - - outputIndex = next((idx for idx in [0,1] if conf.CheckAltivecVbpermqOutput(idx)), None) - if outputIndex is not None: -- conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) -+ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) - else: - myenv.ConfError("Running on ppc64le, but can't find a correct vec_vbpermq output index. Compiler or platform not supported") - diff --git a/mongodb.spec b/mongodb.spec index 27613d0af89745128a1d4e7761e5235be422f138..476dc8f36485a46948e2bedd18e5345852114081 100644 --- a/mongodb.spec +++ b/mongodb.spec @@ -1,17 +1,18 @@ %global __python %{__python3} Name: mongodb -Version: 4.0.1 -Release: 5 +Version: 4.0.23 +Release: 1 Summary: The global cloud database service for modern applications License: AGPLv3 and zlib and ASL 2.0 URL: http://www.mongodb.org Source0: https://github.com/mongodb/mongo/archive/r%{version}.tar.gz Source1: mongod.conf Source2: mongod.service -Patch0000: inconsistent-tabs.patch -Patch0001: python3-buildscripts-tests.patch -Patch6000: CVE-2020-7921.patch +Patch0000: python3-fix.diff +#Patch0000: inconsistent-tabs.patch +#Patch0001: python3-buildscripts-tests.patch +#Patch6000: CVE-2020-7921.patch BuildRequires: gcc-c++ >= 5.3.0 boost-devel >= 1.56 gperftools-devel libpcap-devel libstemmer-devel BuildRequires: openssl-devel pcre-devel python3-scons snappy-devel yaml-cpp-devel zlib-devel systemd BuildRequires: valgrind-devel libcurl-devel python3-devel python3-yaml python3-requests python3-cheetah @@ -118,7 +119,7 @@ if test $1 -ge 1; then fi %files -%doc GNU-AGPL-3.0.txt APACHE-2.0.txt +%doc APACHE-2.0.txt %{_bindir}/{mongo,mongobridge} %files server @@ -139,6 +140,10 @@ fi %{_mandir}/man1/{mongo.1*,mongod.1*,mongos.1*} %changelog +* Wed Mar 17 2021 shinwell_hu - 4.0.23-1 +- Fix CVE-2020-7923 by upgrade to latest stable release in + 4.0.x series + * Fri May 29 2020 panchenbo - 4.0.1-5 - Type:bugfix - ID: NA @@ -151,8 +156,8 @@ fi - SUG: restart - DESC: fix CVE-2020-7921 -* Mon Feb 13 2020 Ling Yang - 4.0.1-3 +* Thu Feb 13 2020 Ling Yang - 4.0.1-3 - Fixed mongodb-test package build errors -* Wed Feb 8 2020 Ling Yang - 4.0.1-2 +* Sat Feb 8 2020 Ling Yang - 4.0.1-2 - Package init diff --git a/python3-buildscripts-tests.patch b/python3-buildscripts-tests.patch deleted file mode 100644 index 17eeeb44dec8e3ba1dde577e8ec647dc06a2a29b..0000000000000000000000000000000000000000 --- a/python3-buildscripts-tests.patch +++ /dev/null @@ -1,1676 +0,0 @@ -diff --git a/SConstruct b/SConstruct -index b76f4876eb..1f6870284c 100644 ---- a/SConstruct -+++ b/SConstruct -@@ -425,7 +425,7 @@ win_version_min_choices = { - } - - add_option('win-version-min', -- choices=win_version_min_choices.keys(), -+ choices=list(win_version_min_choices.keys()), - default=None, - help='minimum Windows version to support', - type='choice', -@@ -551,7 +551,7 @@ except ValueError as e: - def variable_shlex_converter(val): - # If the argument is something other than a string, propogate - # it literally. -- if not isinstance(val, basestring): -+ if not isinstance(val, str): - return val - parse_mode = get_option('variable-parse-mode') - if parse_mode == 'auto': -@@ -887,7 +887,7 @@ SConsignFile(str(sconsDataDir.File('sconsign'))) - def printLocalInfo(): - import sys, SCons - print( "scons version: " + SCons.__version__ ) -- print( "python version: " + " ".join( [ `i` for i in sys.version_info ] ) ) -+ print( "python version: " + " ".join( [ str(i) for i in sys.version_info ] ) ) - - printLocalInfo() - -@@ -2029,7 +2029,7 @@ def doConfigure(myenv): - # to make them real errors. - cloned.Append(CCFLAGS=['-Werror']) - conf = Configure(cloned, help=False, custom_tests = { -- 'CheckFlag' : lambda(ctx) : CheckFlagTest(ctx, tool, extension, flag) -+ 'CheckFlag' : lambda ctx : CheckFlagTest(ctx, tool, extension, flag) - }) - available = conf.CheckFlag() - conf.Finish() -@@ -2503,7 +2503,7 @@ def doConfigure(myenv): - # Select those unique black files that are associated with the - # currently enabled sanitizers, but filter out those that are - # zero length. -- blackfiles = {v for (k, v) in blackfiles_map.iteritems() if k in sanitizer_list} -+ blackfiles = {v for (k, v) in blackfiles_map.items() if k in sanitizer_list} - blackfiles = [f for f in blackfiles if os.stat(f.path).st_size != 0] - - # Filter out any blacklist options that the toolchain doesn't support. -diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py -index af3a53d29f..564f7a4171 100755 ---- a/buildscripts/clang_format.py -+++ b/buildscripts/clang_format.py -@@ -20,7 +20,7 @@ import sys - import tarfile - import tempfile - import threading --import urllib2 -+import urllib.request, urllib.error, urllib.parse - from distutils import spawn # pylint: disable=no-name-in-module - from optparse import OptionParser - from multiprocessing import cpu_count -@@ -96,11 +96,11 @@ def get_clang_format_from_cache_and_extract(url, tarball_ext): - num_tries = 5 - for attempt in range(num_tries): - try: -- resp = urllib2.urlopen(url) -+ resp = urllib.request.urlopen(url) - with open(temp_tar_file, 'wb') as fh: - fh.write(resp.read()) - break -- except urllib2.URLError: -+ except urllib.error.URLError: - if attempt == num_tries - 1: - raise - continue -diff --git a/buildscripts/cpplint.py b/buildscripts/cpplint.py -index 6979cbcd4e..bc9ff038fd 100755 ---- a/buildscripts/cpplint.py -+++ b/buildscripts/cpplint.py -@@ -835,7 +835,7 @@ class _CppLintState(object): - - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" -- for category, count in self.errors_by_category.iteritems(): -+ for category, count in self.errors_by_category.items(): - sys.stderr.write('Category \'%s\' errors found: %d\n' % - (category, count)) - sys.stderr.write('Total errors found: %d\n' % self.error_count) -@@ -1388,7 +1388,7 @@ def FindEndOfExpressionInLine(line, startpos, stack): - On finding an unclosed expression: (-1, None) - Otherwise: (-1, new stack at end of this line) - """ -- for i in xrange(startpos, len(line)): -+ for i in range(startpos, len(line)): - char = line[i] - if char in '([{': - # Found start of parenthesized expression, push to expression stack -@@ -1681,7 +1681,7 @@ def CheckForCopyright(filename, lines, error): - - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. -- for line in xrange(1, min(len(lines), 11)): -+ for line in range(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, -@@ -1832,7 +1832,7 @@ def CheckForBadCharacters(filename, lines, error): - error: The function to call with any errors found. - """ - for linenum, line in enumerate(lines): -- if u'\ufffd' in line: -+ if '\ufffd' in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: -@@ -2878,7 +2878,7 @@ def CheckForFunctionLengths(filename, clean_lines, linenum, - - if starting_func: - body_found = False -- for start_linenum in xrange(linenum, clean_lines.NumLines()): -+ for start_linenum in range(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions -@@ -3355,7 +3355,7 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error): - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] -- for offset in xrange(endlinenum + 1, -+ for offset in range(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): -@@ -3524,7 +3524,7 @@ def IsRValueType(clean_lines, nesting_state, linenum, column): - - # Look for the previous 'for(' in the previous lines. - before_text = match_symbol.group(1) -- for i in xrange(start - 1, max(start - 6, 0), -1): -+ for i in range(start - 1, max(start - 6, 0), -1): - before_text = clean_lines.elided[i] + before_text - if Search(r'for\s*\([^{};]*$', before_text): - # This is the condition inside a for-loop -@@ -3651,12 +3651,12 @@ def IsRValueAllowed(clean_lines, linenum): - True if line is within the region where RValue references are allowed. - """ - # Allow region marked by PUSH/POP macros -- for i in xrange(linenum, 0, -1): -+ for i in range(linenum, 0, -1): - line = clean_lines.elided[i] - if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - if not line.endswith('PUSH'): - return False -- for j in xrange(linenum, clean_lines.NumLines(), 1): -+ for j in range(linenum, clean_lines.NumLines(), 1): - line = clean_lines.elided[j] - if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - return line.endswith('POP') -@@ -4136,7 +4136,7 @@ def CheckCheck(filename, clean_lines, linenum, error): - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] -- for i in xrange(linenum + 1, end_line): -+ for i in range(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - -@@ -4264,7 +4264,7 @@ def GetLineWidth(line): - The width of the line in column positions, accounting for Unicode - combining characters and wide characters. - """ -- if isinstance(line, unicode): -+ if isinstance(line, str): - width = 0 - for uc in unicodedata.normalize('NFC', line): - if unicodedata.east_asian_width(uc) in ('W', 'F'): -@@ -4617,7 +4617,7 @@ def _GetTextInside(text, start_pattern): - - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} -- closing_punctuation = set(matching_punctuation.itervalues()) -+ closing_punctuation = set(matching_punctuation.values()) - - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) -@@ -4943,7 +4943,7 @@ def IsDerivedFunction(clean_lines, linenum): - virt-specifier. - """ - # Scan back a few lines for start of current function -- for i in xrange(linenum, max(-1, linenum - 10), -1): -+ for i in range(linenum, max(-1, linenum - 10), -1): - match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) - if match: - # Look for "override" after the matching closing parenthesis -@@ -4964,7 +4964,7 @@ def IsInitializerList(clean_lines, linenum): - True if current line appears to be inside constructor initializer - list, False otherwise. - """ -- for i in xrange(linenum, 1, -1): -+ for i in range(linenum, 1, -1): - line = clean_lines.elided[i] - if i == linenum: - remove_function_body = Match(r'^(.*)\{\s*$', line) -@@ -5060,7 +5060,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum, - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' -- for i in xrange(startline, linenum + 1): -+ for i in range(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may -@@ -5084,7 +5084,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum, - # appear inside the second set of parentheses on the current line as - # opposed to the first set. - if linenum > 0: -- for i in xrange(linenum - 1, max(0, linenum - 10), -1): -+ for i in range(linenum - 1, max(0, linenum - 10), -1): - previous_line = clean_lines.elided[i] - if not Search(r'[),]\s*$', previous_line): - break -@@ -5115,7 +5115,7 @@ def CheckForNonConstReference(filename, clean_lines, linenum, - # Don't see a whitelisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. -- for i in xrange(2): -+ for i in range(2): - if (linenum > i and - Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): - return -@@ -5277,7 +5277,7 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): - # Try expanding current context to see if we one level of - # parentheses inside a macro. - if linenum > 0: -- for i in xrange(linenum - 1, max(0, linenum - 5), -1): -+ for i in range(linenum - 1, max(0, linenum - 5), -1): - context = clean_lines.elided[i] + context - if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): - return False -@@ -5534,7 +5534,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, - required = {} # A map of header name to linenumber and the template entity. - # Example of required: { '': (1219, 'less<>') } - -- for linenum in xrange(clean_lines.NumLines()): -+ for linenum in range(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue -@@ -5583,7 +5583,7 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, - - # include_dict is modified during iteration, so we iterate over a copy of - # the keys. -- header_keys = include_dict.keys() -+ header_keys = list(include_dict.keys()) - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header -@@ -5678,7 +5678,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error): - end_col = -1 - end_line = -1 - start_col = len(virtual.group(1)) -- for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): -+ for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())): - line = clean_lines.elided[start_line][start_col:] - parameter_list = Match(r'^([^(]*)\(', line) - if parameter_list: -@@ -5693,7 +5693,7 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error): - - # Look for "override" or "final" after the parameter list - # (possibly on the next few lines). -- for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): -+ for i in range(end_line, min(end_line + 3, clean_lines.NumLines())): - line = clean_lines.elided[i][end_col:] - match = Search(r'\b(override|final)\b', line) - if match: -@@ -5920,7 +5920,7 @@ def ProcessFileData(filename, file_extension, lines, error, - - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) -- for line in xrange(clean_lines.NumLines()): -+ for line in range(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) -diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py -index f9162917db..60cdb2b50f 100755 ---- a/buildscripts/errorcodes.py -+++ b/buildscripts/errorcodes.py -@@ -7,12 +7,15 @@ Optionally replaces zero codes in source code with new distinct values. - - from __future__ import absolute_import - from __future__ import print_function -+from __future__ import unicode_literals - -+import io - import bisect - import os.path - import sys - from collections import defaultdict, namedtuple - from optparse import OptionParser -+from functools import reduce - - # Get relative imports to work when the package is not installed on the PYTHONPATH. - if __name__ == "__main__" and __package__ is None: -@@ -56,7 +59,7 @@ def parse_source_files(callback): - if list_files: - print('scanning file: ' + source_file) - -- with open(source_file) as fh: -+ with open(source_file, encoding="utf-8") as fh: - text = fh.read() - - if not any([zz in text for zz in quick]): -diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py -index bb36531164..eb20dda9dd 100755 ---- a/buildscripts/eslint.py -+++ b/buildscripts/eslint.py -@@ -20,7 +20,7 @@ import sys - import tarfile - import tempfile - import threading --import urllib -+import urllib.request, urllib.parse, urllib.error - from distutils import spawn # pylint: disable=no-name-in-module - from optparse import OptionParser - -@@ -84,7 +84,7 @@ def get_eslint_from_cache(dest_file, platform, arch): - - # Download the file - print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) -- urllib.urlretrieve(url, temp_tar_file) -+ urllib.request.urlretrieve(url, temp_tar_file) - - eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) - extract_eslint(temp_tar_file, eslint_distfile) -diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py -index bf8c188151..eb98466d90 100644 ---- a/buildscripts/idl/idl/binder.py -+++ b/buildscripts/idl/idl/binder.py -@@ -727,7 +727,7 @@ def _validate_enum_int(ctxt, idl_enum): - min_value = min(int_values_set) - max_value = max(int_values_set) - -- valid_int = {x for x in xrange(min_value, max_value + 1)} -+ valid_int = {x for x in range(min_value, max_value + 1)} - - if valid_int != int_values_set: - ctxt.add_enum_non_continuous_range_error(idl_enum, idl_enum.name) -diff --git a/buildscripts/idl/idl/bson.py b/buildscripts/idl/idl/bson.py -index 21fb8961f5..10df6ed4c4 100644 ---- a/buildscripts/idl/idl/bson.py -+++ b/buildscripts/idl/idl/bson.py -@@ -87,7 +87,7 @@ def cpp_bson_type_name(name): - def list_valid_types(): - # type: () -> List[unicode] - """Return a list of supported bson types.""" -- return [a for a in _BSON_TYPE_INFORMATION.iterkeys()] -+ return [a for a in _BSON_TYPE_INFORMATION.keys()] - - - def is_valid_bindata_subtype(name): -diff --git a/buildscripts/idl/idl/cpp_types.py b/buildscripts/idl/idl/cpp_types.py -index d275872ca5..81033a251c 100644 ---- a/buildscripts/idl/idl/cpp_types.py -+++ b/buildscripts/idl/idl/cpp_types.py -@@ -28,6 +28,7 @@ from . import writer - - _STD_ARRAY_UINT8_16 = 'std::array' - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - - def is_primitive_scalar_type(cpp_type): - # type: (unicode) -> bool -@@ -75,11 +76,9 @@ def _qualify_array_type(cpp_type): - return "std::vector<%s>" % (cpp_type) - - --class CppTypeBase(object): -+class CppTypeBase(ABC): - """Base type for C++ Type information.""" - -- __metaclass__ = ABCMeta -- - def __init__(self, field): - # type: (ast.Field) -> None - """Construct a CppTypeBase.""" -@@ -521,11 +520,9 @@ def get_cpp_type(field): - return cpp_type_info - - --class BsonCppTypeBase(object): -+class BsonCppTypeBase(ABC): - """Base type for custom C++ support for BSON Types information.""" - -- __metaclass__ = ABCMeta -- - def __init__(self, field): - # type: (ast.Field) -> None - """Construct a BsonCppTypeBase.""" -diff --git a/buildscripts/idl/idl/enum_types.py b/buildscripts/idl/idl/enum_types.py -index 9435136451..727990bef6 100644 ---- a/buildscripts/idl/idl/enum_types.py -+++ b/buildscripts/idl/idl/enum_types.py -@@ -29,11 +29,11 @@ from . import common - from . import syntax - from . import writer - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - --class EnumTypeInfoBase(object): -- """Base type for enumeration type information.""" - -- __metaclass__ = ABCMeta -+class EnumTypeInfoBase(ABC): -+ """Base type for enumeration type information.""" - - def __init__(self, idl_enum): - # type: (Union[syntax.Enum,ast.Enum]) -> None -@@ -115,8 +115,6 @@ class EnumTypeInfoBase(object): - class _EnumTypeInt(EnumTypeInfoBase): - """Type information for integer enumerations.""" - -- __metaclass__ = ABCMeta -- - def get_cpp_type_name(self): - # type: () -> unicode - return common.title_case(self._enum.name) -@@ -190,8 +188,6 @@ def _get_constant_enum_name(idl_enum, enum_value): - class _EnumTypeString(EnumTypeInfoBase): - """Type information for string enumerations.""" - -- __metaclass__ = ABCMeta -- - def get_cpp_type_name(self): - # type: () -> unicode - return common.template_args("${enum_name}Enum", enum_name=common.title_case( -diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py -index 82c22b45b8..4e099e9ffc 100644 ---- a/buildscripts/idl/idl/generator.py -+++ b/buildscripts/idl/idl/generator.py -@@ -33,6 +33,7 @@ from . import enum_types - from . import struct_types - from . import writer - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - - def _get_field_member_name(field): - # type: (ast.Field) -> unicode -@@ -122,11 +123,9 @@ def _get_all_fields(struct): - return sorted([field for field in all_fields], key=lambda f: f.cpp_name) - - --class _FieldUsageCheckerBase(object): -+class _FieldUsageCheckerBase(ABC): - """Check for duplicate fields, and required fields as needed.""" - -- __metaclass__ = ABCMeta -- - def __init__(self, indented_writer): - # type: (writer.IndentedTextWriter) -> None - """Create a field usage checker.""" -@@ -1588,8 +1587,8 @@ def _generate_header(spec, file_name): - str_value = generate_header_str(spec) - - # Generate structs -- with io.open(file_name, mode='wb') as file_handle: -- file_handle.write(str_value.encode()) -+ with io.open(file_name, mode='w') as file_handle: -+ file_handle.write(str_value) - - - def generate_source_str(spec, target_arch, header_file_name): -@@ -1611,8 +1610,8 @@ def _generate_source(spec, target_arch, file_name, header_file_name): - str_value = generate_source_str(spec, target_arch, header_file_name) - - # Generate structs -- with io.open(file_name, mode='wb') as file_handle: -- file_handle.write(str_value.encode()) -+ with io.open(file_name, mode='w') as file_handle: -+ file_handle.write(str_value) - - - def generate_code(spec, target_arch, output_base_dir, header_file_name, source_file_name): -diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py -index 052f9a2178..4cf09295c4 100644 ---- a/buildscripts/idl/idl/parser.py -+++ b/buildscripts/idl/idl/parser.py -@@ -31,6 +31,7 @@ from . import cpp_types - from . import errors - from . import syntax - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - - class _RuleDesc(object): - """ -@@ -548,11 +549,9 @@ def _parse(stream, error_file_name): - return syntax.IDLParsedSpec(spec, None) - - --class ImportResolverBase(object): -+class ImportResolverBase(ABC): - """Base class for resolving imported files.""" - -- __metaclass__ = ABCMeta -- - def __init__(self): - # type: () -> None - """Construct a ImportResolver.""" -diff --git a/buildscripts/idl/idl/struct_types.py b/buildscripts/idl/idl/struct_types.py -index 8e055fe8c4..b57542bb44 100644 ---- a/buildscripts/idl/idl/struct_types.py -+++ b/buildscripts/idl/idl/struct_types.py -@@ -24,6 +24,7 @@ from . import common - from . import cpp_types - from . import writer - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - - class ArgumentInfo(object): - """Class that encapsulates information about an argument to a method.""" -@@ -114,11 +115,9 @@ class MethodInfo(object): - args=args) - - --class StructTypeInfoBase(object): -+class StructTypeInfoBase(ABC): - """Base class for struct and command code generation.""" - -- __metaclass__ = ABCMeta -- - @abstractmethod - def get_constructor_method(self): - # type: () -> MethodInfo -diff --git a/buildscripts/idl/idl/syntax.py b/buildscripts/idl/idl/syntax.py -index 049114b5d9..76643e68a1 100644 ---- a/buildscripts/idl/idl/syntax.py -+++ b/buildscripts/idl/idl/syntax.py -@@ -82,7 +82,7 @@ def _item_and_type(dic): - # type: (Dict[Any, List[Any]]) -> Iterator[Tuple[Any, Any]] - """Return an Iterator of (key, value) pairs from a dictionary.""" - return itertools.chain.from_iterable( -- (_zip_scalar(value, key) for (key, value) in dic.viewitems())) -+ (_zip_scalar(value, key) for (key, value) in dic.items())) - - - class SymbolTable(object): -diff --git a/buildscripts/idl/tests/test_binder.py b/buildscripts/idl/tests/test_binder.py -index d8b6fc1630..1dd5afde4d 100644 ---- a/buildscripts/idl/tests/test_binder.py -+++ b/buildscripts/idl/tests/test_binder.py -@@ -74,7 +74,7 @@ class TestBinder(testcase.IDLTestcase): - cpp_includes: - - 'bar' - - 'foo'""")) -- self.assertEquals(spec.globals.cpp_namespace, "something") -+ self.assertEqual(spec.globals.cpp_namespace, "something") - self.assertListEqual(spec.globals.cpp_includes, ['bar', 'foo']) - - def test_type_positive(self): -diff --git a/buildscripts/linter/base.py b/buildscripts/linter/base.py -index f22f59e4f0..09931a673b 100644 ---- a/buildscripts/linter/base.py -+++ b/buildscripts/linter/base.py -@@ -5,12 +5,11 @@ from __future__ import print_function - from abc import ABCMeta, abstractmethod - from typing import Dict, List, Optional - -+ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) - --class LinterBase(object): -+class LinterBase(ABC): - """Base Class for all linters.""" - -- __metaclass__ = ABCMeta -- - def __init__(self, cmd_name, required_version, cmd_location=None): - # type: (str, str, Optional[str]) -> None - """ -diff --git a/buildscripts/linter/git.py b/buildscripts/linter/git.py -index b4a6898604..d803e1b584 100644 ---- a/buildscripts/linter/git.py -+++ b/buildscripts/linter/git.py -@@ -175,7 +175,7 @@ def get_files_to_check_from_patch(patches, filter_function): - - lines = [] # type: List[str] - for patch in patches: -- with open(patch, "rb") as infile: -+ with open(patch, "r") as infile: - lines += infile.readlines() - - candidates = [check.match(line).group(1) for line in lines if check.match(line)] -diff --git a/buildscripts/linter/parallel.py b/buildscripts/linter/parallel.py -index 0648bfb16e..361da0c559 100644 ---- a/buildscripts/linter/parallel.py -+++ b/buildscripts/linter/parallel.py -@@ -2,7 +2,12 @@ - from __future__ import absolute_import - from __future__ import print_function - --import Queue -+try: -+ import queue -+except ImportError: -+ #Python 2 -+ import Queue as queue -+ - import threading - import time - from multiprocessing import cpu_count -@@ -17,7 +22,7 @@ def parallel_process(items, func): - except NotImplementedError: - cpus = 1 - -- task_queue = Queue.Queue() # type: Queue.Queue -+ task_queue = queue.Queue() # type: queue.Queue - - # Use a list so that worker function will capture this variable - pp_event = threading.Event() -@@ -30,7 +35,7 @@ def parallel_process(items, func): - while not pp_event.is_set(): - try: - item = task_queue.get_nowait() -- except Queue.Empty: -+ except queue.Empty: - # if the queue is empty, exit the worker thread - pp_event.set() - return -diff --git a/buildscripts/resmokeconfig/loggers/__init__.py b/buildscripts/resmokeconfig/loggers/__init__.py -index 5342639c56..7444783cb6 100644 ---- a/buildscripts/resmokeconfig/loggers/__init__.py -+++ b/buildscripts/resmokeconfig/loggers/__init__.py -@@ -16,7 +16,7 @@ def _get_named_loggers(): - named_loggers = {} - - try: -- (root, _dirs, files) = os.walk(dirname).next() -+ (root, _dirs, files) = next(os.walk(dirname)) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): -diff --git a/buildscripts/resmokeconfig/suites/__init__.py b/buildscripts/resmokeconfig/suites/__init__.py -index 87d378616b..4321f53658 100644 ---- a/buildscripts/resmokeconfig/suites/__init__.py -+++ b/buildscripts/resmokeconfig/suites/__init__.py -@@ -16,7 +16,7 @@ def _get_named_suites(): - named_suites = {} - - try: -- (root, _dirs, files) = os.walk(dirname).next() -+ (root, _dirs, files) = next(os.walk(dirname)) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): -diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py -index 66753c389d..4fe50a8176 100644 ---- a/buildscripts/resmokelib/config.py -+++ b/buildscripts/resmokelib/config.py -@@ -62,7 +62,7 @@ DEFAULTS = { - "repeat": 1, - "report_failure_status": "fail", - "report_file": None, -- "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. -+ "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. - "service_executor": None, - "shell_conn_string": None, - "shell_port": None, -diff --git a/buildscripts/resmokelib/core/process.py b/buildscripts/resmokelib/core/process.py -index 84c067d8e3..956c4f9e42 100644 ---- a/buildscripts/resmokelib/core/process.py -+++ b/buildscripts/resmokelib/core/process.py -@@ -182,8 +182,8 @@ class Process(object): - finally: - win32api.CloseHandle(mongo_signal_handle) - -- print "Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ -- str(self._process.pid) -+ print("Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ -+ str(self._process.pid)) - - # Adapted from implementation of Popen.terminate() in subprocess.py of Python 2.7 - # because earlier versions do not catch exceptions. -diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py -index 2e48101d51..7d44980ad8 100644 ---- a/buildscripts/resmokelib/logging/buildlogger.py -+++ b/buildscripts/resmokelib/logging/buildlogger.py -@@ -261,7 +261,7 @@ class BuildloggerServer(object): - """Initialize BuildloggerServer.""" - tmp_globals = {} - self.config = {} -- execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config) -+ exec(compile(open(_BUILDLOGGER_CONFIG).read(), _BUILDLOGGER_CONFIG, 'exec'), tmp_globals, self.config) - - # Rename "slavename" to "username" if present. - if "slavename" in self.config and "username" not in self.config: -diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py -index d878710f1d..5bf1a0881d 100644 ---- a/buildscripts/resmokelib/selector.py -+++ b/buildscripts/resmokelib/selector.py -@@ -71,7 +71,7 @@ class TestFileExplorer(object): - A list of paths as a list(str). - """ - tests = [] -- with open(root_file_path, "rb") as filep: -+ with open(root_file_path, "r") as filep: - for test_path in filep: - test_path = test_path.strip() - tests.append(test_path) -@@ -310,7 +310,7 @@ def make_expression(conf): - elif isinstance(conf, dict): - if len(conf) != 1: - raise ValueError("Tag matching expressions should only contain one key") -- key = conf.keys()[0] -+ key = next(iter(conf.keys())) - value = conf[key] - if key == "$allOf": - return _AllOfExpression(_make_expression_list(value)) -diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py -index 79ccb17786..fbd0a71919 100644 ---- a/buildscripts/resmokelib/testing/executor.py -+++ b/buildscripts/resmokelib/testing/executor.py -@@ -62,7 +62,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes - jobs_to_start = self.num_tests - - # Must be done after getting buildlogger configuration. -- self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)] -+ self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)] - - def run(self): - """Execute the test suite. -@@ -275,7 +275,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes - queue.put(test_case) - - # Add sentinel value for each job to indicate when there are no more items to process. -- for _ in xrange(len(self._jobs)): -+ for _ in range(len(self._jobs)): - queue.put(None) - - return queue -diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py -index 9b4e69c112..3927ed85af 100644 ---- a/buildscripts/resmokelib/testing/fixtures/interface.py -+++ b/buildscripts/resmokelib/testing/fixtures/interface.py -@@ -3,6 +3,7 @@ - from __future__ import absolute_import - - import os.path -+import six - import time - - import pymongo -@@ -25,10 +26,10 @@ def make_fixture(class_name, *args, **kwargs): - return _FIXTURES[class_name](*args, **kwargs) - - --class Fixture(object): -- """Base class for all fixtures.""" -- -- __metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore -+class Fixture(six.with_metaclass(registry.make_registry_metaclass(_FIXTURES), object)): -+ """ -+ Base class for all fixtures. -+ """ - - # We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute - # is defined for all subclasses of Fixture. -diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py -index 2cf58d9fc9..166ecc13fc 100644 ---- a/buildscripts/resmokelib/testing/fixtures/replicaset.py -+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py -@@ -77,11 +77,11 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst - self.replset_name = self.mongod_options.get("replSet", "rs") - - if not self.nodes: -- for i in xrange(self.num_nodes): -+ for i in range(self.num_nodes): - node = self._new_mongod(i, self.replset_name) - self.nodes.append(node) - -- for i in xrange(self.num_nodes): -+ for i in range(self.num_nodes): - if self.linear_chain and i > 0: - self.nodes[i].mongod_options["set_parameters"][ - "failpoint.forceSyncSourceCandidate"] = { -diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py -index 70c1eaa432..b8345c38b4 100644 ---- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py -+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py -@@ -66,7 +66,7 @@ class ShardedClusterFixture(interface.Fixture): # pylint: disable=too-many-inst - self.configsvr.setup() - - if not self.shards: -- for i in xrange(self.num_shards): -+ for i in range(self.num_shards): - if self.num_rs_nodes_per_shard is None: - shard = self._new_standalone_shard(i) - elif isinstance(self.num_rs_nodes_per_shard, int): -diff --git a/buildscripts/resmokelib/testing/hooks/interface.py b/buildscripts/resmokelib/testing/hooks/interface.py -index 0c2f65077b..88baeeea86 100644 ---- a/buildscripts/resmokelib/testing/hooks/interface.py -+++ b/buildscripts/resmokelib/testing/hooks/interface.py -@@ -9,6 +9,8 @@ from ... import errors - from ...logging import loggers - from ...utils import registry - -+import six -+ - _HOOKS = {} # type: ignore - - -@@ -21,11 +23,8 @@ def make_hook(class_name, *args, **kwargs): - return _HOOKS[class_name](*args, **kwargs) - - --class Hook(object): -+class Hook(six.with_metaclass(registry.make_registry_metaclass(_HOOKS), object)): - """Common interface all Hooks will inherit from.""" -- -- __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore -- - REGISTERED_NAME = registry.LEAVE_UNREGISTERED - - def __init__(self, hook_logger, fixture, description): -diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py -index 1a57b6c771..c216ef8118 100644 ---- a/buildscripts/resmokelib/testing/suite.py -+++ b/buildscripts/resmokelib/testing/suite.py -@@ -234,7 +234,7 @@ class Suite(object): # pylint: disable=too-many-instance-attributes - sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken)) - - combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) -- for iteration in xrange(num_iterations): -+ for iteration in range(num_iterations): - # Summarize each execution as a bulleted list of results. - bulleter_sb = [] - summary = self._summarize_report(reports[iteration], start_times[iteration], -diff --git a/buildscripts/resmokelib/testing/summary.py b/buildscripts/resmokelib/testing/summary.py -index dc92e0b5b3..5b49fbfd04 100644 ---- a/buildscripts/resmokelib/testing/summary.py -+++ b/buildscripts/resmokelib/testing/summary.py -@@ -12,6 +12,6 @@ Summary = collections.namedtuple( - def combine(summary1, summary2): - """Return a summary representing the sum of 'summary1' and 'summary2'.""" - args = [] -- for i in xrange(len(Summary._fields)): -+ for i in range(len(Summary._fields)): - args.append(summary1[i] + summary2[i]) - return Summary._make(args) -diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py -index 183e69f9d3..4642547c53 100644 ---- a/buildscripts/resmokelib/testing/testcases/interface.py -+++ b/buildscripts/resmokelib/testing/testcases/interface.py -@@ -7,6 +7,7 @@ from __future__ import absolute_import - - import os - import os.path -+import six - import unittest - - from ... import logging -@@ -22,11 +23,8 @@ def make_test_case(test_kind, *args, **kwargs): - return _TEST_CASES[test_kind](*args, **kwargs) - - --class TestCase(unittest.TestCase): -+class TestCase(six.with_metaclass(registry.make_registry_metaclass(_TEST_CASES), unittest.TestCase)): - """A test case to execute.""" -- -- __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore -- - REGISTERED_NAME = registry.LEAVE_UNREGISTERED - - def __init__(self, logger, test_kind, test_name): -@@ -36,10 +34,10 @@ class TestCase(unittest.TestCase): - if not isinstance(logger, logging.Logger): - raise TypeError("logger must be a Logger instance") - -- if not isinstance(test_kind, basestring): -+ if not isinstance(test_kind, str): - raise TypeError("test_kind must be a string") - -- if not isinstance(test_name, basestring): -+ if not isinstance(test_name, str): - raise TypeError("test_name must be a string") - - # When the TestCase is created by the TestSuiteExecutor (through a call to make_test_case()) -diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py -index 3cb4ee0d50..15469ba50e 100644 ---- a/buildscripts/resmokelib/testing/testcases/jstest.py -+++ b/buildscripts/resmokelib/testing/testcases/jstest.py -@@ -199,7 +199,7 @@ class JSTestCase(interface.ProcessTestCase): - test_cases = [] - try: - # If there are multiple clients, make a new thread for each client. -- for thread_id in xrange(self.num_clients): -+ for thread_id in range(self.num_clients): - logger = self.logger.new_test_thread_logger(self.test_kind, str(thread_id)) - test_case = self._create_test_case_for_thread(logger, thread_id) - test_cases.append(test_case) -diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py -index 6b6a76d1f4..6bef14b5a0 100644 ---- a/buildscripts/resmokelib/utils/__init__.py -+++ b/buildscripts/resmokelib/utils/__init__.py -@@ -48,10 +48,10 @@ def rmtree(path, **kwargs): - See https://github.com/pypa/setuptools/issues/706. - """ - if is_windows(): -- if not isinstance(path, unicode): -- path = unicode(path, "utf-8") -+ if not isinstance(path, str): -+ path = str(path, "utf-8") - else: -- if isinstance(path, unicode): -+ if isinstance(path, str): - path = path.encode("utf-8") - shutil.rmtree(path, **kwargs) - -@@ -72,12 +72,12 @@ def remove_if_exists(path): - - def is_string_list(lst): - """Return true if 'lst' is a list of strings, and false otherwise.""" -- return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst) -+ return isinstance(lst, list) and all(isinstance(x, str) for x in lst) - - - def is_string_set(value): - """Return true if 'value' is a set of strings, and false otherwise.""" -- return isinstance(value, set) and all(isinstance(x, basestring) for x in value) -+ return isinstance(value, set) and all(isinstance(x, str) for x in value) - - - def is_js_file(filename): -diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py -index 8ccb3127f4..c8eecdcb70 100644 ---- a/buildscripts/resmokelib/utils/archival.py -+++ b/buildscripts/resmokelib/utils/archival.py -@@ -1,8 +1,13 @@ - """Archival utility.""" - --from __future__ import absolute_import - --import Queue -+ -+try: -+ import queue -+except ImportError: -+ #Python 2 -+ import Queue as queue -+ - import collections - import json - import math -@@ -45,7 +50,7 @@ def file_list_size(files): - def directory_size(directory): - """Return size (in bytes) of files in 'directory' tree.""" - dir_bytes = 0 -- for root_dir, _, files in os.walk(unicode(directory)): -+ for root_dir, _, files in os.walk(str(directory)): - for name in files: - full_name = os.path.join(root_dir, name) - try: -@@ -103,7 +108,7 @@ class Archival(object): # pylint: disable=too-many-instance-attributes - self._lock = threading.Lock() - - # Start the worker thread to update the 'archival_json_file'. -- self._archive_file_queue = Queue.Queue() -+ self._archive_file_queue = queue.Queue() - self._archive_file_worker = threading.Thread(target=self._update_archive_file_wkr, - args=(self._archive_file_queue, - logger), name="archive_file_worker") -@@ -115,7 +120,7 @@ class Archival(object): # pylint: disable=too-many-instance-attributes - self.s3_client = s3_client - - # Start the worker thread which uploads the archive. -- self._upload_queue = Queue.Queue() -+ self._upload_queue = queue.Queue() - self._upload_worker = threading.Thread(target=self._upload_to_s3_wkr, - args=(self._upload_queue, self._archive_file_queue, - logger, self.s3_client), name="upload_worker") -diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py -index 1e016875f9..d57cb41f41 100644 ---- a/buildscripts/resmokelib/utils/globstar.py -+++ b/buildscripts/resmokelib/utils/globstar.py -@@ -134,7 +134,7 @@ def _list_dir(pathname): - """ - - try: -- (_root, dirs, files) = os.walk(pathname).next() -+ (_root, dirs, files) = next(os.walk(pathname)) - return (dirs, files) - except StopIteration: - return None # 'pathname' directory does not exist -diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py -index 67758197c5..f4c4a4d6c5 100644 ---- a/buildscripts/resmokelib/utils/jscomment.py -+++ b/buildscripts/resmokelib/utils/jscomment.py -@@ -36,7 +36,7 @@ def get_tags(pathname): - # TODO: it might be worth supporting the block (indented) style of YAML lists in - # addition to the flow (bracketed) style - tags = yaml.safe_load(_strip_jscomments(match.group(1))) -- if not isinstance(tags, list) and all(isinstance(tag, basestring) for tag in tags): -+ if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): - raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) - return tags - except yaml.YAMLError as err: -diff --git a/buildscripts/resmokelib/utils/queue.py b/buildscripts/resmokelib/utils/queue.py -index c77692138b..57a635f45e 100644 ---- a/buildscripts/resmokelib/utils/queue.py -+++ b/buildscripts/resmokelib/utils/queue.py -@@ -8,7 +8,12 @@ See https://bugs.python.org/issue1167930 for more details. - - from __future__ import absolute_import - --import Queue as _Queue -+try: -+ import queue as _Queue -+except ImportError: -+ #Python 2 -+ import Queue as _Queue -+ - import time - - # Exception that is raised when get_nowait() is called on an empty Queue. -diff --git a/buildscripts/utils.py b/buildscripts/utils.py -index 5073b26ad8..0ac19aaba1 100644 ---- a/buildscripts/utils.py -+++ b/buildscripts/utils.py -@@ -139,8 +139,8 @@ def find_python(min_version=(2, 5)): - # In case the version of Python is somehow missing sys.version_info or sys.executable. - pass - -- version = re.compile(r"[Pp]ython ([\d\.]+)", re.MULTILINE) -- binaries = ("python27", "python2.7", "python26", "python2.6", "python25", "python2.5", "python") -+ version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE) -+ binaries = ('python3', 'python27', 'python2.7', 'python26', 'python2.6', 'python25', 'python2.5', 'python') - for binary in binaries: - try: - out, err = subprocess.Popen([binary, "-V"], stdout=subprocess.PIPE, -@@ -166,7 +166,7 @@ def replace_with_repr(unicode_error): - # repr() of the offending bytes into the decoded string - # at the position they occurred - offender = unicode_error.object[unicode_error.start:unicode_error.end] -- return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end) -+ return (str(repr(offender).strip("'").strip('"')), unicode_error.end) - - - codecs.register_error("repr", replace_with_repr) -diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py -index f002c4f067..3447e5fef3 100644 ---- a/site_scons/libdeps.py -+++ b/site_scons/libdeps.py -@@ -122,7 +122,7 @@ def __get_libdeps(node): - marked.add(n.target_node) - tsorted.append(n.target_node) - -- except DependencyCycleError, e: -+ except DependencyCycleError as e: - if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]: - e.cycle_nodes.insert(0, n.target_node) - raise -@@ -150,7 +150,7 @@ def __get_syslibdeps(node): - for lib in __get_libdeps(node): - for syslib in node.get_env().Flatten(lib.get_env().get(syslibdeps_env_var, [])): - if syslib: -- if type(syslib) in (str, unicode) and syslib.startswith(missing_syslibdep): -+ if type(syslib) in (str, str) and syslib.startswith(missing_syslibdep): - print("Target '%s' depends on the availability of a " - "system provided library for '%s', " - "but no suitable library was found during configuration." % -@@ -209,7 +209,7 @@ def get_syslibdeps(source, target, env, for_signature): - # they're believed to represent library short names, that should be prefixed with -l - # or the compiler-specific equivalent. I.e., 'm' becomes '-lm', but 'File("m.a") is passed - # through whole cloth. -- if type(d) in (str, unicode): -+ if type(d) in (str, str): - result.append('%s%s%s' % (lib_link_prefix, d, lib_link_suffix)) - else: - result.append(d) -diff --git a/site_scons/mongo/__init__.py b/site_scons/mongo/__init__.py -index 510bd7bcc2..f77478092b 100644 ---- a/site_scons/mongo/__init__.py -+++ b/site_scons/mongo/__init__.py -@@ -5,4 +5,4 @@ - def print_build_failures(): - from SCons.Script import GetBuildFailures - for bf in GetBuildFailures(): -- print "%s failed: %s" % (bf.node, bf.errstr) -+ print("%s failed: %s" % (bf.node, bf.errstr)) -diff --git a/site_scons/mongo/generators.py b/site_scons/mongo/generators.py -index c07e86a4d1..5958e6923b 100644 ---- a/site_scons/mongo/generators.py -+++ b/site_scons/mongo/generators.py -@@ -1,6 +1,6 @@ - # -*- mode: python; -*- - --import md5 -+import hashlib - - # Default and alternative generator definitions go here. - -@@ -44,7 +44,7 @@ def default_variant_dir_generator(target, source, env, for_signature): - - # Hash the named options and their values, and take the first 8 characters of the hash as - # the variant name -- hasher = md5.md5() -+ hasher = hashlib.md5() - for option in variant_options: - hasher.update(option) - hasher.update(str(env.GetOption(option))) -diff --git a/site_scons/site_tools/dagger/__init__.py b/site_scons/site_tools/dagger/__init__.py -index f05228cfe4..f10b4027e1 100644 ---- a/site_scons/site_tools/dagger/__init__.py -+++ b/site_scons/site_tools/dagger/__init__.py -@@ -5,7 +5,7 @@ import logging - - import SCons - --import dagger -+from . import dagger - - def generate(env, **kwargs): - """The entry point for our tool. However, the builder for -diff --git a/site_scons/site_tools/dagger/dagger.py b/site_scons/site_tools/dagger/dagger.py -index bace834783..8c55937ce8 100644 ---- a/site_scons/site_tools/dagger/dagger.py -+++ b/site_scons/site_tools/dagger/dagger.py -@@ -40,8 +40,8 @@ import sys - - import SCons - --import graph --import graph_consts -+from . import graph -+from . import graph_consts - - - LIB_DB = [] # Stores every SCons library nodes -@@ -240,7 +240,7 @@ def write_obj_db(target, source, env): - for obj in OBJ_DB: - __generate_file_rels(obj, g) - -- for exe in EXE_DB.keys(): -+ for exe in list(EXE_DB.keys()): - __generate_exe_rels(exe, g) - - # target is given as a list of target SCons nodes - this builder is only responsible for -diff --git a/site_scons/site_tools/dagger/graph.py b/site_scons/site_tools/dagger/graph.py -index 5ebe6f4506..379d5245e6 100644 ---- a/site_scons/site_tools/dagger/graph.py -+++ b/site_scons/site_tools/dagger/graph.py -@@ -4,11 +4,13 @@ import abc - import json - import copy - --import graph_consts -+from . import graph_consts - - if sys.version_info >= (3, 0): - basestring = str - -+ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) -+ - class Graph(object): - """Graph class for storing the build dependency graph. The graph stores the - directed edges as a nested dict of { RelationshipType: {From_Node: Set of -@@ -141,7 +143,7 @@ class Graph(object): - node_dict["id"] = id - node_dict["node"] = {} - -- for property, value in vars(node).iteritems(): -+ for property, value in vars(node).items(): - if isinstance(value, set): - node_dict["node"][property] = list(value) - else: -@@ -170,10 +172,9 @@ class Graph(object): - sum(len(x) for x in self._edges.values()), hash(self)) - - --class NodeInterface(object): -+class NodeInterface(ABC): - """Abstract base class for all Node Objects - All nodes must have an id and name - """ -- __metaclass__ = abc.ABCMeta - - @abc.abstractproperty - def id(self): -@@ -190,7 +191,7 @@ class NodeLib(NodeInterface): - def __init__(self, id, name, input=None): - if isinstance(input, dict): - should_fail = False -- for k, v in input.iteritems(): -+ for k, v in input.items(): - try: - if isinstance(v, list): - setattr(self, k, set(v)) -@@ -310,7 +311,7 @@ class NodeSymbol(NodeInterface): - if isinstance(input, dict): - should_fail = False - -- for k, v in input.iteritems(): -+ for k, v in input.items(): - try: - if isinstance(v, list): - setattr(self, k, set(v)) -@@ -435,7 +436,7 @@ class NodeFile(NodeInterface): - def __init__(self, id, name, input=None): - if isinstance(input, dict): - should_fail = False -- for k, v in input.iteritems(): -+ for k, v in input.items(): - try: - if isinstance(v, list): - setattr(self, k, set(v)) -@@ -551,7 +552,7 @@ class NodeExe(NodeInterface): - def __init__(self, id, name, input=None): - if isinstance(input, dict): - should_fail = False -- for k, v in input.iteritems(): -+ for k, v in input.items(): - try: - if isinstance(v, list): - setattr(self, k, set(v)) -diff --git a/site_scons/site_tools/dagger/graph_consts.py b/site_scons/site_tools/dagger/graph_consts.py -index 81fe86d75c..a922a4f3f6 100644 ---- a/site_scons/site_tools/dagger/graph_consts.py -+++ b/site_scons/site_tools/dagger/graph_consts.py -@@ -17,8 +17,8 @@ NODE_SYM = 2 - NODE_FILE = 3 - NODE_EXE = 4 - --RELATIONSHIP_TYPES = range(1, 9) --NODE_TYPES = range(1, 5) -+RELATIONSHIP_TYPES = list(range(1, 9)) -+NODE_TYPES = list(range(1, 5)) - - - """Error/query codes""" -diff --git a/site_scons/site_tools/dagger/graph_test.py b/site_scons/site_tools/dagger/graph_test.py -index bc84f5868c..6c0168cf97 100644 ---- a/site_scons/site_tools/dagger/graph_test.py -+++ b/site_scons/site_tools/dagger/graph_test.py -@@ -5,8 +5,8 @@ from JSON - - import json - import unittest --import graph --import graph_consts -+from . import graph -+from . import graph_consts - - - def generate_graph(): -@@ -122,15 +122,15 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): - node = graph.NodeLib("test_node", "test_node") - self.g._nodes = {"test_node": node} - -- self.assertEquals(self.g.get_node("test_node"), node) -+ self.assertEqual(self.g.get_node("test_node"), node) - -- self.assertEquals(self.g.get_node("missing_node"), None) -+ self.assertEqual(self.g.get_node("missing_node"), None) - - def test_add_node(self): - node = graph.NodeLib("test_node", "test_node") - self.g.add_node(node) - -- self.assertEquals(self.g.get_node("test_node"), node) -+ self.assertEqual(self.g.get_node("test_node"), node) - - self.assertRaises(ValueError, self.g.add_node, node) - -@@ -153,16 +153,16 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): - self.g.add_edge(graph_consts.LIB_FIL, self.from_node_lib.id, - self.to_node_file.id) - -- self.assertEquals(self.g.edges[graph_consts.LIB_LIB][ -+ self.assertEqual(self.g.edges[graph_consts.LIB_LIB][ - self.from_node_lib.id], set([self.to_node_lib.id])) - -- self.assertEquals(self.g.edges[graph_consts.LIB_SYM][ -+ self.assertEqual(self.g.edges[graph_consts.LIB_SYM][ - self.from_node_lib.id], set([self.to_node_sym.id])) - -- self.assertEquals(self.g.edges[graph_consts.LIB_FIL][ -+ self.assertEqual(self.g.edges[graph_consts.LIB_FIL][ - self.from_node_lib.id], set([self.to_node_file.id])) - -- self.assertEquals(self.to_node_lib.dependent_libs, -+ self.assertEqual(self.to_node_lib.dependent_libs, - set([self.from_node_lib.id])) - - def test_add_edge_files(self): -@@ -173,14 +173,14 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): - self.g.add_edge(graph_consts.FIL_LIB, self.from_node_file.id, - self.to_node_lib.id) - -- self.assertEquals(self.g.edges[graph_consts.FIL_FIL][ -+ self.assertEqual(self.g.edges[graph_consts.FIL_FIL][ - self.from_node_file.id], set([self.to_node_file.id])) -- self.assertEquals(self.g.edges[graph_consts.FIL_SYM][ -+ self.assertEqual(self.g.edges[graph_consts.FIL_SYM][ - self.from_node_file.id], set([self.to_node_sym.id])) -- self.assertEquals(self.g.edges[graph_consts.FIL_LIB][ -+ self.assertEqual(self.g.edges[graph_consts.FIL_LIB][ - self.from_node_file.id], set([self.to_node_lib.id])) - -- self.assertEquals(self.to_node_file.dependent_files, -+ self.assertEqual(self.to_node_file.dependent_files, - set([self.from_node_file.id])) - - def test_export_to_json(self): -@@ -188,7 +188,7 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): - generated_graph.export_to_json("export_test.json") - generated = open("export_test.json", "r") - correct = open("test_graph.json", "r") -- self.assertEquals(json.load(generated), json.load(correct)) -+ self.assertEqual(json.load(generated), json.load(correct)) - generated.close() - correct.close() - -@@ -205,7 +205,7 @@ class TestGraphMethods(unittest.TestCase, CustomAssertions): - self.assertNodeEquals( - graph_fromJSON.get_node(id), correct_graph.get_node(id)) - -- self.assertEquals(graph_fromJSON.edges, correct_graph.edges) -+ self.assertEqual(graph_fromJSON.edges, correct_graph.edges) - - - if __name__ == '__main__': -diff --git a/site_scons/site_tools/distsrc.py b/site_scons/site_tools/distsrc.py -index 861f5d9e2e..d2dff0b612 100644 ---- a/site_scons/site_tools/distsrc.py -+++ b/site_scons/site_tools/distsrc.py -@@ -20,7 +20,7 @@ import shutil - import tarfile - import time - import zipfile --import StringIO -+import io - - from distutils.spawn import find_executable - -@@ -82,7 +82,7 @@ class DistSrcTarArchive(DistSrcArchive): - - def append_file_contents(self, filename, file_contents, - mtime=time.time(), -- mode=0644, -+ mode=0o644, - uname="root", - gname="root"): - file_metadata = tarfile.TarInfo(name=filename) -@@ -91,7 +91,7 @@ class DistSrcTarArchive(DistSrcArchive): - file_metadata.uname = uname - file_metadata.gname = gname - file_metadata.size = len(file_contents) -- file_buf = StringIO.StringIO(file_contents) -+ file_buf = io.StringIO(file_contents) - if self.archive_mode == 'r': - self.archive_file.close() - self.archive_file = tarfile.open( -@@ -119,7 +119,7 @@ class DistSrcZipArchive(DistSrcArchive): - name=key, - size=item_data.file_size, - mtime=time.mktime(fixed_time), -- mode=0775 if is_dir else 0664, -+ mode=0o775 if is_dir else 0o664, - type=tarfile.DIRTYPE if is_dir else tarfile.REGTYPE, - uid=0, - gid=0, -@@ -129,7 +129,7 @@ class DistSrcZipArchive(DistSrcArchive): - - def append_file_contents(self, filename, file_contents, - mtime=time.time(), -- mode=0644, -+ mode=0o644, - uname="root", - gname="root"): - self.archive_file.writestr(filename, file_contents) -@@ -139,7 +139,7 @@ class DistSrcZipArchive(DistSrcArchive): - - def build_error_action(msg): - def error_stub(target=None, source=None, env=None): -- print msg -+ print(msg) - env.Exit(1) - return [ error_stub ] - -@@ -162,7 +162,7 @@ def distsrc_action_generator(source, target, env, for_signature): - - target_ext = str(target[0])[-3:] - if not target_ext in [ 'zip', 'tar' ]: -- print "Invalid file format for distsrc. Must be tar or zip file" -+ print("Invalid file format for distsrc. Must be tar or zip file") - env.Exit(1) - - git_cmd = "\"%s\" archive --format %s --output %s --prefix ${MONGO_DIST_SRC_PREFIX} HEAD" % ( -diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py -index 9838b63349..fdf0c26030 100644 ---- a/site_scons/site_tools/icecream.py -+++ b/site_scons/site_tools/icecream.py -@@ -99,7 +99,7 @@ def generate(env): - suffixes = _CSuffixes + _CXXSuffixes - for object_builder in SCons.Tool.createObjBuilders(env): - emitterdict = object_builder.builder.emitter -- for suffix in emitterdict.iterkeys(): -+ for suffix in emitterdict.keys(): - if not suffix in suffixes: - continue - base = emitterdict[suffix] -diff --git a/site_scons/site_tools/idl_tool.py b/site_scons/site_tools/idl_tool.py -index c0455c2110..519583b6ca 100755 ---- a/site_scons/site_tools/idl_tool.py -+++ b/site_scons/site_tools/idl_tool.py -@@ -47,7 +47,7 @@ def idl_scanner(node, env, path): - - deps_list = deps_str.splitlines() - -- nodes_deps_list = [ env.File(d) for d in deps_list] -+ nodes_deps_list = [ env.File(d.decode("utf-8")) for d in deps_list] - nodes_deps_list.extend(env.Glob('#buildscripts/idl/*.py')) - nodes_deps_list.extend(env.Glob('#buildscripts/idl/idl/*.py')) - -diff --git a/site_scons/site_tools/jstoh.py b/site_scons/site_tools/jstoh.py -index 26eb6cbbf2..9c71e0c061 100644 ---- a/site_scons/site_tools/jstoh.py -+++ b/site_scons/site_tools/jstoh.py -@@ -1,3 +1,5 @@ -+from __future__ import unicode_literals -+ - import os - import sys - -@@ -39,7 +41,7 @@ def jsToHeader(target, source): - - text = '\n'.join(h) - -- with open(outFile, 'wb') as out: -+ with open(outFile, 'w') as out: - try: - out.write(text) - finally: -@@ -48,7 +50,7 @@ def jsToHeader(target, source): - - if __name__ == "__main__": - if len(sys.argv) < 3: -- print "Must specify [target] [source] " -+ print("Must specify [target] [source] ") - sys.exit(1) - - jsToHeader(sys.argv[1], sys.argv[2:]) -diff --git a/site_scons/site_tools/mongo_benchmark.py b/site_scons/site_tools/mongo_benchmark.py -index b2a1750e3d..47a190dfdd 100644 ---- a/site_scons/site_tools/mongo_benchmark.py -+++ b/site_scons/site_tools/mongo_benchmark.py -@@ -14,7 +14,7 @@ def benchmark_list_builder_action(env, target, source): - ofile = open(str(target[0]), 'wb') - try: - for s in _benchmarks: -- print '\t' + str(s) -+ print('\t' + str(s)) - ofile.write('%s\n' % s) - finally: - ofile.close() -diff --git a/site_scons/site_tools/mongo_integrationtest.py b/site_scons/site_tools/mongo_integrationtest.py -index 0ced90c949..aeda674991 100644 ---- a/site_scons/site_tools/mongo_integrationtest.py -+++ b/site_scons/site_tools/mongo_integrationtest.py -@@ -12,10 +12,10 @@ def register_integration_test(env, test): - env.Alias('$INTEGRATION_TEST_ALIAS', installed_test) - - def integration_test_list_builder_action(env, target, source): -- ofile = open(str(target[0]), 'wb') -+ ofile = open(str(target[0]), 'w') - try: - for s in _integration_tests: -- print '\t' + str(s) -+ print('\t' + str(s)) - ofile.write('%s\n' % s) - finally: - ofile.close() -diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py -index 2ad0f51bfd..1ca644c611 100644 ---- a/site_scons/site_tools/mongo_unittest.py -+++ b/site_scons/site_tools/mongo_unittest.py -@@ -11,10 +11,10 @@ def register_unit_test(env, test): - env.Alias('$UNITTEST_ALIAS', test) - - def unit_test_list_builder_action(env, target, source): -- ofile = open(str(target[0]), 'wb') -+ ofile = open(str(target[0]), 'w') - try: - for s in _unittests: -- print '\t' + str(s) -+ print('\t' + str(s)) - ofile.write('%s\n' % s) - finally: - ofile.close() -diff --git a/site_scons/site_tools/split_dwarf.py b/site_scons/site_tools/split_dwarf.py -index 95130c9e9a..c02d78619f 100644 ---- a/site_scons/site_tools/split_dwarf.py -+++ b/site_scons/site_tools/split_dwarf.py -@@ -52,7 +52,7 @@ def generate(env): - - for object_builder in SCons.Tool.createObjBuilders(env): - emitterdict = object_builder.builder.emitter -- for suffix in emitterdict.iterkeys(): -+ for suffix in emitterdict.keys(): - if not suffix in suffixes: - continue - base = emitterdict[suffix] -diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py -index 511c0ef6e5..0d8a83b83a 100644 ---- a/site_scons/site_tools/thin_archive.py -+++ b/site_scons/site_tools/thin_archive.py -@@ -41,7 +41,7 @@ def exists(env): - for line in pipe.stdout: - if isgnu: - continue # consume all data -- isgnu = re.search(r'^GNU ar', line) -+ isgnu = re.search(b'^GNU ar', line) - - return bool(isgnu) - -diff --git a/site_scons/site_tools/xcode.py b/site_scons/site_tools/xcode.py -index 9ec68c3547..5ddebb2e00 100644 ---- a/site_scons/site_tools/xcode.py -+++ b/site_scons/site_tools/xcode.py -@@ -9,4 +9,4 @@ def generate(env): - - if 'DEVELOPER_DIR' in os.environ: - env['ENV']['DEVELOPER_DIR'] = os.environ['DEVELOPER_DIR'] -- print "NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands" -+ print("NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands") -diff --git a/src/mongo/SConscript b/src/mongo/SConscript -index c8f925b7c8..e18a3829f1 100644 ---- a/src/mongo/SConscript -+++ b/src/mongo/SConscript -@@ -154,7 +154,7 @@ js_engine_ver = get_option("js-engine") if get_option("server-js") == "on" else - - # On windows, we need to escape the backslashes in the command-line - # so that windows paths look okay. --cmd_line = " ".join(sys.argv).encode('string-escape') -+cmd_line = " ".join(sys.argv).encode('unicode_escape') - if env.TargetOSIs('windows'): - cmd_line = cmd_line.replace('\\', r'\\') - -@@ -644,7 +644,7 @@ env.Append(MODULE_BANNERS = [distsrc.File('README'), - - # If no module has introduced a file named LICENSE-Enterprise.txt then this - # is a Community build, so inject the AGPL and the Community license --if sum(itertools.imap(lambda x: x.name == "LICENSE-Enterprise.txt", env['MODULE_BANNERS'])) == 0: -+if sum(map(lambda x: x.name == "LICENSE-Enterprise.txt", env['MODULE_BANNERS'])) == 0: - env.Append(MODULE_BANNERS = [distsrc.File('GNU-AGPL-3.0'), - distsrc.File('LICENSE-Community.txt')]) - -@@ -664,7 +664,7 @@ module_banner_transforms = ["--transform %s=$SERVER_DIST_BASENAME" % d for d in - # Allow modules to map original file name directories to subdirectories - # within the archive (e.g. { "src/mongo/db/modules/enterprise/docs": "snmp"}) - archive_addition_transforms = [] --for full_dir, archive_dir in env["ARCHIVE_ADDITION_DIR_MAP"].items(): -+for full_dir, archive_dir in list(env["ARCHIVE_ADDITION_DIR_MAP"].items()): - archive_addition_transforms.append("--transform \"%s=$SERVER_DIST_BASENAME/%s\"" % - (full_dir, archive_dir)) - -diff --git a/src/mongo/base/generate_error_codes.py b/src/mongo/base/generate_error_codes.py -index 577108c7ec..e9a1dfa552 100644 ---- a/src/mongo/base/generate_error_codes.py -+++ b/src/mongo/base/generate_error_codes.py -@@ -26,6 +26,8 @@ - # delete this exception statement from all source files in the program, - # then also delete it in the license file. - -+from __future__ import unicode_literals -+ - """Generate error_codes.{h,cpp} from error_codes.err. - - Format of error_codes.err: -@@ -98,7 +100,7 @@ def main(argv): - categories=error_classes, - ) - -- with open(output, 'wb') as outfile: -+ with open(output, 'w') as outfile: - outfile.write(text) - - def die(message=None): -diff --git a/src/mongo/db/auth/generate_action_types.py b/src/mongo/db/auth/generate_action_types.py -index b712b29666..39252ed293 100755 ---- a/src/mongo/db/auth/generate_action_types.py -+++ b/src/mongo/db/auth/generate_action_types.py -@@ -227,7 +227,7 @@ def hasDuplicateActionTypes(actionTypes): - prevActionType = sortedActionTypes[0] - for actionType in sortedActionTypes[1:]: - if actionType == prevActionType: -- print 'Duplicate actionType %s\n' % actionType -+ print('Duplicate actionType %s\n' % actionType) - didFail = True - prevActionType = actionType - -@@ -240,7 +240,7 @@ def parseActionTypesFromFile(actionTypesFilename): - - if __name__ == "__main__": - if len(sys.argv) != 4: -- print "Usage: generate_action_types.py
" -+ print("Usage: generate_action_types.py
") - sys.exit(-1) - - actionTypes = parseActionTypesFromFile(sys.argv[1]) -diff --git a/src/mongo/db/fts/generate_stop_words.py b/src/mongo/db/fts/generate_stop_words.py -index 31603eb92e..ae4ad6ccbd 100644 ---- a/src/mongo/db/fts/generate_stop_words.py -+++ b/src/mongo/db/fts/generate_stop_words.py -@@ -1,7 +1,7 @@ - import sys - - def generate( header, source, language_files ): -- out = open( header, "wb" ) -+ out = open( header, "w" ) - out.write( """ - #pragma once - #include -@@ -18,7 +18,7 @@ namespace fts { - - - -- out = open( source, "wb" ) -+ out = open( source, "w" ) - out.write( '#include "%s"' % header.rpartition( "/" )[2].rpartition( "\\" )[2] ) - out.write( """ - namespace mongo { -@@ -34,7 +34,7 @@ namespace fts { - out.write( ' // %s\n' % l_file ) - out.write( ' {\n' ) - out.write( ' const char* const words[] = {\n' ) -- for word in open( l_file, "rb" ): -+ for word in open( l_file, "r" ): - out.write( ' "%s",\n' % word.strip() ) - out.write( ' };\n' ) - out.write( ' const size_t wordcnt = sizeof(words) / sizeof(words[0]);\n' ) -diff --git a/src/mongo/db/fts/unicode/gen_diacritic_map.py b/src/mongo/db/fts/unicode/gen_diacritic_map.py -index 08cfa95cda..7c623aff60 100644 ---- a/src/mongo/db/fts/unicode/gen_diacritic_map.py -+++ b/src/mongo/db/fts/unicode/gen_diacritic_map.py -@@ -45,7 +45,7 @@ def add_diacritic_mapping(codepoint): - # c : recomposed unicode character with diacritics removed - a = chr(codepoint) - d = normalize('NFD', a) -- r = u'' -+ r = '' - - for i in range(len(d)): - if ord(d[i]) not in diacritics: -diff --git a/src/mongo/util/generate_icu_init_cpp.py b/src/mongo/util/generate_icu_init_cpp.py -index 8ae084aeec..7c576f6ffe 100755 ---- a/src/mongo/util/generate_icu_init_cpp.py -+++ b/src/mongo/util/generate_icu_init_cpp.py -@@ -26,6 +26,9 @@ - # delete this exception statement from all source files in the program, - # then also delete it in the license file. - -+from __future__ import unicode_literals -+ -+import array - import optparse - import os - import sys -@@ -110,8 +113,8 @@ MONGO_INITIALIZER(LoadICUData)(InitializerContext* context) { - ''' - decimal_encoded_data = '' - with open(data_file_path, 'rb') as data_file: -- decimal_encoded_data = ','.join([str(ord(byte)) for byte in data_file.read()]) -- with open(cpp_file_path, 'wb') as cpp_file: -+ decimal_encoded_data = ','.join([str(byte) for byte in array.array("B", data_file.read()).tolist()]) -+ with open(cpp_file_path, 'w') as cpp_file: - cpp_file.write(source_template % dict(decimal_encoded_data=decimal_encoded_data)) - - if __name__ == '__main__': diff --git a/python3-fix.diff b/python3-fix.diff new file mode 100644 index 0000000000000000000000000000000000000000..2396ef568036ab08142ca00398f4cd276a8fc85d --- /dev/null +++ b/python3-fix.diff @@ -0,0 +1,20787 @@ +diff -upNr mongo-r4.0.23.orig/buildscripts/clang_format.py mongo-r4.0.23/buildscripts/clang_format.py +--- mongo-r4.0.23.orig/buildscripts/clang_format.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/clang_format.py 2021-03-17 01:21:05.952000000 +0000 +@@ -20,7 +20,7 @@ import sys + import tarfile + import tempfile + import threading +-import urllib2 ++import urllib.request, urllib.error, urllib.parse + from distutils import spawn # pylint: disable=no-name-in-module + from optparse import OptionParser + from multiprocessing import cpu_count +@@ -96,11 +96,11 @@ def get_clang_format_from_cache_and_extr + num_tries = 5 + for attempt in range(num_tries): + try: +- resp = urllib2.urlopen(url) ++ resp = urllib.request.urlopen(url) + with open(temp_tar_file, 'wb') as fh: + fh.write(resp.read()) + break +- except urllib2.URLError: ++ except urllib.error.URLError: + if attempt == num_tries - 1: + raise + continue +diff -upNr mongo-r4.0.23.orig/buildscripts/cpplint.py mongo-r4.0.23/buildscripts/cpplint.py +--- mongo-r4.0.23.orig/buildscripts/cpplint.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/cpplint.py 2021-03-17 01:22:50.868000000 +0000 +@@ -836,7 +836,7 @@ class _CppLintState(object): + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" +- for category, count in self.errors_by_category.iteritems(): ++ for category, count in self.errors_by_category.items(): + sys.stderr.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stderr.write('Total errors found: %d\n' % self.error_count) +@@ -1389,7 +1389,7 @@ def FindEndOfExpressionInLine(line, star + On finding an unclosed expression: (-1, None) + Otherwise: (-1, new stack at end of this line) + """ +- for i in xrange(startpos, len(line)): ++ for i in range(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack +@@ -1682,7 +1682,7 @@ def CheckForCopyright(filename, lines, e + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. +- for line in xrange(1, min(len(lines), 11)): ++ for line in range(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): + CheckForServerSidePublicLicense(line, filename, lines, error) + break +@@ -1904,7 +1904,7 @@ def CheckForBadCharacters(filename, line + error: The function to call with any errors found. + """ + for linenum, line in enumerate(lines): +- if u'\ufffd' in line: ++ if '\ufffd' in line: + error(filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + if '\0' in line: +@@ -2950,7 +2950,7 @@ def CheckForFunctionLengths(filename, cl + + if starting_func: + body_found = False +- for start_linenum in xrange(linenum, clean_lines.NumLines()): ++ for start_linenum in range(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', start_line): # Declarations and trivial functions +@@ -3427,7 +3427,7 @@ def CheckBracesSpacing(filename, clean_l + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] +- for offset in xrange(endlinenum + 1, ++ for offset in range(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): +@@ -3596,7 +3596,7 @@ def IsRValueType(clean_lines, nesting_st + + # Look for the previous 'for(' in the previous lines. + before_text = match_symbol.group(1) +- for i in xrange(start - 1, max(start - 6, 0), -1): ++ for i in range(start - 1, max(start - 6, 0), -1): + before_text = clean_lines.elided[i] + before_text + if Search(r'for\s*\([^{};]*$', before_text): + # This is the condition inside a for-loop +@@ -3723,12 +3723,12 @@ def IsRValueAllowed(clean_lines, linenum + True if line is within the region where RValue references are allowed. + """ + # Allow region marked by PUSH/POP macros +- for i in xrange(linenum, 0, -1): ++ for i in range(linenum, 0, -1): + line = clean_lines.elided[i] + if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): + if not line.endswith('PUSH'): + return False +- for j in xrange(linenum, clean_lines.NumLines(), 1): ++ for j in range(linenum, clean_lines.NumLines(), 1): + line = clean_lines.elided[j] + if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): + return line.endswith('POP') +@@ -4208,7 +4208,7 @@ def CheckCheck(filename, clean_lines, li + expression = lines[linenum][start_pos + 1:end_pos - 1] + else: + expression = lines[linenum][start_pos + 1:] +- for i in xrange(linenum + 1, end_line): ++ for i in range(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + +@@ -4336,7 +4336,7 @@ def GetLineWidth(line): + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ +- if isinstance(line, unicode): ++ if isinstance(line, str): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): +@@ -4689,7 +4689,7 @@ def _GetTextInside(text, start_pattern): + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} +- closing_punctuation = set(matching_punctuation.itervalues()) ++ closing_punctuation = set(matching_punctuation.values()) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) +@@ -5015,7 +5015,7 @@ def IsDerivedFunction(clean_lines, linen + virt-specifier. + """ + # Scan back a few lines for start of current function +- for i in xrange(linenum, max(-1, linenum - 10), -1): ++ for i in range(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis +@@ -5036,7 +5036,7 @@ def IsInitializerList(clean_lines, linen + True if current line appears to be inside constructor initializer + list, False otherwise. + """ +- for i in xrange(linenum, 1, -1): ++ for i in range(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) +@@ -5132,7 +5132,7 @@ def CheckForNonConstReference(filename, + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' +- for i in xrange(startline, linenum + 1): ++ for i in range(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may +@@ -5156,7 +5156,7 @@ def CheckForNonConstReference(filename, + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: +- for i in xrange(linenum - 1, max(0, linenum - 10), -1): ++ for i in range(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break +@@ -5187,7 +5187,7 @@ def CheckForNonConstReference(filename, + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. +- for i in xrange(2): ++ for i in range(2): + if (linenum > i and + Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + return +@@ -5349,7 +5349,7 @@ def CheckCStyleCast(filename, clean_line + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: +- for i in xrange(linenum - 1, max(0, linenum - 5), -1): ++ for i in range(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False +@@ -5606,7 +5606,7 @@ def CheckForIncludeWhatYouUse(filename, + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + +- for linenum in xrange(clean_lines.NumLines()): ++ for linenum in range(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue +@@ -5655,7 +5655,7 @@ def CheckForIncludeWhatYouUse(filename, + + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. +- header_keys = include_dict.keys() ++ header_keys = list(include_dict.keys()) + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header +@@ -5750,7 +5750,7 @@ def CheckRedundantVirtual(filename, clea + end_col = -1 + end_line = -1 + start_col = len(virtual.group(1)) +- for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): ++ for start_line in range(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: +@@ -5765,7 +5765,7 @@ def CheckRedundantVirtual(filename, clea + + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). +- for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): ++ for i in range(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: +@@ -5992,7 +5992,7 @@ def ProcessFileData(filename, file_exten + + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) +- for line in xrange(clean_lines.NumLines()): ++ for line in range(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions) +diff -upNr mongo-r4.0.23.orig/buildscripts/cpplint.py.orig mongo-r4.0.23/buildscripts/cpplint.py.orig +--- mongo-r4.0.23.orig/buildscripts/cpplint.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/cpplint.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,6285 @@ ++#!/usr/bin/env python ++# ++# Copyright (c) 2009 Google Inc. All rights reserved. ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# * Redistributions in binary form must reproduce the above ++# copyright notice, this list of conditions and the following disclaimer ++# in the documentation and/or other materials provided with the ++# distribution. ++# * Neither the name of Google Inc. nor the names of its ++# contributors may be used to endorse or promote products derived from ++# this software without specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ ++"""Does google-lint on c++ files. ++ ++The goal of this script is to identify places in the code that *may* ++be in non-compliance with google style. It does not attempt to fix ++up these problems -- the point is to educate. It does also not ++attempt to find all problems, or to ensure that everything it does ++find is legitimately a problem. ++ ++In particular, we can get very confused by /* and // inside strings! ++We do a small hack, which is to ignore //'s with "'s after them on the ++same line, but it is far from perfect (in either direction). ++""" ++ ++import codecs ++import copy ++import getopt ++import math # for log ++import os ++import re ++import sre_compile ++import string ++import sys ++import unicodedata ++ ++ ++_USAGE = """ ++Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] ++ [--counting=total|toplevel|detailed] [--root=subdir] ++ [--linelength=digits] ++ [file] ... ++ ++ The style guidelines this tries to follow are those in ++ http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml ++ ++ Every problem is given a confidence score from 1-5, with 5 meaning we are ++ certain of the problem, and 1 meaning it could be a legitimate construct. ++ This will miss some errors, and is not a substitute for a code review. ++ ++ To suppress false-positive errors of a certain category, add a ++ 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) ++ suppresses errors of all categories on that line. ++ ++ The files passed in will be linted; at least one file must be provided. ++ Default linted extensions are .cc, .cpp, .cu, .cuh and .h. Change the ++ extensions with the --extensions flag. ++ ++ Flags: ++ ++ output=vs7 ++ By default, the output is formatted to ease emacs parsing. Visual Studio ++ compatible output (vs7) may also be used. Other formats are unsupported. ++ ++ verbose=# ++ Specify a number 0-5 to restrict errors to certain verbosity levels. ++ ++ filter=-x,+y,... ++ Specify a comma-separated list of category-filters to apply: only ++ error messages whose category names pass the filters will be printed. ++ (Category names are printed with the message and look like ++ "[whitespace/indent]".) Filters are evaluated left to right. ++ "-FOO" and "FOO" means "do not print categories that start with FOO". ++ "+FOO" means "do print categories that start with FOO". ++ ++ Examples: --filter=-whitespace,+whitespace/braces ++ --filter=whitespace,runtime/printf,+runtime/printf_format ++ --filter=-,+build/include_what_you_use ++ ++ To see a list of all the categories used in cpplint, pass no arg: ++ --filter= ++ ++ counting=total|toplevel|detailed ++ The total number of errors found is always printed. If ++ 'toplevel' is provided, then the count of errors in each of ++ the top-level categories like 'build' and 'whitespace' will ++ also be printed. If 'detailed' is provided, then a count ++ is provided for each category like 'build/class'. ++ ++ root=subdir ++ The root directory used for deriving header guard CPP variable. ++ By default, the header guard CPP variable is calculated as the relative ++ path to the directory that contains .git, .hg, or .svn. When this flag ++ is specified, the relative path is calculated from the specified ++ directory. If the specified directory does not exist, this flag is ++ ignored. ++ ++ Examples: ++ Assuming that src/.git exists, the header guard CPP variables for ++ src/chrome/browser/ui/browser.h are: ++ ++ No flag => CHROME_BROWSER_UI_BROWSER_H_ ++ --root=chrome => BROWSER_UI_BROWSER_H_ ++ --root=chrome/browser => UI_BROWSER_H_ ++ ++ linelength=digits ++ This is the allowed line length for the project. The default value is ++ 80 characters. ++ ++ Examples: ++ --linelength=120 ++ ++ extensions=extension,extension,... ++ The allowed file extensions that cpplint will check ++ ++ Examples: ++ --extensions=hpp,cpp ++ ++ cpplint.py supports per-directory configurations specified in CPPLINT.cfg ++ files. CPPLINT.cfg file can contain a number of key=value pairs. ++ Currently the following options are supported: ++ ++ set noparent ++ filter=+filter1,-filter2,... ++ exclude_files=regex ++ linelength=80 ++ ++ "set noparent" option prevents cpplint from traversing directory tree ++ upwards looking for more .cfg files in parent directories. This option ++ is usually placed in the top-level project directory. ++ ++ The "filter" option is similar in function to --filter flag. It specifies ++ message filters in addition to the |_DEFAULT_FILTERS| and those specified ++ through --filter command-line flag. ++ ++ "exclude_files" allows to specify a regular expression to be matched against ++ a file name. If the expression matches, the file is skipped and not run ++ through liner. ++ ++ "linelength" allows to specify the allowed line length for the project. ++ ++ CPPLINT.cfg has an effect on files in the same directory and all ++ sub-directories, unless overridden by a nested configuration file. ++ ++ Example file: ++ filter=-build/include_order,+build/include_alpha ++ exclude_files=.*\.cc ++ ++ The above example disables build/include_order warning and enables ++ build/include_alpha as well as excludes all .cc from being ++ processed by linter, in the current directory (where the .cfg ++ file is located) and all sub-directories. ++""" ++ ++# We categorize each error message we print. Here are the categories. ++# We want an explicit list so we can list them all in cpplint --filter=. ++# If you add a new error message with a new category, add it to the list ++# here! cpplint_unittest.py should tell you if you forget to do this. ++_ERROR_CATEGORIES = [ ++ 'build/class', ++ 'build/c++11', ++ 'build/deprecated', ++ 'build/endif_comment', ++ 'build/explicit_make_pair', ++ 'build/forward_decl', ++ 'build/header_guard', ++ 'build/include', ++ 'build/include_alpha', ++ 'build/include_order', ++ 'build/include_what_you_use', ++ 'build/namespaces', ++ 'build/printf_format', ++ 'build/storage_class', ++ 'legal/copyright', ++ 'legal/license', ++ 'mongo/polyfill', ++ 'readability/alt_tokens', ++ 'readability/braces', ++ 'readability/casting', ++ 'readability/check', ++ 'readability/constructors', ++ 'readability/fn_size', ++ 'readability/function', ++ 'readability/inheritance', ++ 'readability/multiline_comment', ++ 'readability/multiline_string', ++ 'readability/namespace', ++ 'readability/nolint', ++ 'readability/nul', ++ 'readability/streams', ++ 'readability/todo', ++ 'readability/utf8', ++ 'runtime/arrays', ++ 'runtime/casting', ++ 'runtime/explicit', ++ 'runtime/int', ++ 'runtime/init', ++ 'runtime/invalid_increment', ++ 'runtime/member_string_references', ++ 'runtime/memset', ++ 'runtime/indentation_namespace', ++ 'runtime/operator', ++ 'runtime/printf', ++ 'runtime/printf_format', ++ 'runtime/references', ++ 'runtime/string', ++ 'runtime/threadsafe_fn', ++ 'runtime/vlog', ++ 'whitespace/blank_line', ++ 'whitespace/braces', ++ 'whitespace/comma', ++ 'whitespace/comments', ++ 'whitespace/empty_conditional_body', ++ 'whitespace/empty_loop_body', ++ 'whitespace/end_of_line', ++ 'whitespace/ending_newline', ++ 'whitespace/forcolon', ++ 'whitespace/indent', ++ 'whitespace/line_length', ++ 'whitespace/newline', ++ 'whitespace/operators', ++ 'whitespace/parens', ++ 'whitespace/semicolon', ++ 'whitespace/tab', ++ 'whitespace/todo' ++ ] ++ ++# The default state of the category filter. This is overridden by the --filter= ++# flag. By default all errors are on, so only add here categories that should be ++# off by default (i.e., categories that must be enabled by the --filter= flags). ++# All entries here should start with a '-' or '+', as in the --filter= flag. ++_DEFAULT_FILTERS = ['-build/include_alpha'] ++ ++# We used to check for high-bit characters, but after much discussion we ++# decided those were OK, as long as they were in UTF-8 and didn't represent ++# hard-coded international strings, which belong in a separate i18n file. ++ ++# C++ headers ++_CPP_HEADERS = frozenset([ ++ # Legacy ++ 'algobase.h', ++ 'algo.h', ++ 'alloc.h', ++ 'builtinbuf.h', ++ 'bvector.h', ++ 'complex.h', ++ 'defalloc.h', ++ 'deque.h', ++ 'editbuf.h', ++ 'fstream.h', ++ 'function.h', ++ 'hash_map', ++ 'hash_map.h', ++ 'hash_set', ++ 'hash_set.h', ++ 'hashtable.h', ++ 'heap.h', ++ 'indstream.h', ++ 'iomanip.h', ++ 'iostream.h', ++ 'istream.h', ++ 'iterator.h', ++ 'list.h', ++ 'map.h', ++ 'multimap.h', ++ 'multiset.h', ++ 'ostream.h', ++ 'pair.h', ++ 'parsestream.h', ++ 'pfstream.h', ++ 'procbuf.h', ++ 'pthread_alloc', ++ 'pthread_alloc.h', ++ 'rope', ++ 'rope.h', ++ 'ropeimpl.h', ++ 'set.h', ++ 'slist', ++ 'slist.h', ++ 'stack.h', ++ 'stdiostream.h', ++ 'stl_alloc.h', ++ 'stl_relops.h', ++ 'streambuf.h', ++ 'stream.h', ++ 'strfile.h', ++ 'strstream.h', ++ 'tempbuf.h', ++ 'tree.h', ++ 'type_traits.h', ++ 'vector.h', ++ # 17.6.1.2 C++ library headers ++ 'algorithm', ++ 'array', ++ 'atomic', ++ 'bitset', ++ 'chrono', ++ 'codecvt', ++ 'complex', ++ 'condition_variable', ++ 'deque', ++ 'exception', ++ 'forward_list', ++ 'fstream', ++ 'functional', ++ 'future', ++ 'initializer_list', ++ 'iomanip', ++ 'ios', ++ 'iosfwd', ++ 'iostream', ++ 'istream', ++ 'iterator', ++ 'limits', ++ 'list', ++ 'locale', ++ 'map', ++ 'memory', ++ 'mutex', ++ 'new', ++ 'numeric', ++ 'ostream', ++ 'queue', ++ 'random', ++ 'ratio', ++ 'regex', ++ 'set', ++ 'sstream', ++ 'stack', ++ 'stdexcept', ++ 'streambuf', ++ 'string', ++ 'strstream', ++ 'system_error', ++ 'thread', ++ 'tuple', ++ 'typeindex', ++ 'typeinfo', ++ 'type_traits', ++ 'unordered_map', ++ 'unordered_set', ++ 'utility', ++ 'valarray', ++ 'vector', ++ # 17.6.1.2 C++ headers for C library facilities ++ 'cassert', ++ 'ccomplex', ++ 'cctype', ++ 'cerrno', ++ 'cfenv', ++ 'cfloat', ++ 'cinttypes', ++ 'ciso646', ++ 'climits', ++ 'clocale', ++ 'cmath', ++ 'csetjmp', ++ 'csignal', ++ 'cstdalign', ++ 'cstdarg', ++ 'cstdbool', ++ 'cstddef', ++ 'cstdint', ++ 'cstdio', ++ 'cstdlib', ++ 'cstring', ++ 'ctgmath', ++ 'ctime', ++ 'cuchar', ++ 'cwchar', ++ 'cwctype', ++ ]) ++ ++ ++# These headers are excluded from [build/include] and [build/include_order] ++# checks: ++# - Anything not following google file name conventions (containing an ++# uppercase character, such as Python.h or nsStringAPI.h, for example). ++# - Lua headers. ++_THIRD_PARTY_HEADERS_PATTERN = re.compile( ++ r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') ++ ++ ++# Assertion macros. These are defined in base/logging.h and ++# testing/base/gunit.h. Note that the _M versions need to come first ++# for substring matching to work. ++_CHECK_MACROS = [ ++ 'DCHECK', 'CHECK', ++ 'EXPECT_TRUE_M', 'EXPECT_TRUE', ++ 'ASSERT_TRUE_M', 'ASSERT_TRUE', ++ 'EXPECT_FALSE_M', 'EXPECT_FALSE', ++ 'ASSERT_FALSE_M', 'ASSERT_FALSE', ++ ] ++ ++# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE ++_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) ++ ++for op, replacement in [('==', 'EQ'), ('!=', 'NE'), ++ ('>=', 'GE'), ('>', 'GT'), ++ ('<=', 'LE'), ('<', 'LT')]: ++ _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement ++ _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement ++ _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement ++ _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement ++ _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement ++ _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement ++ ++for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), ++ ('>=', 'LT'), ('>', 'LE'), ++ ('<=', 'GT'), ('<', 'GE')]: ++ _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement ++ _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement ++ _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement ++ _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement ++ ++# Alternative tokens and their replacements. For full list, see section 2.5 ++# Alternative tokens [lex.digraph] in the C++ standard. ++# ++# Digraphs (such as '%:') are not included here since it's a mess to ++# match those on a word boundary. ++_ALT_TOKEN_REPLACEMENT = { ++ 'and': '&&', ++ 'bitor': '|', ++ 'or': '||', ++ 'xor': '^', ++ 'compl': '~', ++ 'bitand': '&', ++ 'and_eq': '&=', ++ 'or_eq': '|=', ++ 'xor_eq': '^=', ++ 'not': '!', ++ 'not_eq': '!=' ++ } ++ ++# Compile regular expression that matches all the above keywords. The "[ =()]" ++# bit is meant to avoid matching these keywords outside of boolean expressions. ++# ++# False positives include C-style multi-line comments and multi-line strings ++# but those have always been troublesome for cpplint. ++_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( ++ r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') ++ ++ ++# These constants define types of headers for use with ++# _IncludeState.CheckNextIncludeOrder(). ++_C_SYS_HEADER = 1 ++_CPP_SYS_HEADER = 2 ++_LIKELY_MY_HEADER = 3 ++_POSSIBLE_MY_HEADER = 4 ++_OTHER_HEADER = 5 ++ ++# These constants define the current inline assembly state ++_NO_ASM = 0 # Outside of inline assembly block ++_INSIDE_ASM = 1 # Inside inline assembly block ++_END_ASM = 2 # Last line of inline assembly block ++_BLOCK_ASM = 3 # The whole block is an inline assembly block ++ ++# Match start of assembly blocks ++_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' ++ r'(?:\s+(volatile|__volatile__))?' ++ r'\s*[{(]') ++ ++ ++_regexp_compile_cache = {} ++ ++# {str, set(int)}: a map from error categories to sets of linenumbers ++# on which those errors are expected and should be suppressed. ++_error_suppressions = {} ++ ++# The root directory used for deriving header guard CPP variable. ++# This is set by --root flag. ++_root = None ++ ++# The allowed line length of files. ++# This is set by --linelength flag. ++_line_length = 80 ++ ++# The allowed extensions for file names ++# This is set by --extensions flag. ++_valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh']) ++ ++def ParseNolintSuppressions(filename, raw_line, linenum, error): ++ """Updates the global list of error-suppressions. ++ ++ Parses any NOLINT comments on the current line, updating the global ++ error_suppressions store. Reports an error if the NOLINT comment ++ was malformed. ++ ++ Args: ++ filename: str, the name of the input file. ++ raw_line: str, the line of input text, with comments. ++ linenum: int, the number of the current line. ++ error: function, an error handler. ++ """ ++ matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) ++ if matched: ++ if matched.group(1): ++ suppressed_line = linenum + 1 ++ else: ++ suppressed_line = linenum ++ category = matched.group(2) ++ if category in (None, '(*)'): # => "suppress all" ++ _error_suppressions.setdefault(None, set()).add(suppressed_line) ++ else: ++ if category.startswith('(') and category.endswith(')'): ++ category = category[1:-1] ++ if category in _ERROR_CATEGORIES: ++ _error_suppressions.setdefault(category, set()).add(suppressed_line) ++ else: ++ error(filename, linenum, 'readability/nolint', 5, ++ 'Unknown NOLINT error category: %s' % category) ++ ++ ++def ResetNolintSuppressions(): ++ """Resets the set of NOLINT suppressions to empty.""" ++ _error_suppressions.clear() ++ ++ ++def IsErrorSuppressedByNolint(category, linenum): ++ """Returns true if the specified error category is suppressed on this line. ++ ++ Consults the global error_suppressions map populated by ++ ParseNolintSuppressions/ResetNolintSuppressions. ++ ++ Args: ++ category: str, the category of the error. ++ linenum: int, the current line number. ++ Returns: ++ bool, True iff the error should be suppressed due to a NOLINT comment. ++ """ ++ return (linenum in _error_suppressions.get(category, set()) or ++ linenum in _error_suppressions.get(None, set())) ++ ++ ++def Match(pattern, s): ++ """Matches the string with the pattern, caching the compiled regexp.""" ++ # The regexp compilation caching is inlined in both Match and Search for ++ # performance reasons; factoring it out into a separate function turns out ++ # to be noticeably expensive. ++ if pattern not in _regexp_compile_cache: ++ _regexp_compile_cache[pattern] = sre_compile.compile(pattern) ++ return _regexp_compile_cache[pattern].match(s) ++ ++ ++def ReplaceAll(pattern, rep, s): ++ """Replaces instances of pattern in a string with a replacement. ++ ++ The compiled regex is kept in a cache shared by Match and Search. ++ ++ Args: ++ pattern: regex pattern ++ rep: replacement text ++ s: search string ++ ++ Returns: ++ string with replacements made (or original string if no replacements) ++ """ ++ if pattern not in _regexp_compile_cache: ++ _regexp_compile_cache[pattern] = sre_compile.compile(pattern) ++ return _regexp_compile_cache[pattern].sub(rep, s) ++ ++ ++def Search(pattern, s): ++ """Searches the string for the pattern, caching the compiled regexp.""" ++ if pattern not in _regexp_compile_cache: ++ _regexp_compile_cache[pattern] = sre_compile.compile(pattern) ++ return _regexp_compile_cache[pattern].search(s) ++ ++ ++class _IncludeState(object): ++ """Tracks line numbers for includes, and the order in which includes appear. ++ ++ include_list contains list of lists of (header, line number) pairs. ++ It's a lists of lists rather than just one flat list to make it ++ easier to update across preprocessor boundaries. ++ ++ Call CheckNextIncludeOrder() once for each header in the file, passing ++ in the type constants defined above. Calls in an illegal order will ++ raise an _IncludeError with an appropriate error message. ++ ++ """ ++ # self._section will move monotonically through this set. If it ever ++ # needs to move backwards, CheckNextIncludeOrder will raise an error. ++ _INITIAL_SECTION = 0 ++ _MY_H_SECTION = 1 ++ _C_SECTION = 2 ++ _CPP_SECTION = 3 ++ _OTHER_H_SECTION = 4 ++ ++ _TYPE_NAMES = { ++ _C_SYS_HEADER: 'C system header', ++ _CPP_SYS_HEADER: 'C++ system header', ++ _LIKELY_MY_HEADER: 'header this file implements', ++ _POSSIBLE_MY_HEADER: 'header this file may implement', ++ _OTHER_HEADER: 'other header', ++ } ++ _SECTION_NAMES = { ++ _INITIAL_SECTION: "... nothing. (This can't be an error.)", ++ _MY_H_SECTION: 'a header this file implements', ++ _C_SECTION: 'C system header', ++ _CPP_SECTION: 'C++ system header', ++ _OTHER_H_SECTION: 'other header', ++ } ++ ++ def __init__(self): ++ self.include_list = [[]] ++ self.ResetSection('') ++ ++ def FindHeader(self, header): ++ """Check if a header has already been included. ++ ++ Args: ++ header: header to check. ++ Returns: ++ Line number of previous occurrence, or -1 if the header has not ++ been seen before. ++ """ ++ for section_list in self.include_list: ++ for f in section_list: ++ if f[0] == header: ++ return f[1] ++ return -1 ++ ++ def ResetSection(self, directive): ++ """Reset section checking for preprocessor directive. ++ ++ Args: ++ directive: preprocessor directive (e.g. "if", "else"). ++ """ ++ # The name of the current section. ++ self._section = self._INITIAL_SECTION ++ # The path of last found header. ++ self._last_header = '' ++ ++ # Update list of includes. Note that we never pop from the ++ # include list. ++ if directive in ('if', 'ifdef', 'ifndef'): ++ self.include_list.append([]) ++ elif directive in ('else', 'elif'): ++ self.include_list[-1] = [] ++ ++ def SetLastHeader(self, header_path): ++ self._last_header = header_path ++ ++ def CanonicalizeAlphabeticalOrder(self, header_path): ++ """Returns a path canonicalized for alphabetical comparison. ++ ++ - replaces "-" with "_" so they both cmp the same. ++ - removes '-inl' since we don't require them to be after the main header. ++ - lowercase everything, just in case. ++ ++ Args: ++ header_path: Path to be canonicalized. ++ ++ Returns: ++ Canonicalized path. ++ """ ++ return header_path.replace('-inl.h', '.h').replace('-', '_').lower() ++ ++ def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): ++ """Check if a header is in alphabetical order with the previous header. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ header_path: Canonicalized header to be checked. ++ ++ Returns: ++ Returns true if the header is in alphabetical order. ++ """ ++ # If previous section is different from current section, _last_header will ++ # be reset to empty string, so it's always less than current header. ++ # ++ # If previous line was a blank line, assume that the headers are ++ # intentionally sorted the way they are. ++ if (self._last_header > header_path and ++ not Match(r'^\s*$', clean_lines.elided[linenum - 1])): ++ return False ++ return True ++ ++ def CheckNextIncludeOrder(self, header_type): ++ """Returns a non-empty error message if the next header is out of order. ++ ++ This function also updates the internal state to be ready to check ++ the next include. ++ ++ Args: ++ header_type: One of the _XXX_HEADER constants defined above. ++ ++ Returns: ++ The empty string if the header is in the right order, or an ++ error message describing what's wrong. ++ ++ """ ++ error_message = ('Found %s after %s' % ++ (self._TYPE_NAMES[header_type], ++ self._SECTION_NAMES[self._section])) ++ ++ last_section = self._section ++ ++ if header_type == _C_SYS_HEADER: ++ if self._section <= self._C_SECTION: ++ self._section = self._C_SECTION ++ else: ++ self._last_header = '' ++ return error_message ++ elif header_type == _CPP_SYS_HEADER: ++ if self._section <= self._CPP_SECTION: ++ self._section = self._CPP_SECTION ++ else: ++ self._last_header = '' ++ return error_message ++ elif header_type == _LIKELY_MY_HEADER: ++ if self._section <= self._MY_H_SECTION: ++ self._section = self._MY_H_SECTION ++ else: ++ self._section = self._OTHER_H_SECTION ++ elif header_type == _POSSIBLE_MY_HEADER: ++ if self._section <= self._MY_H_SECTION: ++ self._section = self._MY_H_SECTION ++ else: ++ # This will always be the fallback because we're not sure ++ # enough that the header is associated with this file. ++ self._section = self._OTHER_H_SECTION ++ else: ++ assert header_type == _OTHER_HEADER ++ self._section = self._OTHER_H_SECTION ++ ++ if last_section != self._section: ++ self._last_header = '' ++ ++ return '' ++ ++ ++class _CppLintState(object): ++ """Maintains module-wide state..""" ++ ++ def __init__(self): ++ self.verbose_level = 1 # global setting. ++ self.error_count = 0 # global count of reported errors ++ # filters to apply when emitting error messages ++ self.filters = _DEFAULT_FILTERS[:] ++ # backup of filter list. Used to restore the state after each file. ++ self._filters_backup = self.filters[:] ++ self.counting = 'total' # In what way are we counting errors? ++ self.errors_by_category = {} # string to int dict storing error counts ++ ++ # output format: ++ # "emacs" - format that emacs can parse (default) ++ # "vs7" - format that Microsoft Visual Studio 7 can parse ++ self.output_format = 'emacs' ++ ++ def SetOutputFormat(self, output_format): ++ """Sets the output format for errors.""" ++ self.output_format = output_format ++ ++ def SetVerboseLevel(self, level): ++ """Sets the module's verbosity, and returns the previous setting.""" ++ last_verbose_level = self.verbose_level ++ self.verbose_level = level ++ return last_verbose_level ++ ++ def SetCountingStyle(self, counting_style): ++ """Sets the module's counting options.""" ++ self.counting = counting_style ++ ++ def SetFilters(self, filters): ++ """Sets the error-message filters. ++ ++ These filters are applied when deciding whether to emit a given ++ error message. ++ ++ Args: ++ filters: A string of comma-separated filters (eg "+whitespace/indent"). ++ Each filter should start with + or -; else we die. ++ ++ Raises: ++ ValueError: The comma-separated filters did not all start with '+' or '-'. ++ E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" ++ """ ++ # Default filters always have less priority than the flag ones. ++ self.filters = _DEFAULT_FILTERS[:] ++ self.AddFilters(filters) ++ ++ def AddFilters(self, filters): ++ """ Adds more filters to the existing list of error-message filters. """ ++ for filt in filters.split(','): ++ clean_filt = filt.strip() ++ if clean_filt: ++ self.filters.append(clean_filt) ++ for filt in self.filters: ++ if not (filt.startswith('+') or filt.startswith('-')): ++ raise ValueError('Every filter in --filters must start with + or -' ++ ' (%s does not)' % filt) ++ ++ def BackupFilters(self): ++ """ Saves the current filter list to backup storage.""" ++ self._filters_backup = self.filters[:] ++ ++ def RestoreFilters(self): ++ """ Restores filters previously backed up.""" ++ self.filters = self._filters_backup[:] ++ ++ def ResetErrorCounts(self): ++ """Sets the module's error statistic back to zero.""" ++ self.error_count = 0 ++ self.errors_by_category = {} ++ ++ def IncrementErrorCount(self, category): ++ """Bumps the module's error statistic.""" ++ self.error_count += 1 ++ if self.counting in ('toplevel', 'detailed'): ++ if self.counting != 'detailed': ++ category = category.split('/')[0] ++ if category not in self.errors_by_category: ++ self.errors_by_category[category] = 0 ++ self.errors_by_category[category] += 1 ++ ++ def PrintErrorCounts(self): ++ """Print a summary of errors by category, and the total.""" ++ for category, count in self.errors_by_category.iteritems(): ++ sys.stderr.write('Category \'%s\' errors found: %d\n' % ++ (category, count)) ++ sys.stderr.write('Total errors found: %d\n' % self.error_count) ++ ++_cpplint_state = _CppLintState() ++ ++ ++def _OutputFormat(): ++ """Gets the module's output format.""" ++ return _cpplint_state.output_format ++ ++ ++def _SetOutputFormat(output_format): ++ """Sets the module's output format.""" ++ _cpplint_state.SetOutputFormat(output_format) ++ ++ ++def _VerboseLevel(): ++ """Returns the module's verbosity setting.""" ++ return _cpplint_state.verbose_level ++ ++ ++def _SetVerboseLevel(level): ++ """Sets the module's verbosity, and returns the previous setting.""" ++ return _cpplint_state.SetVerboseLevel(level) ++ ++ ++def _SetCountingStyle(level): ++ """Sets the module's counting options.""" ++ _cpplint_state.SetCountingStyle(level) ++ ++ ++def _Filters(): ++ """Returns the module's list of output filters, as a list.""" ++ return _cpplint_state.filters ++ ++ ++def _SetFilters(filters): ++ """Sets the module's error-message filters. ++ ++ These filters are applied when deciding whether to emit a given ++ error message. ++ ++ Args: ++ filters: A string of comma-separated filters (eg "whitespace/indent"). ++ Each filter should start with + or -; else we die. ++ """ ++ _cpplint_state.SetFilters(filters) ++ ++def _AddFilters(filters): ++ """Adds more filter overrides. ++ ++ Unlike _SetFilters, this function does not reset the current list of filters ++ available. ++ ++ Args: ++ filters: A string of comma-separated filters (eg "whitespace/indent"). ++ Each filter should start with + or -; else we die. ++ """ ++ _cpplint_state.AddFilters(filters) ++ ++def _BackupFilters(): ++ """ Saves the current filter list to backup storage.""" ++ _cpplint_state.BackupFilters() ++ ++def _RestoreFilters(): ++ """ Restores filters previously backed up.""" ++ _cpplint_state.RestoreFilters() ++ ++class _FunctionState(object): ++ """Tracks current function name and the number of lines in its body.""" ++ ++ _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. ++ _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. ++ ++ def __init__(self): ++ self.in_a_function = False ++ self.lines_in_function = 0 ++ self.current_function = '' ++ ++ def Begin(self, function_name): ++ """Start analyzing function body. ++ ++ Args: ++ function_name: The name of the function being tracked. ++ """ ++ self.in_a_function = True ++ self.lines_in_function = 0 ++ self.current_function = function_name ++ ++ def Count(self): ++ """Count line in current function body.""" ++ if self.in_a_function: ++ self.lines_in_function += 1 ++ ++ def Check(self, error, filename, linenum): ++ """Report if too many lines in function body. ++ ++ Args: ++ error: The function to call with any errors found. ++ filename: The name of the current file. ++ linenum: The number of the line to check. ++ """ ++ if Match(r'T(EST|est)', self.current_function): ++ base_trigger = self._TEST_TRIGGER ++ else: ++ base_trigger = self._NORMAL_TRIGGER ++ trigger = base_trigger * 2**_VerboseLevel() ++ ++ if self.lines_in_function > trigger: ++ error_level = int(math.log(self.lines_in_function / base_trigger, 2)) ++ # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... ++ if error_level > 5: ++ error_level = 5 ++ error(filename, linenum, 'readability/fn_size', error_level, ++ 'Small and focused functions are preferred:' ++ ' %s has %d non-comment lines' ++ ' (error triggered by exceeding %d lines).' % ( ++ self.current_function, self.lines_in_function, trigger)) ++ ++ def End(self): ++ """Stop analyzing function body.""" ++ self.in_a_function = False ++ ++ ++class _IncludeError(Exception): ++ """Indicates a problem with the include order in a file.""" ++ pass ++ ++ ++class FileInfo(object): ++ """Provides utility functions for filenames. ++ ++ FileInfo provides easy access to the components of a file's path ++ relative to the project root. ++ """ ++ ++ def __init__(self, filename): ++ self._filename = filename ++ ++ def FullName(self): ++ """Make Windows paths like Unix.""" ++ return os.path.abspath(self._filename).replace('\\', '/') ++ ++ def RepositoryName(self): ++ """FullName after removing the local path to the repository. ++ ++ If we have a real absolute path name here we can try to do something smart: ++ detecting the root of the checkout and truncating /path/to/checkout from ++ the name so that we get header guards that don't include things like ++ "C:\Documents and Settings\..." or "/home/username/..." in them and thus ++ people on different computers who have checked the source out to different ++ locations won't see bogus errors. ++ """ ++ fullname = self.FullName() ++ ++ if os.path.exists(fullname): ++ project_dir = os.path.dirname(fullname) ++ ++ if os.path.exists(os.path.join(project_dir, ".svn")): ++ # If there's a .svn file in the current directory, we recursively look ++ # up the directory tree for the top of the SVN checkout ++ root_dir = project_dir ++ one_up_dir = os.path.dirname(root_dir) ++ while os.path.exists(os.path.join(one_up_dir, ".svn")): ++ root_dir = os.path.dirname(root_dir) ++ one_up_dir = os.path.dirname(one_up_dir) ++ ++ prefix = os.path.commonprefix([root_dir, project_dir]) ++ return fullname[len(prefix) + 1:] ++ ++ # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by ++ # searching up from the current path. ++ root_dir = os.path.dirname(fullname) ++ while (root_dir != os.path.dirname(root_dir) and ++ not os.path.exists(os.path.join(root_dir, ".git")) and ++ not os.path.exists(os.path.join(root_dir, ".hg")) and ++ not os.path.exists(os.path.join(root_dir, ".svn"))): ++ root_dir = os.path.dirname(root_dir) ++ ++ if (os.path.exists(os.path.join(root_dir, ".git")) or ++ os.path.exists(os.path.join(root_dir, ".hg")) or ++ os.path.exists(os.path.join(root_dir, ".svn"))): ++ prefix = os.path.commonprefix([root_dir, project_dir]) ++ return fullname[len(prefix) + 1:] ++ ++ # Don't know what to do; header guard warnings may be wrong... ++ return fullname ++ ++ def Split(self): ++ """Splits the file into the directory, basename, and extension. ++ ++ For 'chrome/browser/browser.cc', Split() would ++ return ('chrome/browser', 'browser', '.cc') ++ ++ Returns: ++ A tuple of (directory, basename, extension). ++ """ ++ ++ googlename = self.RepositoryName() ++ project, rest = os.path.split(googlename) ++ return (project,) + os.path.splitext(rest) ++ ++ def BaseName(self): ++ """File base name - text after the final slash, before the final period.""" ++ return self.Split()[1] ++ ++ def Extension(self): ++ """File extension - text following the final period.""" ++ return self.Split()[2] ++ ++ def NoExtension(self): ++ """File has no source file extension.""" ++ return '/'.join(self.Split()[0:2]) ++ ++ def IsSource(self): ++ """File has a source file extension.""" ++ return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') ++ ++ ++def _ShouldPrintError(category, confidence, linenum): ++ """If confidence >= verbose, category passes filter and is not suppressed.""" ++ ++ # There are three ways we might decide not to print an error message: ++ # a "NOLINT(category)" comment appears in the source, ++ # the verbosity level isn't high enough, or the filters filter it out. ++ if IsErrorSuppressedByNolint(category, linenum): ++ return False ++ ++ if confidence < _cpplint_state.verbose_level: ++ return False ++ ++ is_filtered = False ++ for one_filter in _Filters(): ++ if one_filter.startswith('-'): ++ if category.startswith(one_filter[1:]): ++ is_filtered = True ++ elif one_filter.startswith('+'): ++ if category.startswith(one_filter[1:]): ++ is_filtered = False ++ else: ++ assert False # should have been checked for in SetFilter. ++ if is_filtered: ++ return False ++ ++ return True ++ ++ ++def Error(filename, linenum, category, confidence, message): ++ """Logs the fact we've found a lint error. ++ ++ We log where the error was found, and also our confidence in the error, ++ that is, how certain we are this is a legitimate style regression, and ++ not a misidentification or a use that's sometimes justified. ++ ++ False positives can be suppressed by the use of ++ "cpplint(category)" comments on the offending line. These are ++ parsed into _error_suppressions. ++ ++ Args: ++ filename: The name of the file containing the error. ++ linenum: The number of the line containing the error. ++ category: A string used to describe the "category" this bug ++ falls under: "whitespace", say, or "runtime". Categories ++ may have a hierarchy separated by slashes: "whitespace/indent". ++ confidence: A number from 1-5 representing a confidence score for ++ the error, with 5 meaning that we are certain of the problem, ++ and 1 meaning that it could be a legitimate construct. ++ message: The error message. ++ """ ++ if _ShouldPrintError(category, confidence, linenum): ++ _cpplint_state.IncrementErrorCount(category) ++ if _cpplint_state.output_format == 'vs7': ++ sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( ++ filename, linenum, message, category, confidence)) ++ elif _cpplint_state.output_format == 'eclipse': ++ sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( ++ filename, linenum, message, category, confidence)) ++ else: ++ sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( ++ filename, linenum, message, category, confidence)) ++ ++ ++# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. ++_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( ++ r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') ++# Match a single C style comment on the same line. ++_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' ++# Matches multi-line C style comments. ++# This RE is a little bit more complicated than one might expect, because we ++# have to take care of space removals tools so we can handle comments inside ++# statements better. ++# The current rule is: We only clear spaces from both sides when we're at the ++# end of the line. Otherwise, we try to remove spaces from the right side, ++# if this doesn't work we try on left side but only if there's a non-character ++# on the right. ++_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( ++ r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + ++ _RE_PATTERN_C_COMMENTS + r'\s+|' + ++ r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + ++ _RE_PATTERN_C_COMMENTS + r')') ++ ++ ++def IsCppString(line): ++ """Does line terminate so, that the next symbol is in string constant. ++ ++ This function does not consider single-line nor multi-line comments. ++ ++ Args: ++ line: is a partial line of code starting from the 0..n. ++ ++ Returns: ++ True, if next character appended to 'line' is inside a ++ string constant. ++ """ ++ ++ line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" ++ return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 ++ ++ ++def CleanseRawStrings(raw_lines): ++ """Removes C++11 raw strings from lines. ++ ++ Before: ++ static const char kData[] = R"( ++ multi-line string ++ )"; ++ ++ After: ++ static const char kData[] = "" ++ (replaced by blank line) ++ ""; ++ ++ Args: ++ raw_lines: list of raw lines. ++ ++ Returns: ++ list of lines with C++11 raw strings replaced by empty strings. ++ """ ++ ++ delimiter = None ++ lines_without_raw_strings = [] ++ for line in raw_lines: ++ if delimiter: ++ # Inside a raw string, look for the end ++ end = line.find(delimiter) ++ if end >= 0: ++ # Found the end of the string, match leading space for this ++ # line and resume copying the original lines, and also insert ++ # a "" on the last line. ++ leading_space = Match(r'^(\s*)\S', line) ++ line = leading_space.group(1) + '""' + line[end + len(delimiter):] ++ delimiter = None ++ else: ++ # Haven't found the end yet, append a blank line. ++ line = '""' ++ ++ # Look for beginning of a raw string, and replace them with ++ # empty strings. This is done in a loop to handle multiple raw ++ # strings on the same line. ++ while delimiter is None: ++ # Look for beginning of a raw string. ++ # See 2.14.15 [lex.string] for syntax. ++ matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) ++ if matched: ++ delimiter = ')' + matched.group(2) + '"' ++ ++ end = matched.group(3).find(delimiter) ++ if end >= 0: ++ # Raw string ended on same line ++ line = (matched.group(1) + '""' + ++ matched.group(3)[end + len(delimiter):]) ++ delimiter = None ++ else: ++ # Start of a multi-line raw string ++ line = matched.group(1) + '""' ++ else: ++ break ++ ++ lines_without_raw_strings.append(line) ++ ++ # TODO(unknown): if delimiter is not None here, we might want to ++ # emit a warning for unterminated string. ++ return lines_without_raw_strings ++ ++ ++def FindNextMultiLineCommentStart(lines, lineix): ++ """Find the beginning marker for a multiline comment.""" ++ while lineix < len(lines): ++ if lines[lineix].strip().startswith('/*'): ++ # Only return this marker if the comment goes beyond this line ++ if lines[lineix].strip().find('*/', 2) < 0: ++ return lineix ++ lineix += 1 ++ return len(lines) ++ ++ ++def FindNextMultiLineCommentEnd(lines, lineix): ++ """We are inside a comment, find the end marker.""" ++ while lineix < len(lines): ++ if lines[lineix].strip().endswith('*/'): ++ return lineix ++ lineix += 1 ++ return len(lines) ++ ++ ++def RemoveMultiLineCommentsFromRange(lines, begin, end): ++ """Clears a range of lines for multi-line comments.""" ++ # Having // dummy comments makes the lines non-empty, so we will not get ++ # unnecessary blank line warnings later in the code. ++ for i in range(begin, end): ++ lines[i] = '// dummy' ++ ++ ++def RemoveMultiLineComments(filename, lines, error): ++ """Removes multiline (c-style) comments from lines.""" ++ lineix = 0 ++ while lineix < len(lines): ++ lineix_begin = FindNextMultiLineCommentStart(lines, lineix) ++ if lineix_begin >= len(lines): ++ return ++ lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) ++ if lineix_end >= len(lines): ++ error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, ++ 'Could not find end of multi-line comment') ++ return ++ RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) ++ lineix = lineix_end + 1 ++ ++ ++def CleanseComments(line): ++ """Removes //-comments and single-line C-style /* */ comments. ++ ++ Args: ++ line: A line of C++ source. ++ ++ Returns: ++ The line with single-line comments removed. ++ """ ++ commentpos = line.find('//') ++ if commentpos != -1 and not IsCppString(line[:commentpos]): ++ line = line[:commentpos].rstrip() ++ # get rid of /* ... */ ++ return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) ++ ++ ++class CleansedLines(object): ++ """Holds 3 copies of all lines with different preprocessing applied to them. ++ ++ 1) elided member contains lines without strings and comments, ++ 2) lines member contains lines without comments, and ++ 3) raw_lines member contains all the lines without processing. ++ All these three members are of , and of the same length. ++ """ ++ ++ def __init__(self, lines): ++ self.elided = [] ++ self.lines = [] ++ self.raw_lines = lines ++ self.num_lines = len(lines) ++ self.lines_without_raw_strings = CleanseRawStrings(lines) ++ for linenum in range(len(self.lines_without_raw_strings)): ++ self.lines.append(CleanseComments( ++ self.lines_without_raw_strings[linenum])) ++ elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) ++ self.elided.append(CleanseComments(elided)) ++ ++ def NumLines(self): ++ """Returns the number of lines represented.""" ++ return self.num_lines ++ ++ @staticmethod ++ def _CollapseStrings(elided): ++ """Collapses strings and chars on a line to simple "" or '' blocks. ++ ++ We nix strings first so we're not fooled by text like '"http://"' ++ ++ Args: ++ elided: The line being processed. ++ ++ Returns: ++ The line with collapsed strings. ++ """ ++ if _RE_PATTERN_INCLUDE.match(elided): ++ return elided ++ ++ # Remove escaped characters first to make quote/single quote collapsing ++ # basic. Things that look like escaped characters shouldn't occur ++ # outside of strings and chars. ++ elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) ++ ++ # Replace quoted strings and digit separators. Both single quotes ++ # and double quotes are processed in the same loop, otherwise ++ # nested quotes wouldn't work. ++ collapsed = '' ++ while True: ++ # Find the first quote character ++ match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) ++ if not match: ++ collapsed += elided ++ break ++ head, quote, tail = match.groups() ++ ++ if quote == '"': ++ # Collapse double quoted strings ++ second_quote = tail.find('"') ++ if second_quote >= 0: ++ collapsed += head + '""' ++ elided = tail[second_quote + 1:] ++ else: ++ # Unmatched double quote, don't bother processing the rest ++ # of the line since this is probably a multiline string. ++ collapsed += elided ++ break ++ else: ++ # Found single quote, check nearby text to eliminate digit separators. ++ # ++ # There is no special handling for floating point here, because ++ # the integer/fractional/exponent parts would all be parsed ++ # correctly as long as there are digits on both sides of the ++ # separator. So we are fine as long as we don't see something ++ # like "0.'3" (gcc 4.9.0 will not allow this literal). ++ if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): ++ match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) ++ collapsed += head + match_literal.group(1).replace("'", '') ++ elided = match_literal.group(2) ++ else: ++ second_quote = tail.find('\'') ++ if second_quote >= 0: ++ collapsed += head + "''" ++ elided = tail[second_quote + 1:] ++ else: ++ # Unmatched single quote ++ collapsed += elided ++ break ++ ++ return collapsed ++ ++ ++def FindEndOfExpressionInLine(line, startpos, stack): ++ """Find the position just after the end of current parenthesized expression. ++ ++ Args: ++ line: a CleansedLines line. ++ startpos: start searching at this position. ++ stack: nesting stack at startpos. ++ ++ Returns: ++ On finding matching end: (index just after matching end, None) ++ On finding an unclosed expression: (-1, None) ++ Otherwise: (-1, new stack at end of this line) ++ """ ++ for i in xrange(startpos, len(line)): ++ char = line[i] ++ if char in '([{': ++ # Found start of parenthesized expression, push to expression stack ++ stack.append(char) ++ elif char == '<': ++ # Found potential start of template argument list ++ if i > 0 and line[i - 1] == '<': ++ # Left shift operator ++ if stack and stack[-1] == '<': ++ stack.pop() ++ if not stack: ++ return (-1, None) ++ elif i > 0 and Search(r'\boperator\s*$', line[0:i]): ++ # operator<, don't add to stack ++ continue ++ else: ++ # Tentative start of template argument list ++ stack.append('<') ++ elif char in ')]}': ++ # Found end of parenthesized expression. ++ # ++ # If we are currently expecting a matching '>', the pending '<' ++ # must have been an operator. Remove them from expression stack. ++ while stack and stack[-1] == '<': ++ stack.pop() ++ if not stack: ++ return (-1, None) ++ if ((stack[-1] == '(' and char == ')') or ++ (stack[-1] == '[' and char == ']') or ++ (stack[-1] == '{' and char == '}')): ++ stack.pop() ++ if not stack: ++ return (i + 1, None) ++ else: ++ # Mismatched parentheses ++ return (-1, None) ++ elif char == '>': ++ # Found potential end of template argument list. ++ ++ # Ignore "->" and operator functions ++ if (i > 0 and ++ (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): ++ continue ++ ++ # Pop the stack if there is a matching '<'. Otherwise, ignore ++ # this '>' since it must be an operator. ++ if stack: ++ if stack[-1] == '<': ++ stack.pop() ++ if not stack: ++ return (i + 1, None) ++ elif char == ';': ++ # Found something that look like end of statements. If we are currently ++ # expecting a '>', the matching '<' must have been an operator, since ++ # template argument list should not contain statements. ++ while stack and stack[-1] == '<': ++ stack.pop() ++ if not stack: ++ return (-1, None) ++ ++ # Did not find end of expression or unbalanced parentheses on this line ++ return (-1, stack) ++ ++ ++def CloseExpression(clean_lines, linenum, pos): ++ """If input points to ( or { or [ or <, finds the position that closes it. ++ ++ If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the ++ linenum/pos that correspond to the closing of the expression. ++ ++ TODO(unknown): cpplint spends a fair bit of time matching parentheses. ++ Ideally we would want to index all opening and closing parentheses once ++ and have CloseExpression be just a simple lookup, but due to preprocessor ++ tricks, this is not so easy. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ pos: A position on the line. ++ ++ Returns: ++ A tuple (line, linenum, pos) pointer *past* the closing brace, or ++ (line, len(lines), -1) if we never find a close. Note we ignore ++ strings and comments when matching; and the line we return is the ++ 'cleansed' line at linenum. ++ """ ++ ++ line = clean_lines.elided[linenum] ++ if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): ++ return (line, clean_lines.NumLines(), -1) ++ ++ # Check first line ++ (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) ++ if end_pos > -1: ++ return (line, linenum, end_pos) ++ ++ # Continue scanning forward ++ while stack and linenum < clean_lines.NumLines() - 1: ++ linenum += 1 ++ line = clean_lines.elided[linenum] ++ (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) ++ if end_pos > -1: ++ return (line, linenum, end_pos) ++ ++ # Did not find end of expression before end of file, give up ++ return (line, clean_lines.NumLines(), -1) ++ ++ ++def FindStartOfExpressionInLine(line, endpos, stack): ++ """Find position at the matching start of current expression. ++ ++ This is almost the reverse of FindEndOfExpressionInLine, but note ++ that the input position and returned position differs by 1. ++ ++ Args: ++ line: a CleansedLines line. ++ endpos: start searching at this position. ++ stack: nesting stack at endpos. ++ ++ Returns: ++ On finding matching start: (index at matching start, None) ++ On finding an unclosed expression: (-1, None) ++ Otherwise: (-1, new stack at beginning of this line) ++ """ ++ i = endpos ++ while i >= 0: ++ char = line[i] ++ if char in ')]}': ++ # Found end of expression, push to expression stack ++ stack.append(char) ++ elif char == '>': ++ # Found potential end of template argument list. ++ # ++ # Ignore it if it's a "->" or ">=" or "operator>" ++ if (i > 0 and ++ (line[i - 1] == '-' or ++ Match(r'\s>=\s', line[i - 1:]) or ++ Search(r'\boperator\s*$', line[0:i]))): ++ i -= 1 ++ else: ++ stack.append('>') ++ elif char == '<': ++ # Found potential start of template argument list ++ if i > 0 and line[i - 1] == '<': ++ # Left shift operator ++ i -= 1 ++ else: ++ # If there is a matching '>', we can pop the expression stack. ++ # Otherwise, ignore this '<' since it must be an operator. ++ if stack and stack[-1] == '>': ++ stack.pop() ++ if not stack: ++ return (i, None) ++ elif char in '([{': ++ # Found start of expression. ++ # ++ # If there are any unmatched '>' on the stack, they must be ++ # operators. Remove those. ++ while stack and stack[-1] == '>': ++ stack.pop() ++ if not stack: ++ return (-1, None) ++ if ((char == '(' and stack[-1] == ')') or ++ (char == '[' and stack[-1] == ']') or ++ (char == '{' and stack[-1] == '}')): ++ stack.pop() ++ if not stack: ++ return (i, None) ++ else: ++ # Mismatched parentheses ++ return (-1, None) ++ elif char == ';': ++ # Found something that look like end of statements. If we are currently ++ # expecting a '<', the matching '>' must have been an operator, since ++ # template argument list should not contain statements. ++ while stack and stack[-1] == '>': ++ stack.pop() ++ if not stack: ++ return (-1, None) ++ ++ i -= 1 ++ ++ return (-1, stack) ++ ++ ++def ReverseCloseExpression(clean_lines, linenum, pos): ++ """If input points to ) or } or ] or >, finds the position that opens it. ++ ++ If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the ++ linenum/pos that correspond to the opening of the expression. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ pos: A position on the line. ++ ++ Returns: ++ A tuple (line, linenum, pos) pointer *at* the opening brace, or ++ (line, 0, -1) if we never find the matching opening brace. Note ++ we ignore strings and comments when matching; and the line we ++ return is the 'cleansed' line at linenum. ++ """ ++ line = clean_lines.elided[linenum] ++ if line[pos] not in ')}]>': ++ return (line, 0, -1) ++ ++ # Check last line ++ (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) ++ if start_pos > -1: ++ return (line, linenum, start_pos) ++ ++ # Continue scanning backward ++ while stack and linenum > 0: ++ linenum -= 1 ++ line = clean_lines.elided[linenum] ++ (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) ++ if start_pos > -1: ++ return (line, linenum, start_pos) ++ ++ # Did not find start of expression before beginning of file, give up ++ return (line, 0, -1) ++ ++def make_polyfill_regex(): ++ polyfill_required_names = [ ++ '_', ++ 'adopt_lock', ++ 'async', ++ 'chrono', ++ 'condition_variable', ++ 'condition_variable_any', ++ 'cv_status', ++ 'defer_lock', ++ 'future', ++ 'future_status', ++ 'launch', ++ 'lock_guard', ++ 'mutex', ++ 'notify_all_at_thread_exit', ++ 'packaged_task', ++ 'promise', ++ 'recursive_mutex', ++ 'shared_lock,', ++ 'shared_mutex', ++ 'shared_timed_mutex', ++ 'this_thread(?!::at_thread_exit)', ++ 'thread', ++ 'timed_mutex', ++ 'try_to_lock', ++ 'unique_lock', ++ 'unordered_map', ++ 'unordered_multimap', ++ 'unordered_multiset', ++ 'unordered_set', ++ ] ++ ++ qualified_names = ['boost::' + name + "\\b" for name in polyfill_required_names] ++ qualified_names.extend('std::' + name + "\\b" for name in polyfill_required_names) ++ qualified_names_regex = '|'.join(qualified_names) ++ return re.compile(qualified_names_regex) ++_RE_PATTERN_MONGO_POLYFILL=make_polyfill_regex() ++ ++def CheckForMongoPolyfill(filename, clean_lines, linenum, error): ++ line = clean_lines.elided[linenum] ++ if re.search(_RE_PATTERN_MONGO_POLYFILL, line): ++ error(filename, linenum, 'mongodb/polyfill', 5, ++ 'Illegal use of banned name from std::/boost::, use mongo::stdx:: variant instead') ++ ++def CheckForMongoAtomic(filename, clean_lines, linenum, error): ++ line = clean_lines.elided[linenum] ++ if re.search('std::atomic', line): ++ error(filename, linenum, 'mongodb/stdatomic', 5, ++ 'Illegal use of prohibited std::atomic, use AtomicWord or other types ' ++ 'from "mongo/platform/atomic_word.h"') ++ ++def CheckForMongoVolatile(filename, clean_lines, linenum, error): ++ line = clean_lines.elided[linenum] ++ if re.search('[^_]volatile', line) and not "__asm__" in line: ++ error(filename, linenum, 'mongodb/volatile', 5, ++ 'Illegal use of the volatile storage keyword, use AtomicWord instead ' ++ 'from "mongo/platform/atomic_word.h"') ++ ++def CheckForNonMongoAssert(filename, clean_lines, linenum, error): ++ line = clean_lines.elided[linenum] ++ if re.search(r'\bassert\s*\(', line): ++ error(filename, linenum, 'mongodb/assert', 5, ++ 'Illegal use of the bare assert function, use a function from assert_utils.h instead.') ++ ++def CheckForCopyright(filename, lines, error): ++ """Logs an error if no Copyright message appears at the top of the file.""" ++ ++ # We'll say it should occur by line 10. Don't forget there's a ++ # dummy line at the front. ++ for line in xrange(1, min(len(lines), 11)): ++ if re.search(r'Copyright', lines[line], re.I): ++ CheckForServerSidePublicLicense(line, filename, lines, error) ++ break ++ else: # means no copyright line was found ++ error(filename, 0, 'legal/copyright', 5, ++ 'No copyright message found. ' ++ 'You should have a line: "Copyright [year] "') ++ ++def CheckForServerSidePublicLicense(copyright_offset, filename, lines, error): ++ license_header = '''\ ++ * This program is free software: you can redistribute it and/or modify ++ * it under the terms of the Server Side Public License, version 1, ++ * as published by MongoDB, Inc. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * Server Side Public License for more details. ++ * ++ * You should have received a copy of the Server Side Public License ++ * along with this program. If not, see ++ * . ++ * ++ * As a special exception, the copyright holders give permission to link the ++ * code of portions of this program with the OpenSSL library under certain ++ * conditions as described in each individual source file and distribute ++ * linked combinations including the program with the OpenSSL library. You ++ * must comply with the Server Side Public License in all respects for ++ * all of the code used other than as permitted herein. If you modify file(s) ++ * with this exception, you may extend this exception to your version of the ++ * file(s), but you are not obligated to do so. If you do not wish to do so, ++ * delete this exception statement from your version. If you delete this ++ * exception statement from all source files in the program, then also delete ++ * it in the license file. ++ */'''.splitlines() ++ ++ # The following files are in the src/mongo/ directory but technically belong ++ # in src/third_party/ because their copyright does not belong to MongoDB. Note ++ # that we do not need to use os.path.normpath() to match these pathnames on ++ # Windows because FileInfo.RepositoryName() normalizes the path separator for ++ # us already. ++ files_to_ignore = set([ ++ 'src/mongo/shell/linenoise.cpp', ++ 'src/mongo/shell/linenoise.h', ++ 'src/mongo/shell/mk_wcwidth.cpp', ++ 'src/mongo/shell/mk_wcwidth.h', ++ 'src/mongo/util/md5.cpp', ++ 'src/mongo/util/md5.h', ++ 'src/mongo/util/md5main.cpp', ++ 'src/mongo/util/net/ssl_stream.cpp', ++ 'src/mongo/util/scopeguard.h', ++ ]) ++ ++ if FileInfo(filename).RepositoryName() in files_to_ignore: ++ return ++ ++ # We expect the first line of the license header to follow shortly after the ++ # "Copyright" message. ++ for line in xrange(copyright_offset, min(len(lines), copyright_offset + 3)): ++ if re.search(r'This program is free software', lines[line]): ++ license_header_start_line = line ++ for i in xrange(len(license_header)): ++ line = i + license_header_start_line ++ if line >= len(lines) or lines[line] != license_header[i]: ++ error(filename, 0, 'legal/license', 5, ++ 'Incorrect license header found. ' ++ 'Expected "' + license_header[i] + '". ' ++ 'See https://github.com/mongodb/mongo/wiki/Server-Code-Style') ++ # We break here to stop reporting legal/license errors for this file. ++ break ++ ++ # We break here to indicate that we found some license header. ++ break ++ else: ++ error(filename, 0, 'legal/license', 5, ++ 'No license header found. ' ++ 'See https://github.com/mongodb/mongo/wiki/Server-Code-Style') ++ ++def GetIndentLevel(line): ++ """Return the number of leading spaces in line. ++ ++ Args: ++ line: A string to check. ++ ++ Returns: ++ An integer count of leading spaces, possibly zero. ++ """ ++ indent = Match(r'^( *)\S', line) ++ if indent: ++ return len(indent.group(1)) ++ else: ++ return 0 ++ ++ ++def GetHeaderGuardCPPVariable(filename): ++ """Returns the CPP variable that should be used as a header guard. ++ ++ Args: ++ filename: The name of a C++ header file. ++ ++ Returns: ++ The CPP variable that should be used as a header guard in the ++ named file. ++ ++ """ ++ ++ # Restores original filename in case that cpplint is invoked from Emacs's ++ # flymake. ++ filename = re.sub(r'_flymake\.h$', '.h', filename) ++ filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) ++ ++ fileinfo = FileInfo(filename) ++ file_path_from_root = fileinfo.RepositoryName() ++ if _root: ++ file_path_from_root = re.sub('^' + _root + os.sep, '', file_path_from_root) ++ return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' ++ ++ ++def CheckForHeaderGuard(filename, lines, error): ++ """Checks that the file contains a header guard. ++ ++ Logs an error if no #ifndef header guard is present. For other ++ headers, checks that the full pathname is used. ++ ++ Args: ++ filename: The name of the C++ header file. ++ lines: An array of strings, each representing a line of the file. ++ error: The function to call with any errors found. ++ """ ++ ++ # Don't check for header guards if there are error suppression ++ # comments somewhere in this file. ++ # ++ # Because this is silencing a warning for a nonexistent line, we ++ # only support the very specific NOLINT(build/header_guard) syntax, ++ # and not the general NOLINT or NOLINT(*) syntax. ++ for i in lines: ++ if Search(r'//\s*NOLINT\(build/header_guard\)', i): ++ return ++ ++ cppvar = GetHeaderGuardCPPVariable(filename) ++ ++ ifndef = None ++ ifndef_linenum = 0 ++ define = None ++ endif = None ++ endif_linenum = 0 ++ for linenum, line in enumerate(lines): ++ linesplit = line.split() ++ if len(linesplit) >= 2: ++ # find the first occurrence of #ifndef and #define, save arg ++ if not ifndef and linesplit[0] == '#ifndef': ++ # set ifndef to the header guard presented on the #ifndef line. ++ ifndef = linesplit[1] ++ ifndef_linenum = linenum ++ if not define and linesplit[0] == '#define': ++ define = linesplit[1] ++ # find the last occurrence of #endif, save entire line ++ if line.startswith('#endif'): ++ endif = line ++ endif_linenum = linenum ++ ++ if not ifndef: ++ error(filename, 0, 'build/header_guard', 5, ++ 'No #ifndef header guard found, suggested CPP variable is: %s' % ++ cppvar) ++ return ++ ++ if not define: ++ error(filename, 0, 'build/header_guard', 5, ++ 'No #define header guard found, suggested CPP variable is: %s' % ++ cppvar) ++ return ++ ++ # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ ++ # for backward compatibility. ++ if ifndef != cppvar: ++ error_level = 0 ++ if ifndef != cppvar + '_': ++ error_level = 5 ++ ++ ParseNolintSuppressions(filename, lines[ifndef_linenum], ifndef_linenum, ++ error) ++ error(filename, ifndef_linenum, 'build/header_guard', error_level, ++ '#ifndef header guard has wrong style, please use: %s' % cppvar) ++ ++ if define != ifndef: ++ error(filename, 0, 'build/header_guard', 5, ++ '#ifndef and #define don\'t match, suggested CPP variable is: %s' % ++ cppvar) ++ return ++ ++ if endif != ('#endif // %s' % cppvar): ++ error_level = 0 ++ if endif != ('#endif // %s' % (cppvar + '_')): ++ error_level = 5 ++ ++ ParseNolintSuppressions(filename, lines[endif_linenum], endif_linenum, ++ error) ++ error(filename, endif_linenum, 'build/header_guard', error_level, ++ '#endif line should be "#endif // %s"' % cppvar) ++ ++ ++def CheckForBadCharacters(filename, lines, error): ++ """Logs an error for each line containing bad characters. ++ ++ Two kinds of bad characters: ++ ++ 1. Unicode replacement characters: These indicate that either the file ++ contained invalid UTF-8 (likely) or Unicode replacement characters (which ++ it shouldn't). Note that it's possible for this to throw off line ++ numbering if the invalid UTF-8 occurred adjacent to a newline. ++ ++ 2. NUL bytes. These are problematic for some tools. ++ ++ Args: ++ filename: The name of the current file. ++ lines: An array of strings, each representing a line of the file. ++ error: The function to call with any errors found. ++ """ ++ for linenum, line in enumerate(lines): ++ if u'\ufffd' in line: ++ error(filename, linenum, 'readability/utf8', 5, ++ 'Line contains invalid UTF-8 (or Unicode replacement character).') ++ if '\0' in line: ++ error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') ++ ++ ++def CheckForNewlineAtEOF(filename, lines, error): ++ """Logs an error if there is no newline char at the end of the file. ++ ++ Args: ++ filename: The name of the current file. ++ lines: An array of strings, each representing a line of the file. ++ error: The function to call with any errors found. ++ """ ++ ++ # The array lines() was created by adding two newlines to the ++ # original file (go figure), then splitting on \n. ++ # To verify that the file ends in \n, we just have to make sure the ++ # last-but-two element of lines() exists and is empty. ++ if len(lines) < 3 or lines[-2]: ++ error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, ++ 'Could not find a newline character at the end of the file.') ++ ++ ++def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): ++ """Logs an error if we see /* ... */ or "..." that extend past one line. ++ ++ /* ... */ comments are legit inside macros, for one line. ++ Otherwise, we prefer // comments, so it's ok to warn about the ++ other. Likewise, it's ok for strings to extend across multiple ++ lines, as long as a line continuation character (backslash) ++ terminates each line. Although not currently prohibited by the C++ ++ style guide, it's ugly and unnecessary. We don't do well with either ++ in this lint program, so we warn about both. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Remove all \\ (escaped backslashes) from the line. They are OK, and the ++ # second (escaped) slash may trigger later \" detection erroneously. ++ line = line.replace('\\\\', '') ++ ++ if line.count('/*') > line.count('*/'): ++ error(filename, linenum, 'readability/multiline_comment', 5, ++ 'Complex multi-line /*...*/-style comment found. ' ++ 'Lint may give bogus warnings. ' ++ 'Consider replacing these with //-style comments, ' ++ 'with #if 0...#endif, ' ++ 'or with more clearly structured multi-line comments.') ++ ++ if (line.count('"') - line.count('\\"')) % 2: ++ error(filename, linenum, 'readability/multiline_string', 5, ++ 'Multi-line string ("...") found. This lint script doesn\'t ' ++ 'do well with such strings, and may give bogus warnings. ' ++ 'Use C++11 raw strings or concatenation instead.') ++ ++ ++# (non-threadsafe name, thread-safe alternative, validation pattern) ++# ++# The validation pattern is used to eliminate false positives such as: ++# _rand(); // false positive due to substring match. ++# ->rand(); // some member function rand(). ++# ACMRandom rand(seed); // some variable named rand. ++# ISAACRandom rand(); // another variable named rand. ++# ++# Basically we require the return value of these functions to be used ++# in some expression context on the same line by matching on some ++# operator before the function name. This eliminates constructors and ++# member function calls. ++_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)' ++_THREADING_LIST = ( ++ ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'), ++ ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'), ++ ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'), ++ ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'), ++ ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'), ++ ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'), ++ ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'), ++ ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), ++ ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), ++ ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), ++ ('strtok(', 'strtok_r(', ++ _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), ++ ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), ++ ) ++ ++ ++def CheckPosixThreading(filename, clean_lines, linenum, error): ++ """Checks for calls to thread-unsafe functions. ++ ++ Much code has been originally written without consideration of ++ multi-threading. Also, engineers are relying on their old experience; ++ they have learned posix before threading extensions were added. These ++ tests guide the engineers to use thread-safe functions (when using ++ posix directly). ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: ++ # Additional pattern matching check to confirm that this is the ++ # function we are looking for ++ if Search(pattern, line): ++ error(filename, linenum, 'runtime/threadsafe_fn', 2, ++ 'Consider using ' + multithread_safe_func + ++ '...) instead of ' + single_thread_func + ++ '...) for improved thread safety.') ++ ++ ++def CheckVlogArguments(filename, clean_lines, linenum, error): ++ """Checks that VLOG() is only used for defining a logging level. ++ ++ For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and ++ VLOG(FATAL) are not. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): ++ error(filename, linenum, 'runtime/vlog', 5, ++ 'VLOG() should be used with numeric verbosity level. ' ++ 'Use LOG() if you want symbolic severity levels.') ++ ++# Matches invalid increment: *count++, which moves pointer instead of ++# incrementing a value. ++_RE_PATTERN_INVALID_INCREMENT = re.compile( ++ r'^\s*\*\w+(\+\+|--);') ++ ++ ++def CheckInvalidIncrement(filename, clean_lines, linenum, error): ++ """Checks for invalid increment *count++. ++ ++ For example following function: ++ void increment_counter(int* count) { ++ *count++; ++ } ++ is invalid, because it effectively does count++, moving pointer, and should ++ be replaced with ++*count, (*count)++ or *count += 1. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ if _RE_PATTERN_INVALID_INCREMENT.match(line): ++ error(filename, linenum, 'runtime/invalid_increment', 5, ++ 'Changing pointer instead of value (or unused value of operator*).') ++ ++ ++def IsMacroDefinition(clean_lines, linenum): ++ if Search(r'^#define', clean_lines[linenum]): ++ return True ++ ++ if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): ++ return True ++ ++ return False ++ ++ ++def IsForwardClassDeclaration(clean_lines, linenum): ++ return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) ++ ++ ++class _BlockInfo(object): ++ """Stores information about a generic block of code.""" ++ ++ def __init__(self, seen_open_brace): ++ self.seen_open_brace = seen_open_brace ++ self.open_parentheses = 0 ++ self.inline_asm = _NO_ASM ++ self.check_namespace_indentation = False ++ ++ def CheckBegin(self, filename, clean_lines, linenum, error): ++ """Run checks that applies to text up to the opening brace. ++ ++ This is mostly for checking the text after the class identifier ++ and the "{", usually where the base class is specified. For other ++ blocks, there isn't much to check, so we always pass. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ pass ++ ++ def CheckEnd(self, filename, clean_lines, linenum, error): ++ """Run checks that applies to text after the closing brace. ++ ++ This is mostly used for checking end of namespace comments. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ pass ++ ++ def IsBlockInfo(self): ++ """Returns true if this block is a _BlockInfo. ++ ++ This is convenient for verifying that an object is an instance of ++ a _BlockInfo, but not an instance of any of the derived classes. ++ ++ Returns: ++ True for this class, False for derived classes. ++ """ ++ return self.__class__ == _BlockInfo ++ ++ ++class _ExternCInfo(_BlockInfo): ++ """Stores information about an 'extern "C"' block.""" ++ ++ def __init__(self): ++ _BlockInfo.__init__(self, True) ++ ++ ++class _ClassInfo(_BlockInfo): ++ """Stores information about a class.""" ++ ++ def __init__(self, name, class_or_struct, clean_lines, linenum): ++ _BlockInfo.__init__(self, False) ++ self.name = name ++ self.starting_linenum = linenum ++ self.is_derived = False ++ self.check_namespace_indentation = True ++ if class_or_struct == 'struct': ++ self.access = 'public' ++ self.is_struct = True ++ else: ++ self.access = 'private' ++ self.is_struct = False ++ ++ # Remember initial indentation level for this class. Using raw_lines here ++ # instead of elided to account for leading comments. ++ self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) ++ ++ # Try to find the end of the class. This will be confused by things like: ++ # class A { ++ # } *x = { ... ++ # ++ # But it's still good enough for CheckSectionSpacing. ++ self.last_line = 0 ++ depth = 0 ++ for i in range(linenum, clean_lines.NumLines()): ++ line = clean_lines.elided[i] ++ depth += line.count('{') - line.count('}') ++ if not depth: ++ self.last_line = i ++ break ++ ++ def CheckBegin(self, filename, clean_lines, linenum, error): ++ # Look for a bare ':' ++ if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): ++ self.is_derived = True ++ ++ def CheckEnd(self, filename, clean_lines, linenum, error): ++ # Check that closing brace is aligned with beginning of the class. ++ # Only do this if the closing brace is indented by only whitespaces. ++ # This means we will not check single-line class definitions. ++ indent = Match(r'^( *)\}', clean_lines.elided[linenum]) ++ if indent and len(indent.group(1)) != self.class_indent: ++ if self.is_struct: ++ parent = 'struct ' + self.name ++ else: ++ parent = 'class ' + self.name ++ error(filename, linenum, 'whitespace/indent', 3, ++ 'Closing brace should be aligned with beginning of %s' % parent) ++ ++ ++class _NamespaceInfo(_BlockInfo): ++ """Stores information about a namespace.""" ++ ++ def __init__(self, name, linenum): ++ _BlockInfo.__init__(self, False) ++ self.name = name or '' ++ self.starting_linenum = linenum ++ self.check_namespace_indentation = True ++ ++ def CheckEnd(self, filename, clean_lines, linenum, error): ++ """Check end of namespace comments.""" ++ line = clean_lines.raw_lines[linenum] ++ ++ # Check how many lines is enclosed in this namespace. Don't issue ++ # warning for missing namespace comments if there aren't enough ++ # lines. However, do apply checks if there is already an end of ++ # namespace comment and it's incorrect. ++ # ++ # TODO(unknown): We always want to check end of namespace comments ++ # if a namespace is large, but sometimes we also want to apply the ++ # check if a short namespace contained nontrivial things (something ++ # other than forward declarations). There is currently no logic on ++ # deciding what these nontrivial things are, so this check is ++ # triggered by namespace size only, which works most of the time. ++ if (linenum - self.starting_linenum < 10 ++ and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): ++ return ++ ++ # Look for matching comment at end of namespace. ++ # ++ # Note that we accept C style "/* */" comments for terminating ++ # namespaces, so that code that terminate namespaces inside ++ # preprocessor macros can be cpplint clean. ++ # ++ # We also accept stuff like "// end of namespace ." with the ++ # period at the end. ++ # ++ # Besides these, we don't accept anything else, otherwise we might ++ # get false negatives when existing comment is a substring of the ++ # expected namespace. ++ if self.name: ++ # Named namespace ++ if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) + ++ r'[\*/\.\\\s]*$'), ++ line): ++ error(filename, linenum, 'readability/namespace', 5, ++ 'Namespace should be terminated with "// namespace %s"' % ++ self.name) ++ else: ++ # Anonymous namespace ++ if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): ++ # If "// namespace anonymous" or "// anonymous namespace (more text)", ++ # mention "// anonymous namespace" as an acceptable form ++ if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line): ++ error(filename, linenum, 'readability/namespace', 5, ++ 'Anonymous namespace should be terminated with "// namespace"' ++ ' or "// anonymous namespace"') ++ else: ++ error(filename, linenum, 'readability/namespace', 5, ++ 'Anonymous namespace should be terminated with "// namespace"') ++ ++ ++class _PreprocessorInfo(object): ++ """Stores checkpoints of nesting stacks when #if/#else is seen.""" ++ ++ def __init__(self, stack_before_if): ++ # The entire nesting stack before #if ++ self.stack_before_if = stack_before_if ++ ++ # The entire nesting stack up to #else ++ self.stack_before_else = [] ++ ++ # Whether we have already seen #else or #elif ++ self.seen_else = False ++ ++ ++class NestingState(object): ++ """Holds states related to parsing braces.""" ++ ++ def __init__(self): ++ # Stack for tracking all braces. An object is pushed whenever we ++ # see a "{", and popped when we see a "}". Only 3 types of ++ # objects are possible: ++ # - _ClassInfo: a class or struct. ++ # - _NamespaceInfo: a namespace. ++ # - _BlockInfo: some other type of block. ++ self.stack = [] ++ ++ # Top of the previous stack before each Update(). ++ # ++ # Because the nesting_stack is updated at the end of each line, we ++ # had to do some convoluted checks to find out what is the current ++ # scope at the beginning of the line. This check is simplified by ++ # saving the previous top of nesting stack. ++ # ++ # We could save the full stack, but we only need the top. Copying ++ # the full nesting stack would slow down cpplint by ~10%. ++ self.previous_stack_top = [] ++ ++ # Stack of _PreprocessorInfo objects. ++ self.pp_stack = [] ++ ++ def SeenOpenBrace(self): ++ """Check if we have seen the opening brace for the innermost block. ++ ++ Returns: ++ True if we have seen the opening brace, False if the innermost ++ block is still expecting an opening brace. ++ """ ++ return (not self.stack) or self.stack[-1].seen_open_brace ++ ++ def InNamespaceBody(self): ++ """Check if we are currently one level inside a namespace body. ++ ++ Returns: ++ True if top of the stack is a namespace block, False otherwise. ++ """ ++ return self.stack and isinstance(self.stack[-1], _NamespaceInfo) ++ ++ def InExternC(self): ++ """Check if we are currently one level inside an 'extern "C"' block. ++ ++ Returns: ++ True if top of the stack is an extern block, False otherwise. ++ """ ++ return self.stack and isinstance(self.stack[-1], _ExternCInfo) ++ ++ def InClassDeclaration(self): ++ """Check if we are currently one level inside a class or struct declaration. ++ ++ Returns: ++ True if top of the stack is a class/struct, False otherwise. ++ """ ++ return self.stack and isinstance(self.stack[-1], _ClassInfo) ++ ++ def InAsmBlock(self): ++ """Check if we are currently one level inside an inline ASM block. ++ ++ Returns: ++ True if the top of the stack is a block containing inline ASM. ++ """ ++ return self.stack and self.stack[-1].inline_asm != _NO_ASM ++ ++ def InTemplateArgumentList(self, clean_lines, linenum, pos): ++ """Check if current position is inside template argument list. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ pos: position just after the suspected template argument. ++ Returns: ++ True if (linenum, pos) is inside template arguments. ++ """ ++ while linenum < clean_lines.NumLines(): ++ # Find the earliest character that might indicate a template argument ++ line = clean_lines.elided[linenum] ++ match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) ++ if not match: ++ linenum += 1 ++ pos = 0 ++ continue ++ token = match.group(1) ++ pos += len(match.group(0)) ++ ++ # These things do not look like template argument list: ++ # class Suspect { ++ # class Suspect x; } ++ if token in ('{', '}', ';'): return False ++ ++ # These things look like template argument list: ++ # template ++ # template ++ # template ++ # template ++ if token in ('>', '=', '[', ']', '.'): return True ++ ++ # Check if token is an unmatched '<'. ++ # If not, move on to the next character. ++ if token != '<': ++ pos += 1 ++ if pos >= len(line): ++ linenum += 1 ++ pos = 0 ++ continue ++ ++ # We can't be sure if we just find a single '<', and need to ++ # find the matching '>'. ++ (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) ++ if end_pos < 0: ++ # Not sure if template argument list or syntax error in file ++ return False ++ linenum = end_line ++ pos = end_pos ++ return False ++ ++ def UpdatePreprocessor(self, line): ++ """Update preprocessor stack. ++ ++ We need to handle preprocessors due to classes like this: ++ #ifdef SWIG ++ struct ResultDetailsPageElementExtensionPoint { ++ #else ++ struct ResultDetailsPageElementExtensionPoint : public Extension { ++ #endif ++ ++ We make the following assumptions (good enough for most files): ++ - Preprocessor condition evaluates to true from #if up to first ++ #else/#elif/#endif. ++ ++ - Preprocessor condition evaluates to false from #else/#elif up ++ to #endif. We still perform lint checks on these lines, but ++ these do not affect nesting stack. ++ ++ Args: ++ line: current line to check. ++ """ ++ if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): ++ # Beginning of #if block, save the nesting stack here. The saved ++ # stack will allow us to restore the parsing state in the #else case. ++ self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) ++ elif Match(r'^\s*#\s*(else|elif)\b', line): ++ # Beginning of #else block ++ if self.pp_stack: ++ if not self.pp_stack[-1].seen_else: ++ # This is the first #else or #elif block. Remember the ++ # whole nesting stack up to this point. This is what we ++ # keep after the #endif. ++ self.pp_stack[-1].seen_else = True ++ self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) ++ ++ # Restore the stack to how it was before the #if ++ self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) ++ else: ++ # TODO(unknown): unexpected #else, issue warning? ++ pass ++ elif Match(r'^\s*#\s*endif\b', line): ++ # End of #if or #else blocks. ++ if self.pp_stack: ++ # If we saw an #else, we will need to restore the nesting ++ # stack to its former state before the #else, otherwise we ++ # will just continue from where we left off. ++ if self.pp_stack[-1].seen_else: ++ # Here we can just use a shallow copy since we are the last ++ # reference to it. ++ self.stack = self.pp_stack[-1].stack_before_else ++ # Drop the corresponding #if ++ self.pp_stack.pop() ++ else: ++ # TODO(unknown): unexpected #endif, issue warning? ++ pass ++ ++ # TODO(unknown): Update() is too long, but we will refactor later. ++ def Update(self, filename, clean_lines, linenum, error): ++ """Update nesting state with current line. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Remember top of the previous nesting stack. ++ # ++ # The stack is always pushed/popped and not modified in place, so ++ # we can just do a shallow copy instead of copy.deepcopy. Using ++ # deepcopy would slow down cpplint by ~28%. ++ if self.stack: ++ self.previous_stack_top = self.stack[-1] ++ else: ++ self.previous_stack_top = None ++ ++ # Update pp_stack ++ self.UpdatePreprocessor(line) ++ ++ # Count parentheses. This is to avoid adding struct arguments to ++ # the nesting stack. ++ if self.stack: ++ inner_block = self.stack[-1] ++ depth_change = line.count('(') - line.count(')') ++ inner_block.open_parentheses += depth_change ++ ++ # Also check if we are starting or ending an inline assembly block. ++ if inner_block.inline_asm in (_NO_ASM, _END_ASM): ++ if (depth_change != 0 and ++ inner_block.open_parentheses == 1 and ++ _MATCH_ASM.match(line)): ++ # Enter assembly block ++ inner_block.inline_asm = _INSIDE_ASM ++ else: ++ # Not entering assembly block. If previous line was _END_ASM, ++ # we will now shift to _NO_ASM state. ++ inner_block.inline_asm = _NO_ASM ++ elif (inner_block.inline_asm == _INSIDE_ASM and ++ inner_block.open_parentheses == 0): ++ # Exit assembly block ++ inner_block.inline_asm = _END_ASM ++ ++ # Consume namespace declaration at the beginning of the line. Do ++ # this in a loop so that we catch same line declarations like this: ++ # namespace proto2 { namespace bridge { class MessageSet; } } ++ while True: ++ # Match start of namespace. The "\b\s*" below catches namespace ++ # declarations even if it weren't followed by a whitespace, this ++ # is so that we don't confuse our namespace checker. The ++ # missing spaces will be flagged by CheckSpacing. ++ namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) ++ if not namespace_decl_match: ++ break ++ ++ new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) ++ self.stack.append(new_namespace) ++ ++ line = namespace_decl_match.group(2) ++ if line.find('{') != -1: ++ new_namespace.seen_open_brace = True ++ line = line[line.find('{') + 1:] ++ ++ # Look for a class declaration in whatever is left of the line ++ # after parsing namespaces. The regexp accounts for decorated classes ++ # such as in: ++ # class LOCKABLE API Object { ++ # }; ++ class_decl_match = Match( ++ r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' ++ r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' ++ r'(.*)$', line) ++ if (class_decl_match and ++ (not self.stack or self.stack[-1].open_parentheses == 0)): ++ # We do not want to accept classes that are actually template arguments: ++ # template , ++ # template class Ignore3> ++ # void Function() {}; ++ # ++ # To avoid template argument cases, we scan forward and look for ++ # an unmatched '>'. If we see one, assume we are inside a ++ # template argument list. ++ end_declaration = len(class_decl_match.group(1)) ++ if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): ++ self.stack.append(_ClassInfo( ++ class_decl_match.group(3), class_decl_match.group(2), ++ clean_lines, linenum)) ++ line = class_decl_match.group(4) ++ ++ # If we have not yet seen the opening brace for the innermost block, ++ # run checks here. ++ if not self.SeenOpenBrace(): ++ self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) ++ ++ # Update access control if we are inside a class/struct ++ if self.stack and isinstance(self.stack[-1], _ClassInfo): ++ classinfo = self.stack[-1] ++ access_match = Match( ++ r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' ++ r':(?:[^:]|$)', ++ line) ++ if access_match: ++ classinfo.access = access_match.group(2) ++ ++ # Check that access keywords are indented +1 space. Skip this ++ # check if the keywords are not preceded by whitespaces. ++ indent = access_match.group(1) ++ if (len(indent) != classinfo.class_indent + 1 and ++ Match(r'^\s*$', indent)): ++ if classinfo.is_struct: ++ parent = 'struct ' + classinfo.name ++ else: ++ parent = 'class ' + classinfo.name ++ slots = '' ++ if access_match.group(3): ++ slots = access_match.group(3) ++ error(filename, linenum, 'whitespace/indent', 3, ++ '%s%s: should be indented +1 space inside %s' % ( ++ access_match.group(2), slots, parent)) ++ ++ # Consume braces or semicolons from what's left of the line ++ while True: ++ # Match first brace, semicolon, or closed parenthesis. ++ matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) ++ if not matched: ++ break ++ ++ token = matched.group(1) ++ if token == '{': ++ # If namespace or class hasn't seen a opening brace yet, mark ++ # namespace/class head as complete. Push a new block onto the ++ # stack otherwise. ++ if not self.SeenOpenBrace(): ++ self.stack[-1].seen_open_brace = True ++ elif Match(r'^extern\s*"[^"]*"\s*\{', line): ++ self.stack.append(_ExternCInfo()) ++ else: ++ self.stack.append(_BlockInfo(True)) ++ if _MATCH_ASM.match(line): ++ self.stack[-1].inline_asm = _BLOCK_ASM ++ ++ elif token == ';' or token == ')': ++ # If we haven't seen an opening brace yet, but we already saw ++ # a semicolon, this is probably a forward declaration. Pop ++ # the stack for these. ++ # ++ # Similarly, if we haven't seen an opening brace yet, but we ++ # already saw a closing parenthesis, then these are probably ++ # function arguments with extra "class" or "struct" keywords. ++ # Also pop these stack for these. ++ if not self.SeenOpenBrace(): ++ self.stack.pop() ++ else: # token == '}' ++ # Perform end of block checks and pop the stack. ++ if self.stack: ++ self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) ++ self.stack.pop() ++ line = matched.group(2) ++ ++ def InnermostClass(self): ++ """Get class info on the top of the stack. ++ ++ Returns: ++ A _ClassInfo object if we are inside a class, or None otherwise. ++ """ ++ for i in range(len(self.stack), 0, -1): ++ classinfo = self.stack[i - 1] ++ if isinstance(classinfo, _ClassInfo): ++ return classinfo ++ return None ++ ++ def CheckCompletedBlocks(self, filename, error): ++ """Checks that all classes and namespaces have been completely parsed. ++ ++ Call this when all lines in a file have been processed. ++ Args: ++ filename: The name of the current file. ++ error: The function to call with any errors found. ++ """ ++ # Note: This test can result in false positives if #ifdef constructs ++ # get in the way of brace matching. See the testBuildClass test in ++ # cpplint_unittest.py for an example of this. ++ for obj in self.stack: ++ if isinstance(obj, _ClassInfo): ++ error(filename, obj.starting_linenum, 'build/class', 5, ++ 'Failed to find complete declaration of class %s' % ++ obj.name) ++ elif isinstance(obj, _NamespaceInfo): ++ error(filename, obj.starting_linenum, 'build/namespaces', 5, ++ 'Failed to find complete declaration of namespace %s' % ++ obj.name) ++ ++ ++def CheckForNonStandardConstructs(filename, clean_lines, linenum, ++ nesting_state, error): ++ r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. ++ ++ Complain about several constructs which gcc-2 accepts, but which are ++ not standard C++. Warning about these in lint is one way to ease the ++ transition to new compilers. ++ - put storage class first (e.g. "static const" instead of "const static"). ++ - "%lld" instead of %qd" in printf-type functions. ++ - "%1$d" is non-standard in printf-type functions. ++ - "\%" is an undefined character escape sequence. ++ - text after #endif is not allowed. ++ - invalid inner-style forward declaration. ++ - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', ++ line): ++ error(filename, linenum, 'build/deprecated', 3, ++ '>? and ))?' ++ # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' ++ error(filename, linenum, 'runtime/member_string_references', 2, ++ 'const string& members are dangerous. It is much better to use ' ++ 'alternatives, such as pointers or simple constants.') ++ ++ # Everything else in this function operates on class declarations. ++ # Return early if the top of the nesting stack is not a class, or if ++ # the class head is not completed yet. ++ classinfo = nesting_state.InnermostClass() ++ if not classinfo or not classinfo.seen_open_brace: ++ return ++ ++ # The class may have been declared with namespace or classname qualifiers. ++ # The constructor and destructor will not have those qualifiers. ++ base_classname = classinfo.name.split('::')[-1] ++ ++ # Look for single-argument constructors that aren't marked explicit. ++ # Technically a valid construct, but against style. Also look for ++ # non-single-argument constructors which are also technically valid, but ++ # strongly suggest something is wrong. ++ explicit_constructor_match = Match( ++ r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' ++ r'\(((?:[^()]|\([^()]*\))*)\)' ++ % re.escape(base_classname), ++ line) ++ ++ if explicit_constructor_match: ++ is_marked_explicit = explicit_constructor_match.group(1) ++ ++ if not explicit_constructor_match.group(2): ++ constructor_args = [] ++ else: ++ constructor_args = explicit_constructor_match.group(2).split(',') ++ ++ # collapse arguments so that commas in template parameter lists and function ++ # argument parameter lists don't split arguments in two ++ i = 0 ++ while i < len(constructor_args): ++ constructor_arg = constructor_args[i] ++ while (constructor_arg.count('<') > constructor_arg.count('>') or ++ constructor_arg.count('(') > constructor_arg.count(')')): ++ constructor_arg += ',' + constructor_args[i + 1] ++ del constructor_args[i + 1] ++ constructor_args[i] = constructor_arg ++ i += 1 ++ ++ defaulted_args = [arg for arg in constructor_args if '=' in arg] ++ noarg_constructor = (not constructor_args or # empty arg list ++ # 'void' arg specifier ++ (len(constructor_args) == 1 and ++ constructor_args[0].strip() == 'void')) ++ onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg ++ not noarg_constructor) or ++ # all but at most one arg defaulted ++ (len(constructor_args) >= 1 and ++ not noarg_constructor and ++ len(defaulted_args) >= len(constructor_args) - 1)) ++ initializer_list_constructor = bool( ++ onearg_constructor and ++ Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) ++ copy_constructor = bool( ++ onearg_constructor and ++ Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' ++ % re.escape(base_classname), constructor_args[0].strip())) ++ ++ if (not is_marked_explicit and ++ onearg_constructor and ++ not initializer_list_constructor and ++ not copy_constructor): ++ if defaulted_args: ++ error(filename, linenum, 'runtime/explicit', 5, ++ 'Constructors callable with one argument ' ++ 'should be marked explicit.') ++ else: ++ error(filename, linenum, 'runtime/explicit', 5, ++ 'Single-parameter constructors should be marked explicit.') ++ elif is_marked_explicit and not onearg_constructor: ++ if noarg_constructor: ++ error(filename, linenum, 'runtime/explicit', 5, ++ 'Zero-parameter constructors should not be marked explicit.') ++ else: ++ error(filename, linenum, 'runtime/explicit', 0, ++ 'Constructors that require multiple arguments ' ++ 'should not be marked explicit.') ++ ++ ++def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): ++ """Checks for the correctness of various spacing around function calls. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Since function calls often occur inside if/for/while/switch ++ # expressions - which have their own, more liberal conventions - we ++ # first see if we should be looking inside such an expression for a ++ # function call, to which we can apply more strict standards. ++ fncall = line # if there's no control flow construct, look at whole line ++ for pattern in (r'\bif\s*\((.*)\)\s*{', ++ r'\bfor\s*\((.*)\)\s*{', ++ r'\bwhile\s*\((.*)\)\s*[{;]', ++ r'\bswitch\s*\((.*)\)\s*{'): ++ match = Search(pattern, line) ++ if match: ++ fncall = match.group(1) # look inside the parens for function calls ++ break ++ ++ # Except in if/for/while/switch, there should never be space ++ # immediately inside parens (eg "f( 3, 4 )"). We make an exception ++ # for nested parens ( (a+b) + c ). Likewise, there should never be ++ # a space before a ( when it's a function argument. I assume it's a ++ # function argument when the char before the whitespace is legal in ++ # a function name (alnum + _) and we're not starting a macro. Also ignore ++ # pointers and references to arrays and functions coz they're too tricky: ++ # we use a very simple way to recognize these: ++ # " (something)(maybe-something)" or ++ # " (something)(maybe-something," or ++ # " (something)[something]" ++ # Note that we assume the contents of [] to be short enough that ++ # they'll never need to wrap. ++ if ( # Ignore control structures. ++ not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', ++ fncall) and ++ # Ignore pointers/references to functions. ++ not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and ++ # Ignore pointers/references to arrays. ++ not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): ++ if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call ++ error(filename, linenum, 'whitespace/parens', 4, ++ 'Extra space after ( in function call') ++ elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): ++ error(filename, linenum, 'whitespace/parens', 2, ++ 'Extra space after (') ++ if (Search(r'\w\s+\(', fncall) and ++ not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and ++ not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall)): ++ # TODO(unknown): Space after an operator function seem to be a common ++ # error, silence those for now by restricting them to highest verbosity. ++ if Search(r'\boperator_*\b', line): ++ error(filename, linenum, 'whitespace/parens', 0, ++ 'Extra space before ( in function call') ++ else: ++ error(filename, linenum, 'whitespace/parens', 4, ++ 'Extra space before ( in function call') ++ # If the ) is followed only by a newline or a { + newline, assume it's ++ # part of a control statement (if/while/etc), and don't complain ++ if Search(r'[^)]\s+\)\s*[^{\s]', fncall): ++ # If the closing parenthesis is preceded by only whitespaces, ++ # try to give a more descriptive error message. ++ if Search(r'^\s+\)', fncall): ++ error(filename, linenum, 'whitespace/parens', 2, ++ 'Closing ) should be moved to the previous line') ++ else: ++ error(filename, linenum, 'whitespace/parens', 2, ++ 'Extra space before )') ++ ++ ++def IsBlankLine(line): ++ """Returns true if the given line is blank. ++ ++ We consider a line to be blank if the line is empty or consists of ++ only white spaces. ++ ++ Args: ++ line: A line of a string. ++ ++ Returns: ++ True, if the given line is blank. ++ """ ++ return not line or line.isspace() ++ ++ ++def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, ++ error): ++ is_namespace_indent_item = ( ++ len(nesting_state.stack) > 1 and ++ nesting_state.stack[-1].check_namespace_indentation and ++ isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and ++ nesting_state.previous_stack_top == nesting_state.stack[-2]) ++ ++ if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, ++ clean_lines.elided, line): ++ CheckItemIndentationInNamespace(filename, clean_lines.elided, ++ line, error) ++ ++ ++def CheckForFunctionLengths(filename, clean_lines, linenum, ++ function_state, error): ++ """Reports for long function bodies. ++ ++ For an overview why this is done, see: ++ http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions ++ ++ Uses a simplistic algorithm assuming other style guidelines ++ (especially spacing) are followed. ++ Only checks unindented functions, so class members are unchecked. ++ Trivial bodies are unchecked, so constructors with huge initializer lists ++ may be missed. ++ Blank/comment lines are not counted so as to avoid encouraging the removal ++ of vertical space and comments just to get through a lint check. ++ NOLINT *on the last line of a function* disables this check. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ function_state: Current function name and lines in body so far. ++ error: The function to call with any errors found. ++ """ ++ lines = clean_lines.lines ++ line = lines[linenum] ++ joined_line = '' ++ ++ starting_func = False ++ regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... ++ match_result = Match(regexp, line) ++ if match_result: ++ # If the name is all caps and underscores, figure it's a macro and ++ # ignore it, unless it's TEST or TEST_F. ++ function_name = match_result.group(1).split()[-1] ++ if function_name == 'TEST' or function_name == 'TEST_F' or ( ++ not Match(r'[A-Z_]+$', function_name)): ++ starting_func = True ++ ++ if starting_func: ++ body_found = False ++ for start_linenum in xrange(linenum, clean_lines.NumLines()): ++ start_line = lines[start_linenum] ++ joined_line += ' ' + start_line.lstrip() ++ if Search(r'(;|})', start_line): # Declarations and trivial functions ++ body_found = True ++ break # ... ignore ++ elif Search(r'{', start_line): ++ body_found = True ++ function = Search(r'((\w|:)*)\(', line).group(1) ++ if Match(r'TEST', function): # Handle TEST... macros ++ parameter_regexp = Search(r'(\(.*\))', joined_line) ++ if parameter_regexp: # Ignore bad syntax ++ function += parameter_regexp.group(1) ++ else: ++ function += '()' ++ function_state.Begin(function) ++ break ++ if not body_found: ++ # No body for the function (or evidence of a non-function) was found. ++ error(filename, linenum, 'readability/fn_size', 5, ++ 'Lint failed to find start of function body.') ++ elif Match(r'^\}\s*$', line): # function end ++ function_state.Check(error, filename, linenum) ++ function_state.End() ++ elif not Match(r'^\s*$', line): ++ function_state.Count() # Count non-blank/non-comment lines. ++ ++ ++_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') ++ ++ ++def CheckComment(line, filename, linenum, next_line_start, error): ++ """Checks for common mistakes in comments. ++ ++ Args: ++ line: The line in question. ++ filename: The name of the current file. ++ linenum: The number of the line to check. ++ next_line_start: The first non-whitespace column of the next line. ++ error: The function to call with any errors found. ++ """ ++ commentpos = line.find('//') ++ if commentpos != -1: ++ # Check if the // may be in quotes. If so, ignore it ++ # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison ++ if (line.count('"', 0, commentpos) - ++ line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes ++ # Allow one space for new scopes, two spaces otherwise: ++ if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and ++ ((commentpos >= 1 and ++ line[commentpos-1] not in string.whitespace) or ++ (commentpos >= 2 and ++ line[commentpos-2] not in string.whitespace))): ++ error(filename, linenum, 'whitespace/comments', 2, ++ 'At least two spaces is best between code and comments') ++ ++ # Checks for common mistakes in TODO comments. ++ comment = line[commentpos:] ++ match = _RE_PATTERN_TODO.match(comment) ++ if match: ++ # One whitespace is correct; zero whitespace is handled elsewhere. ++ leading_whitespace = match.group(1) ++ if len(leading_whitespace) > 1: ++ error(filename, linenum, 'whitespace/todo', 2, ++ 'Too many spaces before TODO') ++ ++ username = match.group(2) ++ if not username: ++ error(filename, linenum, 'readability/todo', 2, ++ 'Missing username in TODO; it should look like ' ++ '"// TODO(my_username): Stuff."') ++ ++ middle_whitespace = match.group(3) ++ # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison ++ if middle_whitespace != ' ' and middle_whitespace != '': ++ error(filename, linenum, 'whitespace/todo', 2, ++ 'TODO(my_username) should be followed by a space') ++ ++ # If the comment contains an alphanumeric character, there ++ # should be a space somewhere between it and the //. ++ if Match(r'//[^ ]*\w', comment): ++ error(filename, linenum, 'whitespace/comments', 4, ++ 'Should have a space between // and comment') ++ ++def CheckAccess(filename, clean_lines, linenum, nesting_state, error): ++ """Checks for improper use of DISALLOW* macros. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] # get rid of comments and strings ++ ++ matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' ++ r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) ++ if not matched: ++ return ++ if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): ++ if nesting_state.stack[-1].access != 'private': ++ error(filename, linenum, 'readability/constructors', 3, ++ '%s must be in the private: section' % matched.group(1)) ++ ++ else: ++ # Found DISALLOW* macro outside a class declaration, or perhaps it ++ # was used inside a function when it should have been part of the ++ # class declaration. We could issue a warning here, but it ++ # probably resulted in a compiler error already. ++ pass ++ ++ ++def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): ++ """Checks for the correctness of various spacing issues in the code. ++ ++ Things we check for: spaces around operators, spaces after ++ if/for/while/switch, no spaces around parens in function calls, two ++ spaces between code and comment, don't start a block with a blank ++ line, don't end a function with a blank line, don't add a blank line ++ after public/protected/private, don't have too many blank lines in a row. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ ++ # Don't use "elided" lines here, otherwise we can't check commented lines. ++ # Don't want to use "raw" either, because we don't want to check inside C++11 ++ # raw strings, ++ raw = clean_lines.lines_without_raw_strings ++ line = raw[linenum] ++ ++ # Before nixing comments, check if the line is blank for no good ++ # reason. This includes the first line after a block is opened, and ++ # blank lines at the end of a function (ie, right before a line like '}' ++ # ++ # Skip all the blank line checks if we are immediately inside a ++ # namespace body. In other words, don't issue blank line warnings ++ # for this block: ++ # namespace { ++ # ++ # } ++ # ++ # A warning about missing end of namespace comments will be issued instead. ++ # ++ # Also skip blank line checks for 'extern "C"' blocks, which are formatted ++ # like namespaces. ++ if (IsBlankLine(line) and ++ not nesting_state.InNamespaceBody() and ++ not nesting_state.InExternC()): ++ elided = clean_lines.elided ++ prev_line = elided[linenum - 1] ++ prevbrace = prev_line.rfind('{') ++ # TODO(unknown): Don't complain if line before blank line, and line after, ++ # both start with alnums and are indented the same amount. ++ # This ignores whitespace at the start of a namespace block ++ # because those are not usually indented. ++ if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: ++ # OK, we have a blank line at the start of a code block. Before we ++ # complain, we check if it is an exception to the rule: The previous ++ # non-empty line has the parameters of a function header that are indented ++ # 4 spaces (because they did not fit in a 80 column line when placed on ++ # the same line as the function name). We also check for the case where ++ # the previous line is indented 6 spaces, which may happen when the ++ # initializers of a constructor do not fit into a 80 column line. ++ exception = False ++ if Match(r' {6}\w', prev_line): # Initializer list? ++ # We are looking for the opening column of initializer list, which ++ # should be indented 4 spaces to cause 6 space indentation afterwards. ++ search_position = linenum-2 ++ while (search_position >= 0 ++ and Match(r' {6}\w', elided[search_position])): ++ search_position -= 1 ++ exception = (search_position >= 0 ++ and elided[search_position][:5] == ' :') ++ else: ++ # Search for the function arguments or an initializer list. We use a ++ # simple heuristic here: If the line is indented 4 spaces; and we have a ++ # closing paren, without the opening paren, followed by an opening brace ++ # or colon (for initializer lists) we assume that it is the last line of ++ # a function header. If we have a colon indented 4 spaces, it is an ++ # initializer list. ++ exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', ++ prev_line) ++ or Match(r' {4}:', prev_line)) ++ ++ if not exception: ++ error(filename, linenum, 'whitespace/blank_line', 2, ++ 'Redundant blank line at the start of a code block ' ++ 'should be deleted.') ++ # Ignore blank lines at the end of a block in a long if-else ++ # chain, like this: ++ # if (condition1) { ++ # // Something followed by a blank line ++ # ++ # } else if (condition2) { ++ # // Something else ++ # } ++ if linenum + 1 < clean_lines.NumLines(): ++ next_line = raw[linenum + 1] ++ if (next_line ++ and Match(r'\s*}', next_line) ++ and next_line.find('} else ') == -1): ++ error(filename, linenum, 'whitespace/blank_line', 3, ++ 'Redundant blank line at the end of a code block ' ++ 'should be deleted.') ++ ++ matched = Match(r'\s*(public|protected|private):', prev_line) ++ if matched: ++ error(filename, linenum, 'whitespace/blank_line', 3, ++ 'Do not leave a blank line after "%s:"' % matched.group(1)) ++ ++ # Next, check comments ++ next_line_start = 0 ++ if linenum + 1 < clean_lines.NumLines(): ++ next_line = raw[linenum + 1] ++ next_line_start = len(next_line) - len(next_line.lstrip()) ++ CheckComment(line, filename, linenum, next_line_start, error) ++ ++ # get rid of comments and strings ++ line = clean_lines.elided[linenum] ++ ++ # You shouldn't have spaces before your brackets, except maybe after ++ # 'delete []' or 'return []() {};' ++ if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): ++ error(filename, linenum, 'whitespace/braces', 5, ++ 'Extra space before [') ++ ++ # In range-based for, we wanted spaces before and after the colon, but ++ # not around "::" tokens that might appear. ++ if (Search(r'for *\(.*[^:]:[^: ]', line) or ++ Search(r'for *\(.*[^: ]:[^:]', line)): ++ error(filename, linenum, 'whitespace/forcolon', 2, ++ 'Missing space around colon in range-based for loop') ++ ++ ++def CheckOperatorSpacing(filename, clean_lines, linenum, error): ++ """Checks for horizontal spacing around operators. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Don't try to do spacing checks for operator methods. Do this by ++ # replacing the troublesome characters with something else, ++ # preserving column position for all other characters. ++ # ++ # The replacement is done repeatedly to avoid false positives from ++ # operators that call operators. ++ while True: ++ match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) ++ if match: ++ line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) ++ else: ++ break ++ ++ # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". ++ # Otherwise not. Note we only check for non-spaces on *both* sides; ++ # sometimes people put non-spaces on one side when aligning ='s among ++ # many lines (not that this is behavior that I approve of...) ++ if Search(r'[\w.]=[\w.]', line) and not Search(r'\b(if|while) ', line): ++ error(filename, linenum, 'whitespace/operators', 4, ++ 'Missing spaces around =') ++ ++ # It's ok not to have spaces around binary operators like + - * /, but if ++ # there's too little whitespace, we get concerned. It's hard to tell, ++ # though, so we punt on this one for now. TODO. ++ ++ # You should always have whitespace around binary operators. ++ # ++ # Check <= and >= first to avoid false positives with < and >, then ++ # check non-include lines for spacing around < and >. ++ # ++ # If the operator is followed by a comma, assume it's be used in a ++ # macro context and don't do any checks. This avoids false ++ # positives. ++ # ++ # Note that && is not included here. Those are checked separately ++ # in CheckRValueReference ++ match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) ++ if match: ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around %s' % match.group(1)) ++ elif not Match(r'#.*include', line): ++ # Look for < that is not surrounded by spaces. This is only ++ # triggered if both sides are missing spaces, even though ++ # technically should should flag if at least one side is missing a ++ # space. This is done to avoid some false positives with shifts. ++ match = Match(r'^(.*[^\s<])<[^\s=<,]', line) ++ if match: ++ (_, _, end_pos) = CloseExpression( ++ clean_lines, linenum, len(match.group(1))) ++ if end_pos <= -1: ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around <') ++ ++ # Look for > that is not surrounded by spaces. Similar to the ++ # above, we only trigger if both sides are missing spaces to avoid ++ # false positives with shifts. ++ match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) ++ if match: ++ (_, _, start_pos) = ReverseCloseExpression( ++ clean_lines, linenum, len(match.group(1))) ++ if start_pos <= -1: ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around >') ++ ++ # We allow no-spaces around << when used like this: 10<<20, but ++ # not otherwise (particularly, not when used as streams) ++ # ++ # We also allow operators following an opening parenthesis, since ++ # those tend to be macros that deal with operators. ++ match = Search(r'(operator|\S)(?:L|UL|ULL|l|ul|ull)?<<([^\s,=])', line) ++ if (match and match.group(1) != '(' and ++ not (match.group(1).isdigit() and match.group(2).isdigit()) and ++ not (match.group(1) == 'operator' and match.group(2) == ';')): ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around <<') ++ ++ # We allow no-spaces around >> for almost anything. This is because ++ # C++11 allows ">>" to close nested templates, which accounts for ++ # most cases when ">>" is not followed by a space. ++ # ++ # We still warn on ">>" followed by alpha character, because that is ++ # likely due to ">>" being used for right shifts, e.g.: ++ # value >> alpha ++ # ++ # When ">>" is used to close templates, the alphanumeric letter that ++ # follows would be part of an identifier, and there should still be ++ # a space separating the template type and the identifier. ++ # type> alpha ++ match = Search(r'>>[a-zA-Z_]', line) ++ if match: ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around >>') ++ ++ # There shouldn't be space around unary operators ++ match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) ++ if match: ++ error(filename, linenum, 'whitespace/operators', 4, ++ 'Extra space for operator %s' % match.group(1)) ++ ++ ++def CheckParenthesisSpacing(filename, clean_lines, linenum, error): ++ """Checks for horizontal spacing around parentheses. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # No spaces after an if, while, switch, or for ++ match = Search(r' (if\(|for\(|while\(|switch\()', line) ++ if match: ++ error(filename, linenum, 'whitespace/parens', 5, ++ 'Missing space before ( in %s' % match.group(1)) ++ ++ # For if/for/while/switch, the left and right parens should be ++ # consistent about how many spaces are inside the parens, and ++ # there should either be zero or one spaces inside the parens. ++ # We don't want: "if ( foo)" or "if ( foo )". ++ # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. ++ match = Search(r'\b(if|for|while|switch)\s*' ++ r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', ++ line) ++ if match: ++ if len(match.group(2)) != len(match.group(4)): ++ if not (match.group(3) == ';' and ++ len(match.group(2)) == 1 + len(match.group(4)) or ++ not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): ++ error(filename, linenum, 'whitespace/parens', 5, ++ 'Mismatching spaces inside () in %s' % match.group(1)) ++ if len(match.group(2)) not in [0, 1]: ++ error(filename, linenum, 'whitespace/parens', 5, ++ 'Should have zero or one spaces inside ( and ) in %s' % ++ match.group(1)) ++ ++ ++def CheckCommaSpacing(filename, clean_lines, linenum, error): ++ """Checks for horizontal spacing near commas and semicolons. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ raw = clean_lines.lines_without_raw_strings ++ line = clean_lines.elided[linenum] ++ ++ # You should always have a space after a comma (either as fn arg or operator) ++ # ++ # This does not apply when the non-space character following the ++ # comma is another comma, since the only time when that happens is ++ # for empty macro arguments. ++ # ++ # We run this check in two passes: first pass on elided lines to ++ # verify that lines contain missing whitespaces, second pass on raw ++ # lines to confirm that those missing whitespaces are not due to ++ # elided comments. ++ if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and ++ Search(r',[^,\s]', raw[linenum])): ++ error(filename, linenum, 'whitespace/comma', 3, ++ 'Missing space after ,') ++ ++ # You should always have a space after a semicolon ++ # except for few corner cases ++ # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more ++ # space after ; ++ if Search(r';[^\s};\\)/]', line): ++ error(filename, linenum, 'whitespace/semicolon', 3, ++ 'Missing space after ;') ++ ++ ++def CheckBracesSpacing(filename, clean_lines, linenum, error): ++ """Checks for horizontal spacing near commas. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Except after an opening paren, or after another opening brace (in case of ++ # an initializer list, for instance), you should have spaces before your ++ # braces. And since you should never have braces at the beginning of a line, ++ # this is an easy test. ++ match = Match(r'^(.*[^ ({]){', line) ++ if match: ++ # Try a bit harder to check for brace initialization. This ++ # happens in one of the following forms: ++ # Constructor() : initializer_list_{} { ... } ++ # Constructor{}.MemberFunction() ++ # Type variable{}; ++ # FunctionCall(type{}, ...); ++ # LastArgument(..., type{}); ++ # LOG(INFO) << type{} << " ..."; ++ # map_of_type[{...}] = ...; ++ # ternary = expr ? new type{} : nullptr; ++ # OuterTemplate{}> ++ # ++ # We check for the character following the closing brace, and ++ # silence the warning if it's one of those listed above, i.e. ++ # "{.;,)<>]:". ++ # ++ # To account for nested initializer list, we allow any number of ++ # closing braces up to "{;,)<". We can't simply silence the ++ # warning on first sight of closing brace, because that would ++ # cause false negatives for things that are not initializer lists. ++ # Silence this: But not this: ++ # Outer{ if (...) { ++ # Inner{...} if (...){ // Missing space before { ++ # }; } ++ # ++ # There is a false negative with this approach if people inserted ++ # spurious semicolons, e.g. "if (cond){};", but we will catch the ++ # spurious semicolon with a separate check. ++ (endline, endlinenum, endpos) = CloseExpression( ++ clean_lines, linenum, len(match.group(1))) ++ trailing_text = '' ++ if endpos > -1: ++ trailing_text = endline[endpos:] ++ for offset in xrange(endlinenum + 1, ++ min(endlinenum + 3, clean_lines.NumLines() - 1)): ++ trailing_text += clean_lines.elided[offset] ++ if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): ++ error(filename, linenum, 'whitespace/braces', 5, ++ 'Missing space before {') ++ ++ # Make sure '} else {' has spaces. ++ if Search(r'}else', line): ++ error(filename, linenum, 'whitespace/braces', 5, ++ 'Missing space before else') ++ ++ # You shouldn't have a space before a semicolon at the end of the line. ++ # There's a special case for "for" since the style guide allows space before ++ # the semicolon there. ++ if Search(r':\s*;\s*$', line): ++ error(filename, linenum, 'whitespace/semicolon', 5, ++ 'Semicolon defining empty statement. Use {} instead.') ++ elif Search(r'^\s*;\s*$', line): ++ error(filename, linenum, 'whitespace/semicolon', 5, ++ 'Line contains only semicolon. If this should be an empty statement, ' ++ 'use {} instead.') ++ elif (Search(r'\s+;\s*$', line) and ++ not Search(r'\bfor\b', line)): ++ error(filename, linenum, 'whitespace/semicolon', 5, ++ 'Extra space before last semicolon. If this should be an empty ' ++ 'statement, use {} instead.') ++ ++ ++def IsDecltype(clean_lines, linenum, column): ++ """Check if the token ending on (linenum, column) is decltype(). ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: the number of the line to check. ++ column: end column of the token to check. ++ Returns: ++ True if this token is decltype() expression, False otherwise. ++ """ ++ (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) ++ if start_col < 0: ++ return False ++ if Search(r'\bdecltype\s*$', text[0:start_col]): ++ return True ++ return False ++ ++ ++def IsTemplateParameterList(clean_lines, linenum, column): ++ """Check if the token ending on (linenum, column) is the end of template<>. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: the number of the line to check. ++ column: end column of the token to check. ++ Returns: ++ True if this token is end of a template parameter list, False otherwise. ++ """ ++ (_, startline, startpos) = ReverseCloseExpression( ++ clean_lines, linenum, column) ++ if (startpos > -1 and ++ Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])): ++ return True ++ return False ++ ++ ++def IsRValueType(clean_lines, nesting_state, linenum, column): ++ """Check if the token ending on (linenum, column) is a type. ++ ++ Assumes that text to the right of the column is "&&" or a function ++ name. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ linenum: the number of the line to check. ++ column: end column of the token to check. ++ Returns: ++ True if this token is a type, False if we are not sure. ++ """ ++ prefix = clean_lines.elided[linenum][0:column] ++ ++ # Get one word to the left. If we failed to do so, this is most ++ # likely not a type, since it's unlikely that the type name and "&&" ++ # would be split across multiple lines. ++ match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix) ++ if not match: ++ return False ++ ++ # Check text following the token. If it's "&&>" or "&&," or "&&...", it's ++ # most likely a rvalue reference used inside a template. ++ suffix = clean_lines.elided[linenum][column:] ++ if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix): ++ return True ++ ++ # Check for simple type and end of templates: ++ # int&& variable ++ # vector&& variable ++ # ++ # Because this function is called recursively, we also need to ++ # recognize pointer and reference types: ++ # int* Function() ++ # int& Function() ++ if match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool', ++ 'short', 'int', 'long', 'signed', 'unsigned', ++ 'float', 'double', 'void', 'auto', '>', '*', '&']: ++ return True ++ ++ # If we see a close parenthesis, look for decltype on the other side. ++ # decltype would unambiguously identify a type, anything else is ++ # probably a parenthesized expression and not a type. ++ if match.group(2) == ')': ++ return IsDecltype( ++ clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1) ++ ++ # Check for casts and cv-qualifiers. ++ # match.group(1) remainder ++ # -------------- --------- ++ # const_cast< type&& ++ # const type&& ++ # type const&& ++ if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|' ++ r'reinterpret_cast\s*<|\w+\s)\s*$', ++ match.group(1)): ++ return True ++ ++ # Look for a preceding symbol that might help differentiate the context. ++ # These are the cases that would be ambiguous: ++ # match.group(1) remainder ++ # -------------- --------- ++ # Call ( expression && ++ # Declaration ( type&& ++ # sizeof ( type&& ++ # if ( expression && ++ # while ( expression && ++ # for ( type&& ++ # for( ; expression && ++ # statement ; type&& ++ # block { type&& ++ # constructor { expression && ++ start = linenum ++ line = match.group(1) ++ match_symbol = None ++ while start >= 0: ++ # We want to skip over identifiers and commas to get to a symbol. ++ # Commas are skipped so that we can find the opening parenthesis ++ # for function parameter lists. ++ match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line) ++ if match_symbol: ++ break ++ start -= 1 ++ line = clean_lines.elided[start] ++ ++ if not match_symbol: ++ # Probably the first statement in the file is an rvalue reference ++ return True ++ ++ if match_symbol.group(2) == '}': ++ # Found closing brace, probably an indicate of this: ++ # block{} type&& ++ return True ++ ++ if match_symbol.group(2) == ';': ++ # Found semicolon, probably one of these: ++ # for(; expression && ++ # statement; type&& ++ ++ # Look for the previous 'for(' in the previous lines. ++ before_text = match_symbol.group(1) ++ for i in xrange(start - 1, max(start - 6, 0), -1): ++ before_text = clean_lines.elided[i] + before_text ++ if Search(r'for\s*\([^{};]*$', before_text): ++ # This is the condition inside a for-loop ++ return False ++ ++ # Did not find a for-init-statement before this semicolon, so this ++ # is probably a new statement and not a condition. ++ return True ++ ++ if match_symbol.group(2) == '{': ++ # Found opening brace, probably one of these: ++ # block{ type&& = ... ; } ++ # constructor{ expression && expression } ++ ++ # Look for a closing brace or a semicolon. If we see a semicolon ++ # first, this is probably a rvalue reference. ++ line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1] ++ end = start ++ depth = 1 ++ while True: ++ for ch in line: ++ if ch == ';': ++ return True ++ elif ch == '{': ++ depth += 1 ++ elif ch == '}': ++ depth -= 1 ++ if depth == 0: ++ return False ++ end += 1 ++ if end >= clean_lines.NumLines(): ++ break ++ line = clean_lines.elided[end] ++ # Incomplete program? ++ return False ++ ++ if match_symbol.group(2) == '(': ++ # Opening parenthesis. Need to check what's to the left of the ++ # parenthesis. Look back one extra line for additional context. ++ before_text = match_symbol.group(1) ++ if linenum > 1: ++ before_text = clean_lines.elided[linenum - 1] + before_text ++ before_text = match_symbol.group(1) ++ ++ # Patterns that are likely to be types: ++ # [](type&& ++ # for (type&& ++ # sizeof(type&& ++ # operator=(type&& ++ # ++ if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text): ++ return True ++ ++ # Patterns that are likely to be expressions: ++ # if (expression && ++ # while (expression && ++ # : initializer(expression && ++ # , initializer(expression && ++ # ( FunctionCall(expression && ++ # + FunctionCall(expression && ++ # + (expression && ++ # ++ # The last '+' represents operators such as '+' and '-'. ++ if Search(r'(?:\bif|\bwhile|[-+=%^(]*>)?\s*$', ++ match_symbol.group(1)) ++ if match_func: ++ # Check for constructors, which don't have return types. ++ if Search(r'\b(?:explicit|inline)$', match_func.group(1)): ++ return True ++ implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix) ++ if (implicit_constructor and ++ implicit_constructor.group(1) == implicit_constructor.group(2)): ++ return True ++ return IsRValueType(clean_lines, nesting_state, linenum, ++ len(match_func.group(1))) ++ ++ # Nothing before the function name. If this is inside a block scope, ++ # this is probably a function call. ++ return not (nesting_state.previous_stack_top and ++ nesting_state.previous_stack_top.IsBlockInfo()) ++ ++ if match_symbol.group(2) == '>': ++ # Possibly a closing bracket, check that what's on the other side ++ # looks like the start of a template. ++ return IsTemplateParameterList( ++ clean_lines, start, len(match_symbol.group(1))) ++ ++ # Some other symbol, usually something like "a=b&&c". This is most ++ # likely not a type. ++ return False ++ ++ ++def IsDeletedOrDefault(clean_lines, linenum): ++ """Check if current constructor or operator is deleted or default. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ Returns: ++ True if this is a deleted or default constructor. ++ """ ++ open_paren = clean_lines.elided[linenum].find('(') ++ if open_paren < 0: ++ return False ++ (close_line, _, close_paren) = CloseExpression( ++ clean_lines, linenum, open_paren) ++ if close_paren < 0: ++ return False ++ return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:]) ++ ++ ++def IsRValueAllowed(clean_lines, linenum): ++ """Check if RValue reference is allowed on a particular line. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ Returns: ++ True if line is within the region where RValue references are allowed. ++ """ ++ # Allow region marked by PUSH/POP macros ++ for i in xrange(linenum, 0, -1): ++ line = clean_lines.elided[i] ++ if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): ++ if not line.endswith('PUSH'): ++ return False ++ for j in xrange(linenum, clean_lines.NumLines(), 1): ++ line = clean_lines.elided[j] ++ if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): ++ return line.endswith('POP') ++ ++ # Allow operator= ++ line = clean_lines.elided[linenum] ++ if Search(r'\boperator\s*=\s*\(', line): ++ return IsDeletedOrDefault(clean_lines, linenum) ++ ++ # Allow constructors ++ match = Match(r'\s*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line) ++ if match and match.group(1) == match.group(2): ++ return IsDeletedOrDefault(clean_lines, linenum) ++ if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line): ++ return IsDeletedOrDefault(clean_lines, linenum) ++ ++ if Match(r'\s*[\w<>]+\s*\(', line): ++ previous_line = 'ReturnType' ++ if linenum > 0: ++ previous_line = clean_lines.elided[linenum - 1] ++ if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line): ++ return IsDeletedOrDefault(clean_lines, linenum) ++ ++ return False ++ ++ ++def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error): ++ """Check for rvalue references. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ # Find lines missing spaces around &&. ++ # TODO(unknown): currently we don't check for rvalue references ++ # with spaces surrounding the && to avoid false positives with ++ # boolean expressions. ++ line = clean_lines.elided[linenum] ++ match = Match(r'^(.*\S)&&', line) ++ if not match: ++ match = Match(r'(.*)&&\S', line) ++ if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)): ++ return ++ ++ # Either poorly formed && or an rvalue reference, check the context ++ # to get a more accurate error message. Mostly we want to determine ++ # if what's to the left of "&&" is a type or not. ++ and_pos = len(match.group(1)) ++ if IsRValueType(clean_lines, nesting_state, linenum, and_pos): ++ if not IsRValueAllowed(clean_lines, linenum): ++ error(filename, linenum, 'build/c++11', 3, ++ 'RValue references are an unapproved C++ feature.') ++ else: ++ error(filename, linenum, 'whitespace/operators', 3, ++ 'Missing spaces around &&') ++ ++ ++def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): ++ """Checks for additional blank line issues related to sections. ++ ++ Currently the only thing checked here is blank line before protected/private. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ class_info: A _ClassInfo objects. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ # Skip checks if the class is small, where small means 25 lines or less. ++ # 25 lines seems like a good cutoff since that's the usual height of ++ # terminals, and any class that can't fit in one screen can't really ++ # be considered "small". ++ # ++ # Also skip checks if we are on the first line. This accounts for ++ # classes that look like ++ # class Foo { public: ... }; ++ # ++ # If we didn't find the end of the class, last_line would be zero, ++ # and the check will be skipped by the first condition. ++ if (class_info.last_line - class_info.starting_linenum <= 24 or ++ linenum <= class_info.starting_linenum): ++ return ++ ++ matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) ++ if matched: ++ # Issue warning if the line before public/protected/private was ++ # not a blank line, but don't do this if the previous line contains ++ # "class" or "struct". This can happen two ways: ++ # - We are at the beginning of the class. ++ # - We are forward-declaring an inner class that is semantically ++ # private, but needed to be public for implementation reasons. ++ # Also ignores cases where the previous line ends with a backslash as can be ++ # common when defining classes in C macros. ++ prev_line = clean_lines.lines[linenum - 1] ++ if (not IsBlankLine(prev_line) and ++ not Search(r'\b(class|struct)\b', prev_line) and ++ not Search(r'\\$', prev_line)): ++ # Try a bit harder to find the beginning of the class. This is to ++ # account for multi-line base-specifier lists, e.g.: ++ # class Derived ++ # : public Base { ++ end_class_head = class_info.starting_linenum ++ for i in range(class_info.starting_linenum, linenum): ++ if Search(r'\{\s*$', clean_lines.lines[i]): ++ end_class_head = i ++ break ++ if end_class_head < linenum - 1: ++ error(filename, linenum, 'whitespace/blank_line', 3, ++ '"%s:" should be preceded by a blank line' % matched.group(1)) ++ ++ ++def GetPreviousNonBlankLine(clean_lines, linenum): ++ """Return the most recent non-blank line and its line number. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file contents. ++ linenum: The number of the line to check. ++ ++ Returns: ++ A tuple with two elements. The first element is the contents of the last ++ non-blank line before the current line, or the empty string if this is the ++ first non-blank line. The second is the line number of that line, or -1 ++ if this is the first non-blank line. ++ """ ++ ++ prevlinenum = linenum - 1 ++ while prevlinenum >= 0: ++ prevline = clean_lines.elided[prevlinenum] ++ if not IsBlankLine(prevline): # if not a blank line... ++ return (prevline, prevlinenum) ++ prevlinenum -= 1 ++ return ('', -1) ++ ++ ++def CheckBraces(filename, clean_lines, linenum, error): ++ """Looks for misplaced braces (e.g. at the end of line). ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ ++ line = clean_lines.elided[linenum] # get rid of comments and strings ++ ++ if Match(r'\s*{\s*$', line): ++ # We allow an open brace to start a line in the case where someone is using ++ # braces in a block to explicitly create a new scope, which is commonly used ++ # to control the lifetime of stack-allocated variables. Braces are also ++ # used for brace initializers inside function calls. We don't detect this ++ # perfectly: we just don't complain if the last non-whitespace character on ++ # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the ++ # previous line starts a preprocessor block. ++ prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] ++ if (not Search(r'[,;:}{(]\s*$', prevline) and ++ not Match(r'\s*#', prevline)): ++ error(filename, linenum, 'whitespace/braces', 4, ++ '{ should almost always be at the end of the previous line') ++ ++ # An else clause should be on the same line as the preceding closing brace. ++ if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): ++ prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] ++ if Match(r'\s*}\s*$', prevline): ++ error(filename, linenum, 'whitespace/newline', 4, ++ 'An else should appear on the same line as the preceding }') ++ ++ # If braces come on one side of an else, they should be on both. ++ # However, we have to worry about "else if" that spans multiple lines! ++ if Search(r'else if\s*\(', line): # could be multi-line if ++ brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) ++ # find the ( after the if ++ pos = line.find('else if') ++ pos = line.find('(', pos) ++ if pos > 0: ++ (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) ++ brace_on_right = endline[endpos:].find('{') != -1 ++ if brace_on_left != brace_on_right: # must be brace after if ++ error(filename, linenum, 'readability/braces', 5, ++ 'If an else has a brace on one side, it should have it on both') ++ elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): ++ error(filename, linenum, 'readability/braces', 5, ++ 'If an else has a brace on one side, it should have it on both') ++ ++ # Likewise, an else should never have the else clause on the same line ++ if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): ++ error(filename, linenum, 'whitespace/newline', 4, ++ 'Else clause should never be on same line as else (use 2 lines)') ++ ++ # In the same way, a do/while should never be on one line ++ if Match(r'\s*do [^\s{]', line): ++ error(filename, linenum, 'whitespace/newline', 4, ++ 'do/while clauses should not be on a single line') ++ ++ # Check single-line if/else bodies. The style guide says 'curly braces are not ++ # required for single-line statements'. We additionally allow multi-line, ++ # single statements, but we reject anything with more than one semicolon in ++ # it. This means that the first semicolon after the if should be at the end of ++ # its line, and the line after that should have an indent level equal to or ++ # lower than the if. We also check for ambiguous if/else nesting without ++ # braces. ++ if_else_match = Search(r'\b(if\s*\(|else\b)', line) ++ if if_else_match and not Match(r'\s*#', line): ++ if_indent = GetIndentLevel(line) ++ endline, endlinenum, endpos = line, linenum, if_else_match.end() ++ if_match = Search(r'\bif\s*\(', line) ++ if if_match: ++ # This could be a multiline if condition, so find the end first. ++ pos = if_match.end() - 1 ++ (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) ++ # Check for an opening brace, either directly after the if or on the next ++ # line. If found, this isn't a single-statement conditional. ++ if (not Match(r'\s*{', endline[endpos:]) ++ and not (Match(r'\s*$', endline[endpos:]) ++ and endlinenum < (len(clean_lines.elided) - 1) ++ and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): ++ while (endlinenum < len(clean_lines.elided) ++ and ';' not in clean_lines.elided[endlinenum][endpos:]): ++ endlinenum += 1 ++ endpos = 0 ++ if endlinenum < len(clean_lines.elided): ++ endline = clean_lines.elided[endlinenum] ++ # We allow a mix of whitespace and closing braces (e.g. for one-liner ++ # methods) and a single \ after the semicolon (for macros) ++ endpos = endline.find(';') ++ if not Match(r';[\s}]*(\\?)$', endline[endpos:]): ++ # Semicolon isn't the last character, there's something trailing. ++ # Output a warning if the semicolon is not contained inside ++ # a lambda expression. ++ if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', ++ endline): ++ error(filename, linenum, 'readability/braces', 4, ++ 'If/else bodies with multiple statements require braces') ++ elif endlinenum < len(clean_lines.elided) - 1: ++ # Make sure the next line is dedented ++ next_line = clean_lines.elided[endlinenum + 1] ++ next_indent = GetIndentLevel(next_line) ++ # With ambiguous nested if statements, this will error out on the ++ # if that *doesn't* match the else, regardless of whether it's the ++ # inner one or outer one. ++ if (if_match and Match(r'\s*else\b', next_line) ++ and next_indent != if_indent): ++ error(filename, linenum, 'readability/braces', 4, ++ 'Else clause should be indented at the same level as if. ' ++ 'Ambiguous nested if/else chains require braces.') ++ elif next_indent > if_indent: ++ error(filename, linenum, 'readability/braces', 4, ++ 'If/else bodies with multiple statements require braces') ++ ++ ++def CheckTrailingSemicolon(filename, clean_lines, linenum, error): ++ """Looks for redundant trailing semicolon. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ ++ line = clean_lines.elided[linenum] ++ ++ # Block bodies should not be followed by a semicolon. Due to C++11 ++ # brace initialization, there are more places where semicolons are ++ # required than not, so we use a whitelist approach to check these ++ # rather than a blacklist. These are the places where "};" should ++ # be replaced by just "}": ++ # 1. Some flavor of block following closing parenthesis: ++ # for (;;) {}; ++ # while (...) {}; ++ # switch (...) {}; ++ # Function(...) {}; ++ # if (...) {}; ++ # if (...) else if (...) {}; ++ # ++ # 2. else block: ++ # if (...) else {}; ++ # ++ # 3. const member function: ++ # Function(...) const {}; ++ # ++ # 4. Block following some statement: ++ # x = 42; ++ # {}; ++ # ++ # 5. Block at the beginning of a function: ++ # Function(...) { ++ # {}; ++ # } ++ # ++ # Note that naively checking for the preceding "{" will also match ++ # braces inside multi-dimensional arrays, but this is fine since ++ # that expression will not contain semicolons. ++ # ++ # 6. Block following another block: ++ # while (true) {} ++ # {}; ++ # ++ # 7. End of namespaces: ++ # namespace {}; ++ # ++ # These semicolons seems far more common than other kinds of ++ # redundant semicolons, possibly due to people converting classes ++ # to namespaces. For now we do not warn for this case. ++ # ++ # Try matching case 1 first. ++ match = Match(r'^(.*\)\s*)\{', line) ++ if match: ++ # Matched closing parenthesis (case 1). Check the token before the ++ # matching opening parenthesis, and don't warn if it looks like a ++ # macro. This avoids these false positives: ++ # - macro that defines a base class ++ # - multi-line macro that defines a base class ++ # - macro that defines the whole class-head ++ # ++ # But we still issue warnings for macros that we know are safe to ++ # warn, specifically: ++ # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P ++ # - TYPED_TEST ++ # - INTERFACE_DEF ++ # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: ++ # ++ # We implement a whitelist of safe macros instead of a blacklist of ++ # unsafe macros, even though the latter appears less frequently in ++ # google code and would have been easier to implement. This is because ++ # the downside for getting the whitelist wrong means some extra ++ # semicolons, while the downside for getting the blacklist wrong ++ # would result in compile errors. ++ # ++ # In addition to macros, we also don't want to warn on compound ++ # literals and lambdas. ++ closing_brace_pos = match.group(1).rfind(')') ++ opening_parenthesis = ReverseCloseExpression( ++ clean_lines, linenum, closing_brace_pos) ++ if opening_parenthesis[2] > -1: ++ line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] ++ macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) ++ func = Match(r'^(.*\])\s*$', line_prefix) ++ if ((macro and ++ macro.group(1) not in ( ++ 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', ++ 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', ++ 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or ++ (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or ++ Search(r'\s+=\s*$', line_prefix)): ++ match = None ++ if (match and ++ opening_parenthesis[1] > 1 and ++ Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): ++ # Multi-line lambda-expression ++ match = None ++ ++ else: ++ # Try matching cases 2-3. ++ match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) ++ if not match: ++ # Try matching cases 4-6. These are always matched on separate lines. ++ # ++ # Note that we can't simply concatenate the previous line to the ++ # current line and do a single match, otherwise we may output ++ # duplicate warnings for the blank line case: ++ # if (cond) { ++ # // blank line ++ # } ++ prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] ++ if prevline and Search(r'[;{}]\s*$', prevline): ++ match = Match(r'^(\s*)\{', line) ++ ++ # Check matching closing brace ++ if match: ++ (endline, endlinenum, endpos) = CloseExpression( ++ clean_lines, linenum, len(match.group(1))) ++ if endpos > -1 and Match(r'^\s*;', endline[endpos:]): ++ # Current {} pair is eligible for semicolon check, and we have found ++ # the redundant semicolon, output warning here. ++ # ++ # Note: because we are scanning forward for opening braces, and ++ # outputting warnings for the matching closing brace, if there are ++ # nested blocks with trailing semicolons, we will get the error ++ # messages in reversed order. ++ error(filename, endlinenum, 'readability/braces', 4, ++ "You don't need a ; after a }") ++ ++ ++def CheckEmptyBlockBody(filename, clean_lines, linenum, error): ++ """Look for empty loop/conditional body with only a single semicolon. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ ++ # Search for loop keywords at the beginning of the line. Because only ++ # whitespaces are allowed before the keywords, this will also ignore most ++ # do-while-loops, since those lines should start with closing brace. ++ # ++ # We also check "if" blocks here, since an empty conditional block ++ # is likely an error. ++ line = clean_lines.elided[linenum] ++ matched = Match(r'\s*(for|while|if)\s*\(', line) ++ if matched: ++ # Find the end of the conditional expression ++ (end_line, end_linenum, end_pos) = CloseExpression( ++ clean_lines, linenum, line.find('(')) ++ ++ # Output warning if what follows the condition expression is a semicolon. ++ # No warning for all other cases, including whitespace or newline, since we ++ # have a separate check for semicolons preceded by whitespace. ++ if end_pos >= 0 and Match(r';', end_line[end_pos:]): ++ if matched.group(1) == 'if': ++ error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, ++ 'Empty conditional bodies should use {}') ++ else: ++ error(filename, end_linenum, 'whitespace/empty_loop_body', 5, ++ 'Empty loop bodies should use {} or continue') ++ ++ ++def FindCheckMacro(line): ++ """Find a replaceable CHECK-like macro. ++ ++ Args: ++ line: line to search on. ++ Returns: ++ (macro name, start position), or (None, -1) if no replaceable ++ macro is found. ++ """ ++ for macro in _CHECK_MACROS: ++ i = line.find(macro) ++ if i >= 0: ++ # Find opening parenthesis. Do a regular expression match here ++ # to make sure that we are matching the expected CHECK macro, as ++ # opposed to some other macro that happens to contain the CHECK ++ # substring. ++ matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) ++ if not matched: ++ continue ++ return (macro, len(matched.group(1))) ++ return (None, -1) ++ ++ ++def CheckCheck(filename, clean_lines, linenum, error): ++ """Checks the use of CHECK and EXPECT macros. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ ++ # Decide the set of replacement macros that should be suggested ++ lines = clean_lines.elided ++ (check_macro, start_pos) = FindCheckMacro(lines[linenum]) ++ if not check_macro: ++ return ++ ++ # Find end of the boolean expression by matching parentheses ++ (last_line, end_line, end_pos) = CloseExpression( ++ clean_lines, linenum, start_pos) ++ if end_pos < 0: ++ return ++ ++ # If the check macro is followed by something other than a ++ # semicolon, assume users will log their own custom error messages ++ # and don't suggest any replacements. ++ if not Match(r'\s*;', last_line[end_pos:]): ++ return ++ ++ if linenum == end_line: ++ expression = lines[linenum][start_pos + 1:end_pos - 1] ++ else: ++ expression = lines[linenum][start_pos + 1:] ++ for i in xrange(linenum + 1, end_line): ++ expression += lines[i] ++ expression += last_line[0:end_pos - 1] ++ ++ # Parse expression so that we can take parentheses into account. ++ # This avoids false positives for inputs like "CHECK((a < 4) == b)", ++ # which is not replaceable by CHECK_LE. ++ lhs = '' ++ rhs = '' ++ operator = None ++ while expression: ++ matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' ++ r'==|!=|>=|>|<=|<|\()(.*)$', expression) ++ if matched: ++ token = matched.group(1) ++ if token == '(': ++ # Parenthesized operand ++ expression = matched.group(2) ++ (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) ++ if end < 0: ++ return # Unmatched parenthesis ++ lhs += '(' + expression[0:end] ++ expression = expression[end:] ++ elif token in ('&&', '||'): ++ # Logical and/or operators. This means the expression ++ # contains more than one term, for example: ++ # CHECK(42 < a && a < b); ++ # ++ # These are not replaceable with CHECK_LE, so bail out early. ++ return ++ elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): ++ # Non-relational operator ++ lhs += token ++ expression = matched.group(2) ++ else: ++ # Relational operator ++ operator = token ++ rhs = matched.group(2) ++ break ++ else: ++ # Unparenthesized operand. Instead of appending to lhs one character ++ # at a time, we do another regular expression match to consume several ++ # characters at once if possible. Trivial benchmark shows that this ++ # is more efficient when the operands are longer than a single ++ # character, which is generally the case. ++ matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) ++ if not matched: ++ matched = Match(r'^(\s*\S)(.*)$', expression) ++ if not matched: ++ break ++ lhs += matched.group(1) ++ expression = matched.group(2) ++ ++ # Only apply checks if we got all parts of the boolean expression ++ if not (lhs and operator and rhs): ++ return ++ ++ # Check that rhs do not contain logical operators. We already know ++ # that lhs is fine since the loop above parses out && and ||. ++ if rhs.find('&&') > -1 or rhs.find('||') > -1: ++ return ++ ++ # At least one of the operands must be a constant literal. This is ++ # to avoid suggesting replacements for unprintable things like ++ # CHECK(variable != iterator) ++ # ++ # The following pattern matches decimal, hex integers, strings, and ++ # characters (in that order). ++ lhs = lhs.strip() ++ rhs = rhs.strip() ++ match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' ++ if Match(match_constant, lhs) or Match(match_constant, rhs): ++ # Note: since we know both lhs and rhs, we can provide a more ++ # descriptive error message like: ++ # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) ++ # Instead of: ++ # Consider using CHECK_EQ instead of CHECK(a == b) ++ # ++ # We are still keeping the less descriptive message because if lhs ++ # or rhs gets long, the error message might become unreadable. ++ error(filename, linenum, 'readability/check', 2, ++ 'Consider using %s instead of %s(a %s b)' % ( ++ _CHECK_REPLACEMENT[check_macro][operator], ++ check_macro, operator)) ++ ++ ++def CheckAltTokens(filename, clean_lines, linenum, error): ++ """Check alternative keywords being used in boolean expressions. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Avoid preprocessor lines ++ if Match(r'^\s*#', line): ++ return ++ ++ # Last ditch effort to avoid multi-line comments. This will not help ++ # if the comment started before the current line or ended after the ++ # current line, but it catches most of the false positives. At least, ++ # it provides a way to workaround this warning for people who use ++ # multi-line comments in preprocessor macros. ++ # ++ # TODO(unknown): remove this once cpplint has better support for ++ # multi-line comments. ++ if line.find('/*') >= 0 or line.find('*/') >= 0: ++ return ++ ++ for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): ++ error(filename, linenum, 'readability/alt_tokens', 2, ++ 'Use operator %s instead of %s' % ( ++ _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) ++ ++ ++def GetLineWidth(line): ++ """Determines the width of the line in column positions. ++ ++ Args: ++ line: A string, which may be a Unicode string. ++ ++ Returns: ++ The width of the line in column positions, accounting for Unicode ++ combining characters and wide characters. ++ """ ++ if isinstance(line, unicode): ++ width = 0 ++ for uc in unicodedata.normalize('NFC', line): ++ if unicodedata.east_asian_width(uc) in ('W', 'F'): ++ width += 2 ++ elif not unicodedata.combining(uc): ++ width += 1 ++ return width ++ else: ++ return len(line) ++ ++ ++def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, ++ error): ++ """Checks rules from the 'C++ style rules' section of cppguide.html. ++ ++ Most of these rules are hard to test (naming, comment style), but we ++ do what we can. In particular we check for 2-space indents, line lengths, ++ tab usage, spaces inside code, etc. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ file_extension: The extension (without the dot) of the filename. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ ++ # Don't use "elided" lines here, otherwise we can't check commented lines. ++ # Don't want to use "raw" either, because we don't want to check inside C++11 ++ # raw strings, ++ raw_lines = clean_lines.lines_without_raw_strings ++ line = raw_lines[linenum] ++ ++ if line.find('\t') != -1: ++ error(filename, linenum, 'whitespace/tab', 1, ++ 'Tab found; better to use spaces') ++ ++ # One or three blank spaces at the beginning of the line is weird; it's ++ # hard to reconcile that with 2-space indents. ++ # NOTE: here are the conditions rob pike used for his tests. Mine aren't ++ # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces ++ # if(RLENGTH > 20) complain = 0; ++ # if(match($0, " +(error|private|public|protected):")) complain = 0; ++ # if(match(prev, "&& *$")) complain = 0; ++ # if(match(prev, "\\|\\| *$")) complain = 0; ++ # if(match(prev, "[\",=><] *$")) complain = 0; ++ # if(match($0, " <<")) complain = 0; ++ # if(match(prev, " +for \\(")) complain = 0; ++ # if(prevodd && match(prevprev, " +for \\(")) complain = 0; ++ scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' ++ classinfo = nesting_state.InnermostClass() ++ initial_spaces = 0 ++ cleansed_line = clean_lines.elided[linenum] ++ while initial_spaces < len(line) and line[initial_spaces] == ' ': ++ initial_spaces += 1 ++ if line and line[-1].isspace(): ++ error(filename, linenum, 'whitespace/end_of_line', 4, ++ 'Line ends in whitespace. Consider deleting these extra spaces.') ++ # There are certain situations we allow one space, notably for ++ # section labels, and also lines containing multi-line raw strings. ++ elif ((initial_spaces == 1 or initial_spaces == 3) and ++ not Match(scope_or_label_pattern, cleansed_line) and ++ not (clean_lines.raw_lines[linenum] != line and ++ Match(r'^\s*""', line))): ++ error(filename, linenum, 'whitespace/indent', 3, ++ 'Weird number of spaces at line-start. ' ++ 'Are you using a 2-space indent?') ++ ++ # Check if the line is a header guard. ++ is_header_guard = False ++ if file_extension == 'h': ++ cppvar = GetHeaderGuardCPPVariable(filename) ++ if (line.startswith('#ifndef %s' % cppvar) or ++ line.startswith('#define %s' % cppvar) or ++ line.startswith('#endif // %s' % cppvar)): ++ is_header_guard = True ++ # #include lines and header guards can be long, since there's no clean way to ++ # split them. ++ # ++ # URLs can be long too. It's possible to split these, but it makes them ++ # harder to cut&paste. ++ # ++ # The "$Id:...$" comment may also get very long without it being the ++ # developers fault. ++ if (not line.startswith('#include') and not is_header_guard and ++ not Match(r'^\s*//.*http(s?)://\S*$', line) and ++ not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): ++ line_width = GetLineWidth(line) ++ extended_length = int((_line_length * 1.25)) ++ if line_width > extended_length: ++ error(filename, linenum, 'whitespace/line_length', 4, ++ 'Lines should very rarely be longer than %i characters' % ++ extended_length) ++ elif line_width > _line_length: ++ error(filename, linenum, 'whitespace/line_length', 2, ++ 'Lines should be <= %i characters long' % _line_length) ++ ++ if (cleansed_line.count(';') > 1 and ++ # for loops are allowed two ;'s (and may run over two lines). ++ cleansed_line.find('for') == -1 and ++ (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or ++ GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and ++ # It's ok to have many commands in a switch case that fits in 1 line ++ not ((cleansed_line.find('case ') != -1 or ++ cleansed_line.find('default:') != -1) and ++ cleansed_line.find('break;') != -1)): ++ error(filename, linenum, 'whitespace/newline', 0, ++ 'More than one command on the same line') ++ ++ # Some more style checks ++ CheckBraces(filename, clean_lines, linenum, error) ++ CheckTrailingSemicolon(filename, clean_lines, linenum, error) ++ CheckEmptyBlockBody(filename, clean_lines, linenum, error) ++ CheckAccess(filename, clean_lines, linenum, nesting_state, error) ++ CheckSpacing(filename, clean_lines, linenum, nesting_state, error) ++ CheckOperatorSpacing(filename, clean_lines, linenum, error) ++ CheckParenthesisSpacing(filename, clean_lines, linenum, error) ++ CheckCommaSpacing(filename, clean_lines, linenum, error) ++ CheckBracesSpacing(filename, clean_lines, linenum, error) ++ CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) ++ CheckRValueReference(filename, clean_lines, linenum, nesting_state, error) ++ CheckCheck(filename, clean_lines, linenum, error) ++ CheckAltTokens(filename, clean_lines, linenum, error) ++ classinfo = nesting_state.InnermostClass() ++ if classinfo: ++ CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) ++ ++ ++_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') ++# Matches the first component of a filename delimited by -s and _s. That is: ++# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' ++# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' ++# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' ++# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' ++_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') ++ ++ ++def _DropCommonSuffixes(filename): ++ """Drops common suffixes like _test.cc or -inl.h from filename. ++ ++ For example: ++ >>> _DropCommonSuffixes('foo/foo-inl.h') ++ 'foo/foo' ++ >>> _DropCommonSuffixes('foo/bar/foo.cc') ++ 'foo/bar/foo' ++ >>> _DropCommonSuffixes('foo/foo_internal.h') ++ 'foo/foo' ++ >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') ++ 'foo/foo_unusualinternal' ++ ++ Args: ++ filename: The input filename. ++ ++ Returns: ++ The filename with the common suffix removed. ++ """ ++ for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', ++ 'inl.h', 'impl.h', 'internal.h'): ++ if (filename.endswith(suffix) and len(filename) > len(suffix) and ++ filename[-len(suffix) - 1] in ('-', '_')): ++ return filename[:-len(suffix) - 1] ++ return os.path.splitext(filename)[0] ++ ++ ++def _IsTestFilename(filename): ++ """Determines if the given filename has a suffix that identifies it as a test. ++ ++ Args: ++ filename: The input filename. ++ ++ Returns: ++ True if 'filename' looks like a test, False otherwise. ++ """ ++ if (filename.endswith('_test.cc') or ++ filename.endswith('_unittest.cc') or ++ filename.endswith('_regtest.cc')): ++ return True ++ else: ++ return False ++ ++ ++def _ClassifyInclude(fileinfo, include, is_system): ++ """Figures out what kind of header 'include' is. ++ ++ Args: ++ fileinfo: The current file cpplint is running over. A FileInfo instance. ++ include: The path to a #included file. ++ is_system: True if the #include used <> rather than "". ++ ++ Returns: ++ One of the _XXX_HEADER constants. ++ ++ For example: ++ >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) ++ _C_SYS_HEADER ++ >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) ++ _CPP_SYS_HEADER ++ >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) ++ _LIKELY_MY_HEADER ++ >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), ++ ... 'bar/foo_other_ext.h', False) ++ _POSSIBLE_MY_HEADER ++ >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) ++ _OTHER_HEADER ++ """ ++ # This is a list of all standard c++ header files, except ++ # those already checked for above. ++ is_cpp_h = include in _CPP_HEADERS ++ ++ if is_system: ++ if is_cpp_h: ++ return _CPP_SYS_HEADER ++ else: ++ return _C_SYS_HEADER ++ ++ # If the target file and the include we're checking share a ++ # basename when we drop common extensions, and the include ++ # lives in . , then it's likely to be owned by the target file. ++ target_dir, target_base = ( ++ os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) ++ include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) ++ if target_base == include_base and ( ++ include_dir == target_dir or ++ include_dir == os.path.normpath(target_dir + '/../public')): ++ return _LIKELY_MY_HEADER ++ ++ # If the target and include share some initial basename ++ # component, it's possible the target is implementing the ++ # include, so it's allowed to be first, but we'll never ++ # complain if it's not there. ++ target_first_component = _RE_FIRST_COMPONENT.match(target_base) ++ include_first_component = _RE_FIRST_COMPONENT.match(include_base) ++ if (target_first_component and include_first_component and ++ target_first_component.group(0) == ++ include_first_component.group(0)): ++ return _POSSIBLE_MY_HEADER ++ ++ return _OTHER_HEADER ++ ++ ++ ++def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): ++ """Check rules that are applicable to #include lines. ++ ++ Strings on #include lines are NOT removed from elided line, to make ++ certain tasks easier. However, to prevent false positives, checks ++ applicable to #include lines in CheckLanguage must be put here. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ include_state: An _IncludeState instance in which the headers are inserted. ++ error: The function to call with any errors found. ++ """ ++ fileinfo = FileInfo(filename) ++ line = clean_lines.lines[linenum] ++ ++ # "include" should use the new style "foo/bar.h" instead of just "bar.h" ++ # Only do this check if the included header follows google naming ++ # conventions. If not, assume that it's a 3rd party API that ++ # requires special include conventions. ++ # ++ # We also make an exception for Lua headers, which follow google ++ # naming convention but not the include convention. ++ match = Match(r'#include\s*"([^/]+\.h)"', line) ++ if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): ++ error(filename, linenum, 'build/include', 4, ++ 'Include the directory when naming .h files') ++ ++ # we shouldn't include a file more than once. actually, there are a ++ # handful of instances where doing so is okay, but in general it's ++ # not. ++ match = _RE_PATTERN_INCLUDE.search(line) ++ if match: ++ include = match.group(2) ++ is_system = (match.group(1) == '<') ++ duplicate_line = include_state.FindHeader(include) ++ if duplicate_line >= 0: ++ error(filename, linenum, 'build/include', 4, ++ '"%s" already included at %s:%s' % ++ (include, filename, duplicate_line)) ++ elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): ++ include_state.include_list[-1].append((include, linenum)) ++ ++ # We want to ensure that headers appear in the right order: ++ # 1) for foo.cc, foo.h (preferred location) ++ # 2) c system files ++ # 3) cpp system files ++ # 4) for foo.cc, foo.h (deprecated location) ++ # 5) other google headers ++ # ++ # We classify each include statement as one of those 5 types ++ # using a number of techniques. The include_state object keeps ++ # track of the highest type seen, and complains if we see a ++ # lower type after that. ++ error_message = include_state.CheckNextIncludeOrder( ++ _ClassifyInclude(fileinfo, include, is_system)) ++ if error_message: ++ error(filename, linenum, 'build/include_order', 4, ++ '%s. Should be: %s.h, c system, c++ system, other.' % ++ (error_message, fileinfo.BaseName())) ++ canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) ++ if not include_state.IsInAlphabeticalOrder( ++ clean_lines, linenum, canonical_include): ++ error(filename, linenum, 'build/include_alpha', 4, ++ 'Include "%s" not in alphabetical order' % include) ++ include_state.SetLastHeader(canonical_include) ++ ++ # Look for any of the stream classes that are part of standard C++. ++ match = _RE_PATTERN_INCLUDE.match(line) ++ if match: ++ include = match.group(2) ++ if Match(r'(f|ind|io|i|o|parse|pf|stdio|str|)?stream$', include): ++ # Many unit tests use cout, so we exempt them. ++ if not _IsTestFilename(filename): ++ # Suggest a different header for ostream ++ if include == 'ostream': ++ error(filename, linenum, 'readability/streams', 3, ++ 'For logging, include "base/logging.h" instead of .') ++ else: ++ error(filename, linenum, 'readability/streams', 3, ++ 'Streams are highly discouraged.') ++ ++ ++def _GetTextInside(text, start_pattern): ++ r"""Retrieves all the text between matching open and close parentheses. ++ ++ Given a string of lines and a regular expression string, retrieve all the text ++ following the expression and between opening punctuation symbols like ++ (, [, or {, and the matching close-punctuation symbol. This properly nested ++ occurrences of the punctuations, so for the text like ++ printf(a(), b(c())); ++ a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. ++ start_pattern must match string having an open punctuation symbol at the end. ++ ++ Args: ++ text: The lines to extract text. Its comments and strings must be elided. ++ It can be single line and can span multiple lines. ++ start_pattern: The regexp string indicating where to start extracting ++ the text. ++ Returns: ++ The extracted text. ++ None if either the opening string or ending punctuation could not be found. ++ """ ++ # TODO(unknown): Audit cpplint.py to see what places could be profitably ++ # rewritten to use _GetTextInside (and use inferior regexp matching today). ++ ++ # Give opening punctuations to get the matching close-punctuations. ++ matching_punctuation = {'(': ')', '{': '}', '[': ']'} ++ closing_punctuation = set(matching_punctuation.itervalues()) ++ ++ # Find the position to start extracting text. ++ match = re.search(start_pattern, text, re.M) ++ if not match: # start_pattern not found in text. ++ return None ++ start_position = match.end(0) ++ ++ assert start_position > 0, ( ++ 'start_pattern must ends with an opening punctuation.') ++ assert text[start_position - 1] in matching_punctuation, ( ++ 'start_pattern must ends with an opening punctuation.') ++ # Stack of closing punctuations we expect to have in text after position. ++ punctuation_stack = [matching_punctuation[text[start_position - 1]]] ++ position = start_position ++ while punctuation_stack and position < len(text): ++ if text[position] == punctuation_stack[-1]: ++ punctuation_stack.pop() ++ elif text[position] in closing_punctuation: ++ # A closing punctuation without matching opening punctuations. ++ return None ++ elif text[position] in matching_punctuation: ++ punctuation_stack.append(matching_punctuation[text[position]]) ++ position += 1 ++ if punctuation_stack: ++ # Opening punctuations left without matching close-punctuations. ++ return None ++ # punctuations match. ++ return text[start_position:position - 1] ++ ++ ++# Patterns for matching call-by-reference parameters. ++# ++# Supports nested templates up to 2 levels deep using this messy pattern: ++# < (?: < (?: < [^<>]* ++# > ++# | [^<>] )* ++# > ++# | [^<>] )* ++# > ++_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* ++_RE_PATTERN_TYPE = ( ++ r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' ++ r'(?:\w|' ++ r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' ++ r'::)+') ++# A call-by-reference parameter ends with '& identifier'. ++_RE_PATTERN_REF_PARAM = re.compile( ++ r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' ++ r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') ++# A call-by-const-reference parameter either ends with 'const& identifier' ++# or looks like 'const type& identifier' when 'type' is atomic. ++_RE_PATTERN_CONST_REF_PARAM = ( ++ r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + ++ r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') ++ ++ ++def CheckLanguage(filename, clean_lines, linenum, file_extension, ++ include_state, nesting_state, error): ++ """Checks rules from the 'C++ language rules' section of cppguide.html. ++ ++ Some of these rules are hard to test (function overloading, using ++ uint32 inappropriately), but we do the best we can. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ file_extension: The extension (without the dot) of the filename. ++ include_state: An _IncludeState instance in which the headers are inserted. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ # If the line is empty or consists of entirely a comment, no need to ++ # check it. ++ line = clean_lines.elided[linenum] ++ if not line: ++ return ++ ++ match = _RE_PATTERN_INCLUDE.search(line) ++ if match: ++ CheckIncludeLine(filename, clean_lines, linenum, include_state, error) ++ return ++ ++ # Reset include state across preprocessor directives. This is meant ++ # to silence warnings for conditional includes. ++ match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) ++ if match: ++ include_state.ResetSection(match.group(1)) ++ ++ # Make Windows paths like Unix. ++ fullname = os.path.abspath(filename).replace('\\', '/') ++ ++ # Perform other checks now that we are sure that this is not an include line ++ CheckCasts(filename, clean_lines, linenum, error) ++ CheckGlobalStatic(filename, clean_lines, linenum, error) ++ CheckPrintf(filename, clean_lines, linenum, error) ++ ++ if file_extension == 'h': ++ # TODO(unknown): check that 1-arg constructors are explicit. ++ # How to tell it's a constructor? ++ # (handled in CheckForNonStandardConstructs for now) ++ # TODO(unknown): check that classes declare or disable copy/assign ++ # (level 1 error) ++ pass ++ ++ # Check if people are using the verboten C basic types. The only exception ++ # we regularly allow is "unsigned short port" for port. ++ if Search(r'\bshort port\b', line): ++ if not Search(r'\bunsigned short port\b', line): ++ error(filename, linenum, 'runtime/int', 4, ++ 'Use "unsigned short" for ports, not "short"') ++ else: ++ match = Search(r'\b(short|long(?! +double)|long long)\b', line) ++ if match: ++ error(filename, linenum, 'runtime/int', 4, ++ 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) ++ ++ # Check if some verboten operator overloading is going on ++ # TODO(unknown): catch out-of-line unary operator&: ++ # class X {}; ++ # int operator&(const X& x) { return 42; } // unary operator& ++ # The trick is it's hard to tell apart from binary operator&: ++ # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& ++ if Search(r'\boperator\s*&\s*\(\s*\)', line): ++ error(filename, linenum, 'runtime/operator', 4, ++ 'Unary operator& is dangerous. Do not use it.') ++ ++ # Check for suspicious usage of "if" like ++ # } if (a == b) { ++ if Search(r'\}\s*if\s*\(', line): ++ error(filename, linenum, 'readability/braces', 4, ++ 'Did you mean "else if"? If not, start a new line for "if".') ++ ++ # Check for potential format string bugs like printf(foo). ++ # We constrain the pattern not to pick things like DocidForPrintf(foo). ++ # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) ++ # TODO(unknown): Catch the following case. Need to change the calling ++ # convention of the whole function to process multiple line to handle it. ++ # printf( ++ # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); ++ printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') ++ if printf_args: ++ match = Match(r'([\w.\->()]+)$', printf_args) ++ if match and match.group(1) != '__VA_ARGS__': ++ function_name = re.search(r'\b((?:string)?printf)\s*\(', ++ line, re.I).group(1) ++ error(filename, linenum, 'runtime/printf', 4, ++ 'Potential format string bug. Do %s("%%s", %s) instead.' ++ % (function_name, match.group(1))) ++ ++ # Check for potential memset bugs like memset(buf, sizeof(buf), 0). ++ match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) ++ if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): ++ error(filename, linenum, 'runtime/memset', 4, ++ 'Did you mean "memset(%s, 0, %s)"?' ++ % (match.group(1), match.group(2))) ++ ++ if Search(r'\busing namespace\b', line): ++ error(filename, linenum, 'build/namespaces', 5, ++ 'Do not use namespace using-directives. ' ++ 'Use using-declarations instead.') ++ ++ # Detect variable-length arrays. ++ match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) ++ if (match and match.group(2) != 'return' and match.group(2) != 'delete' and ++ match.group(3).find(']') == -1): ++ # Split the size using space and arithmetic operators as delimiters. ++ # If any of the resulting tokens are not compile time constants then ++ # report the error. ++ tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) ++ is_const = True ++ skip_next = False ++ for tok in tokens: ++ if skip_next: ++ skip_next = False ++ continue ++ ++ if Search(r'sizeof\(.+\)', tok): continue ++ if Search(r'arraysize\(\w+\)', tok): continue ++ ++ tok = tok.lstrip('(') ++ tok = tok.rstrip(')') ++ if not tok: continue ++ if Match(r'\d+', tok): continue ++ if Match(r'0[xX][0-9a-fA-F]+', tok): continue ++ if Match(r'k[A-Z0-9]\w*', tok): continue ++ if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue ++ if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue ++ # A catch all for tricky sizeof cases, including 'sizeof expression', ++ # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' ++ # requires skipping the next token because we split on ' ' and '*'. ++ if tok.startswith('sizeof'): ++ skip_next = True ++ continue ++ is_const = False ++ break ++ if not is_const: ++ error(filename, linenum, 'runtime/arrays', 1, ++ 'Do not use variable-length arrays. Use an appropriately named ' ++ "('k' followed by CamelCase) compile-time constant for the size.") ++ ++ # If DISALLOW_COPY_AND_ASSIGN DISALLOW_IMPLICIT_CONSTRUCTORS is present, ++ # then it should be the last thing in the class declaration. ++ match = Match( ++ (r'\s*' ++ r'(DISALLOW_(COPY_AND_ASSIGN|IMPLICIT_CONSTRUCTORS))' ++ r'\(.*\);$'), ++ line) ++ if match and linenum + 1 < clean_lines.NumLines(): ++ next_line = clean_lines.elided[linenum + 1] ++ # We allow some, but not all, declarations of variables to be present ++ # in the statement that defines the class. The [\w\*,\s]* fragment of ++ # the regular expression below allows users to declare instances of ++ # the class or pointers to instances, but not less common types such ++ # as function pointers or arrays. It's a tradeoff between allowing ++ # reasonable code and avoiding trying to parse more C++ using regexps. ++ if not Search(r'^\s*}[\w\*,\s]*;', next_line): ++ error(filename, linenum, 'readability/constructors', 3, ++ match.group(1) + ' should be the last thing in the class') ++ ++ # Check for use of unnamed namespaces in header files. Registration ++ # macros are typically OK, so we allow use of "namespace {" on lines ++ # that end with backslashes. ++ if (file_extension == 'h' ++ and Search(r'\bnamespace\s*{', line) ++ and line[-1] != '\\'): ++ error(filename, linenum, 'build/namespaces', 4, ++ 'Do not use unnamed namespaces in header files. See ' ++ 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' ++ ' for more information.') ++ ++ ++def CheckGlobalStatic(filename, clean_lines, linenum, error): ++ """Check for unsafe global or static objects. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Match two lines at a time to support multiline declarations ++ if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): ++ line += clean_lines.elided[linenum + 1].strip() ++ ++ # Check for people declaring static/global STL strings at the top level. ++ # This is dangerous because the C++ language does not guarantee that ++ # globals with constructors are initialized before the first access. ++ match = Match( ++ r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', ++ line) ++ ++ # Remove false positives: ++ # - String pointers (as opposed to values). ++ # string *pointer ++ # const string *pointer ++ # string const *pointer ++ # string *const pointer ++ # ++ # - Functions and template specializations. ++ # string Function(... ++ # string Class::Method(... ++ # ++ # - Operators. These are matched separately because operator names ++ # cross non-word boundaries, and trying to match both operators ++ # and functions at the same time would decrease accuracy of ++ # matching identifiers. ++ # string Class::operator*() ++ if (match and ++ not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and ++ not Search(r'\boperator\W', line) and ++ not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))): ++ error(filename, linenum, 'runtime/string', 4, ++ 'For a static/global string constant, use a C style string instead: ' ++ '"%schar %s[]".' % ++ (match.group(1), match.group(2))) ++ ++ if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): ++ error(filename, linenum, 'runtime/init', 4, ++ 'You seem to be initializing a member variable with itself.') ++ ++ ++def CheckPrintf(filename, clean_lines, linenum, error): ++ """Check for printf related issues. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # When snprintf is used, the second argument shouldn't be a literal. ++ match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) ++ if match and match.group(2) != '0': ++ # If 2nd arg is zero, snprintf is used to calculate size. ++ error(filename, linenum, 'runtime/printf', 3, ++ 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' ++ 'to snprintf.' % (match.group(1), match.group(2))) ++ ++ # Check if some verboten C functions are being used. ++ if Search(r'\bsprintf\s*\(', line): ++ error(filename, linenum, 'runtime/printf', 5, ++ 'Never use sprintf. Use snprintf instead.') ++ match = Search(r'\b(strcpy|strcat)\s*\(', line) ++ if match: ++ error(filename, linenum, 'runtime/printf', 4, ++ 'Almost always, snprintf is better than %s' % match.group(1)) ++ ++ ++def IsDerivedFunction(clean_lines, linenum): ++ """Check if current line contains an inherited function. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ Returns: ++ True if current line contains a function with "override" ++ virt-specifier. ++ """ ++ # Scan back a few lines for start of current function ++ for i in xrange(linenum, max(-1, linenum - 10), -1): ++ match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) ++ if match: ++ # Look for "override" after the matching closing parenthesis ++ line, _, closing_paren = CloseExpression( ++ clean_lines, i, len(match.group(1))) ++ return (closing_paren >= 0 and ++ Search(r'\boverride\b', line[closing_paren:])) ++ return False ++ ++ ++def IsInitializerList(clean_lines, linenum): ++ """Check if current line is inside constructor initializer list. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ Returns: ++ True if current line appears to be inside constructor initializer ++ list, False otherwise. ++ """ ++ for i in xrange(linenum, 1, -1): ++ line = clean_lines.elided[i] ++ if i == linenum: ++ remove_function_body = Match(r'^(.*)\{\s*$', line) ++ if remove_function_body: ++ line = remove_function_body.group(1) ++ ++ if Search(r'\s:\s*\w+[({]', line): ++ # A lone colon tend to indicate the start of a constructor ++ # initializer list. It could also be a ternary operator, which ++ # also tend to appear in constructor initializer lists as ++ # opposed to parameter lists. ++ return True ++ if Search(r'\}\s*,\s*$', line): ++ # A closing brace followed by a comma is probably the end of a ++ # brace-initialized member in constructor initializer list. ++ return True ++ if Search(r'[{};]\s*$', line): ++ # Found one of the following: ++ # - A closing brace or semicolon, probably the end of the previous ++ # function. ++ # - An opening brace, probably the start of current class or namespace. ++ # ++ # Current line is probably not inside an initializer list since ++ # we saw one of those things without seeing the starting colon. ++ return False ++ ++ # Got to the beginning of the file without seeing the start of ++ # constructor initializer list. ++ return False ++ ++ ++def CheckForNonConstReference(filename, clean_lines, linenum, ++ nesting_state, error): ++ """Check for non-const references. ++ ++ Separate from CheckLanguage since it scans backwards from current ++ line, instead of scanning forward. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: The function to call with any errors found. ++ """ ++ # Do nothing if there is no '&' on current line. ++ line = clean_lines.elided[linenum] ++ if '&' not in line: ++ return ++ ++ # If a function is inherited, current function doesn't have much of ++ # a choice, so any non-const references should not be blamed on ++ # derived function. ++ if IsDerivedFunction(clean_lines, linenum): ++ return ++ ++ # Long type names may be broken across multiple lines, usually in one ++ # of these forms: ++ # LongType ++ # ::LongTypeContinued &identifier ++ # LongType:: ++ # LongTypeContinued &identifier ++ # LongType< ++ # ...>::LongTypeContinued &identifier ++ # ++ # If we detected a type split across two lines, join the previous ++ # line to current line so that we can match const references ++ # accordingly. ++ # ++ # Note that this only scans back one line, since scanning back ++ # arbitrary number of lines would be expensive. If you have a type ++ # that spans more than 2 lines, please use a typedef. ++ if linenum > 1: ++ previous = None ++ if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): ++ # previous_line\n + ::current_line ++ previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', ++ clean_lines.elided[linenum - 1]) ++ elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): ++ # previous_line::\n + current_line ++ previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', ++ clean_lines.elided[linenum - 1]) ++ if previous: ++ line = previous.group(1) + line.lstrip() ++ else: ++ # Check for templated parameter that is split across multiple lines ++ endpos = line.rfind('>') ++ if endpos > -1: ++ (_, startline, startpos) = ReverseCloseExpression( ++ clean_lines, linenum, endpos) ++ if startpos > -1 and startline < linenum: ++ # Found the matching < on an earlier line, collect all ++ # pieces up to current line. ++ line = '' ++ for i in xrange(startline, linenum + 1): ++ line += clean_lines.elided[i].strip() ++ ++ # Check for non-const references in function parameters. A single '&' may ++ # found in the following places: ++ # inside expression: binary & for bitwise AND ++ # inside expression: unary & for taking the address of something ++ # inside declarators: reference parameter ++ # We will exclude the first two cases by checking that we are not inside a ++ # function body, including one that was just introduced by a trailing '{'. ++ # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. ++ if (nesting_state.previous_stack_top and ++ not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or ++ isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): ++ # Not at toplevel, not within a class, and not within a namespace ++ return ++ ++ # Avoid initializer lists. We only need to scan back from the ++ # current line for something that starts with ':'. ++ # ++ # We don't need to check the current line, since the '&' would ++ # appear inside the second set of parentheses on the current line as ++ # opposed to the first set. ++ if linenum > 0: ++ for i in xrange(linenum - 1, max(0, linenum - 10), -1): ++ previous_line = clean_lines.elided[i] ++ if not Search(r'[),]\s*$', previous_line): ++ break ++ if Match(r'^\s*:\s+\S', previous_line): ++ return ++ ++ # Avoid preprocessors ++ if Search(r'\\\s*$', line): ++ return ++ ++ # Avoid constructor initializer lists ++ if IsInitializerList(clean_lines, linenum): ++ return ++ ++ # We allow non-const references in a few standard places, like functions ++ # called "swap()" or iostream operators like "<<" or ">>". Do not check ++ # those function parameters. ++ # ++ # We also accept & in static_assert, which looks like a function but ++ # it's actually a declaration expression. ++ whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' ++ r'operator\s*[<>][<>]|' ++ r'static_assert|COMPILE_ASSERT' ++ r')\s*\(') ++ if Search(whitelisted_functions, line): ++ return ++ elif not Search(r'\S+\([^)]*$', line): ++ # Don't see a whitelisted function on this line. Actually we ++ # didn't see any function name on this line, so this is likely a ++ # multi-line parameter list. Try a bit harder to catch this case. ++ for i in xrange(2): ++ if (linenum > i and ++ Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): ++ return ++ ++ decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body ++ for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): ++ if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): ++ error(filename, linenum, 'runtime/references', 2, ++ 'Is this a non-const reference? ' ++ 'If so, make const or use a pointer: ' + ++ ReplaceAll(' *<', '<', parameter)) ++ ++ ++def CheckCasts(filename, clean_lines, linenum, error): ++ """Various cast related checks. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Check to see if they're using an conversion function cast. ++ # I just try to capture the most common basic types, though there are more. ++ # Parameterless conversion functions, such as bool(), are allowed as they are ++ # probably a member operator declaration or default constructor. ++ match = Search( ++ r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b' ++ r'(int|float|double|bool|char|int32|uint32|int64|uint64)' ++ r'(\([^)].*)', line) ++ expecting_function = ExpectingFunctionArgs(clean_lines, linenum) ++ if match and not expecting_function: ++ matched_type = match.group(2) ++ ++ # matched_new_or_template is used to silence two false positives: ++ # - New operators ++ # - Template arguments with function types ++ # ++ # For template arguments, we match on types immediately following ++ # an opening bracket without any spaces. This is a fast way to ++ # silence the common case where the function type is the first ++ # template argument. False negative with less-than comparison is ++ # avoided because those operators are usually followed by a space. ++ # ++ # function // bracket + no space = false positive ++ # value < double(42) // bracket + space = true positive ++ matched_new_or_template = match.group(1) ++ ++ # Avoid arrays by looking for brackets that come after the closing ++ # parenthesis. ++ if Match(r'\([^()]+\)\s*\[', match.group(3)): ++ return ++ ++ # Other things to ignore: ++ # - Function pointers ++ # - Casts to pointer types ++ # - Placement new ++ # - Alias declarations ++ matched_funcptr = match.group(3) ++ if (matched_new_or_template is None and ++ not (matched_funcptr and ++ (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', ++ matched_funcptr) or ++ matched_funcptr.startswith('(*)'))) and ++ not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and ++ not Search(r'new\(\S+\)\s*' + matched_type, line)): ++ error(filename, linenum, 'readability/casting', 4, ++ 'Using deprecated casting style. ' ++ 'Use static_cast<%s>(...) instead' % ++ matched_type) ++ ++ if not expecting_function: ++ CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', ++ r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) ++ ++ # This doesn't catch all cases. Consider (const char * const)"hello". ++ # ++ # (char *) "foo" should always be a const_cast (reinterpret_cast won't ++ # compile). ++ if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', ++ r'\((char\s?\*+\s?)\)\s*"', error): ++ pass ++ else: ++ # Check pointer casts for other than string constants ++ CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', ++ r'\((\w+\s?\*+\s?)\)', error) ++ ++ # In addition, we look for people taking the address of a cast. This ++ # is dangerous -- casts can assign to temporaries, so the pointer doesn't ++ # point where you think. ++ # ++ # Some non-identifier character is required before the '&' for the ++ # expression to be recognized as a cast. These are casts: ++ # expression = &static_cast(temporary()); ++ # function(&(int*)(temporary())); ++ # ++ # This is not a cast: ++ # reference_type&(int* function_param); ++ match = Search( ++ r'(?:[^\w]&\(([^)]+)\)[\w(])|' ++ r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) ++ if match and match.group(1) != '*': ++ # Try a better error message when the & is bound to something ++ # dereferenced by the casted pointer, as opposed to the casted ++ # pointer itself. ++ parenthesis_error = False ++ match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) ++ if match: ++ _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) ++ if x1 >= 0 and clean_lines.elided[y1][x1] == '(': ++ _, y2, x2 = CloseExpression(clean_lines, y1, x1) ++ if x2 >= 0: ++ extended_line = clean_lines.elided[y2][x2:] ++ if y2 < clean_lines.NumLines() - 1: ++ extended_line += clean_lines.elided[y2 + 1] ++ if Match(r'\s*(?:->|\[)', extended_line): ++ parenthesis_error = True ++ ++ if parenthesis_error: ++ error(filename, linenum, 'readability/casting', 4, ++ ('Are you taking an address of something dereferenced ' ++ 'from a cast? Wrapping the dereferenced expression in ' ++ 'parentheses will make the binding more obvious')) ++ else: ++ error(filename, linenum, 'runtime/casting', 4, ++ ('Are you taking an address of a cast? ' ++ 'This is dangerous: could be a temp var. ' ++ 'Take the address before doing the cast, rather than after')) ++ ++ ++def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): ++ """Checks for a C-style cast by looking for the pattern. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ cast_type: The string for the C++ cast to recommend. This is either ++ reinterpret_cast, static_cast, or const_cast, depending. ++ pattern: The regular expression used to find C-style casts. ++ error: The function to call with any errors found. ++ ++ Returns: ++ True if an error was emitted. ++ False otherwise. ++ """ ++ line = clean_lines.elided[linenum] ++ match = Search(pattern, line) ++ if not match: ++ return False ++ ++ # Exclude lines with keywords that tend to look like casts ++ context = line[0:match.start(1) - 1] ++ if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): ++ return False ++ ++ # Try expanding current context to see if we one level of ++ # parentheses inside a macro. ++ if linenum > 0: ++ for i in xrange(linenum - 1, max(0, linenum - 5), -1): ++ context = clean_lines.elided[i] + context ++ if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): ++ return False ++ ++ # operator++(int) and operator--(int) ++ if context.endswith(' operator++') or context.endswith(' operator--'): ++ return False ++ ++ # A single unnamed argument for a function tends to look like old ++ # style cast. If we see those, don't issue warnings for deprecated ++ # casts, instead issue warnings for unnamed arguments where ++ # appropriate. ++ # ++ # These are things that we want warnings for, since the style guide ++ # explicitly require all parameters to be named: ++ # Function(int); ++ # Function(int) { ++ # ConstMember(int) const; ++ # ConstMember(int) const { ++ # ExceptionMember(int) throw (...); ++ # ExceptionMember(int) throw (...) { ++ # PureVirtual(int) = 0; ++ # ++ # These are functions of some sort, where the compiler would be fine ++ # if they had named parameters, but people often omit those ++ # identifiers to reduce clutter: ++ # (FunctionPointer)(int); ++ # (FunctionPointer)(int) = value; ++ # Function((function_pointer_arg)(int)) ++ # Function((function_pointer_arg)(int), int param) ++ # ; ++ # <(FunctionPointerTemplateArgument)(int)>; ++ remainder = line[match.end(0):] ++ if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),])', ++ remainder): ++ # Looks like an unnamed parameter. ++ ++ # Don't warn on any kind of template arguments. ++ if Match(r'^\s*>', remainder): ++ return False ++ ++ # Don't warn on assignments to function pointers, but keep warnings for ++ # unnamed parameters to pure virtual functions. Note that this pattern ++ # will also pass on assignments of "0" to function pointers, but the ++ # preferred values for those would be "nullptr" or "NULL". ++ matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) ++ if matched_zero and matched_zero.group(1) != '0': ++ return False ++ ++ # Don't warn on function pointer declarations. For this we need ++ # to check what came before the "(type)" string. ++ if Match(r'.*\)\s*$', line[0:match.start(0)]): ++ return False ++ ++ # Don't warn if the parameter is named with block comments, e.g.: ++ # Function(int /*unused_param*/); ++ raw_line = clean_lines.raw_lines[linenum] ++ if '/*' in raw_line: ++ return False ++ ++ # Passed all filters, issue warning here. ++ error(filename, linenum, 'readability/function', 3, ++ 'All parameters should be named in a function') ++ return True ++ ++ # At this point, all that should be left is actual casts. ++ error(filename, linenum, 'readability/casting', 4, ++ 'Using C-style cast. Use %s<%s>(...) instead' % ++ (cast_type, match.group(1))) ++ ++ return True ++ ++ ++def ExpectingFunctionArgs(clean_lines, linenum): ++ """Checks whether where function type arguments are expected. ++ ++ Args: ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ ++ Returns: ++ True if the line at 'linenum' is inside something that expects arguments ++ of function types. ++ """ ++ line = clean_lines.elided[linenum] ++ return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or ++ (linenum >= 2 and ++ (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', ++ clean_lines.elided[linenum - 1]) or ++ Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', ++ clean_lines.elided[linenum - 2]) or ++ Search(r'\bstd::m?function\s*\<\s*$', ++ clean_lines.elided[linenum - 1])))) ++ ++ ++_HEADERS_CONTAINING_TEMPLATES = ( ++ ('', ('deque',)), ++ ('', ('unary_function', 'binary_function', ++ 'plus', 'minus', 'multiplies', 'divides', 'modulus', ++ 'negate', ++ 'equal_to', 'not_equal_to', 'greater', 'less', ++ 'greater_equal', 'less_equal', ++ 'logical_and', 'logical_or', 'logical_not', ++ 'unary_negate', 'not1', 'binary_negate', 'not2', ++ 'bind1st', 'bind2nd', ++ 'pointer_to_unary_function', ++ 'pointer_to_binary_function', ++ 'ptr_fun', ++ 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', ++ 'mem_fun_ref_t', ++ 'const_mem_fun_t', 'const_mem_fun1_t', ++ 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', ++ 'mem_fun_ref', ++ )), ++ ('', ('numeric_limits',)), ++ ('', ('list',)), ++ ('', ('map', 'multimap',)), ++ ('', ('allocator',)), ++ ('', ('queue', 'priority_queue',)), ++ ('', ('set', 'multiset',)), ++ ('', ('stack',)), ++ ('', ('char_traits', 'basic_string',)), ++ ('', ('pair',)), ++ ('', ('vector',)), ++ ++ # gcc extensions. ++ # Note: std::hash is their hash, ::hash is our hash ++ ('', ('hash_map', 'hash_multimap',)), ++ ('', ('hash_set', 'hash_multiset',)), ++ ('', ('slist',)), ++ ) ++ ++_RE_PATTERN_STRING = re.compile(r'\bstring\b') ++ ++_re_pattern_algorithm_header = [] ++for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', ++ 'transform'): ++ # Match max(..., ...), max(..., ...), but not foo->max, foo.max or ++ # type::max(). ++ _re_pattern_algorithm_header.append( ++ (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), ++ _template, ++ '')) ++ ++_re_pattern_templates = [] ++for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: ++ for _template in _templates: ++ _re_pattern_templates.append( ++ (re.compile(r'(\<|\b)' + _template + r'\s*\<'), ++ _template + '<>', ++ _header)) ++ ++ ++def FilesBelongToSameModule(filename_cc, filename_h): ++ """Check if these two filenames belong to the same module. ++ ++ The concept of a 'module' here is a as follows: ++ foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the ++ same 'module' if they are in the same directory. ++ some/path/public/xyzzy and some/path/internal/xyzzy are also considered ++ to belong to the same module here. ++ ++ If the filename_cc contains a longer path than the filename_h, for example, ++ '/absolute/path/to/base/sysinfo.cc', and this file would include ++ 'base/sysinfo.h', this function also produces the prefix needed to open the ++ header. This is used by the caller of this function to more robustly open the ++ header file. We don't have access to the real include paths in this context, ++ so we need this guesswork here. ++ ++ Known bugs: tools/base/bar.cc and base/bar.h belong to the same module ++ according to this implementation. Because of this, this function gives ++ some false positives. This should be sufficiently rare in practice. ++ ++ Args: ++ filename_cc: is the path for the .cc file ++ filename_h: is the path for the header path ++ ++ Returns: ++ Tuple with a bool and a string: ++ bool: True if filename_cc and filename_h belong to the same module. ++ string: the additional prefix needed to open the header file. ++ """ ++ ++ if not filename_cc.endswith('.cc'): ++ return (False, '') ++ filename_cc = filename_cc[:-len('.cc')] ++ if filename_cc.endswith('_unittest'): ++ filename_cc = filename_cc[:-len('_unittest')] ++ elif filename_cc.endswith('_test'): ++ filename_cc = filename_cc[:-len('_test')] ++ filename_cc = filename_cc.replace('/public/', '/') ++ filename_cc = filename_cc.replace('/internal/', '/') ++ ++ if not filename_h.endswith('.h'): ++ return (False, '') ++ filename_h = filename_h[:-len('.h')] ++ if filename_h.endswith('-inl'): ++ filename_h = filename_h[:-len('-inl')] ++ filename_h = filename_h.replace('/public/', '/') ++ filename_h = filename_h.replace('/internal/', '/') ++ ++ files_belong_to_same_module = filename_cc.endswith(filename_h) ++ common_path = '' ++ if files_belong_to_same_module: ++ common_path = filename_cc[:-len(filename_h)] ++ return files_belong_to_same_module, common_path ++ ++ ++def UpdateIncludeState(filename, include_dict, io=codecs): ++ """Fill up the include_dict with new includes found from the file. ++ ++ Args: ++ filename: the name of the header to read. ++ include_dict: a dictionary in which the headers are inserted. ++ io: The io factory to use to read the file. Provided for testability. ++ ++ Returns: ++ True if a header was successfully added. False otherwise. ++ """ ++ headerfile = None ++ try: ++ headerfile = io.open(filename, 'r', 'utf8', 'replace') ++ except IOError: ++ return False ++ linenum = 0 ++ for line in headerfile: ++ linenum += 1 ++ clean_line = CleanseComments(line) ++ match = _RE_PATTERN_INCLUDE.search(clean_line) ++ if match: ++ include = match.group(2) ++ include_dict.setdefault(include, linenum) ++ return True ++ ++ ++def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, ++ io=codecs): ++ """Reports for missing stl includes. ++ ++ This function will output warnings to make sure you are including the headers ++ necessary for the stl containers and functions that you use. We only give one ++ reason to include a header. For example, if you use both equal_to<> and ++ less<> in a .h file, only one (the latter in the file) of these will be ++ reported as a reason to include the . ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ include_state: An _IncludeState instance. ++ error: The function to call with any errors found. ++ io: The IO factory to use to read the header file. Provided for unittest ++ injection. ++ """ ++ required = {} # A map of header name to linenumber and the template entity. ++ # Example of required: { '': (1219, 'less<>') } ++ ++ for linenum in xrange(clean_lines.NumLines()): ++ line = clean_lines.elided[linenum] ++ if not line or line[0] == '#': ++ continue ++ ++ # String is special -- it is a non-templatized type in STL. ++ matched = _RE_PATTERN_STRING.search(line) ++ if matched: ++ # Don't warn about strings in non-STL namespaces: ++ # (We check only the first match per line; good enough.) ++ prefix = line[:matched.start()] ++ if prefix.endswith('std::') or not prefix.endswith('::'): ++ required[''] = (linenum, 'string') ++ ++ for pattern, template, header in _re_pattern_algorithm_header: ++ if pattern.search(line): ++ required[header] = (linenum, template) ++ ++ # The following function is just a speed up, no semantics are changed. ++ if not '<' in line: # Reduces the cpu time usage by skipping lines. ++ continue ++ ++ for pattern, template, header in _re_pattern_templates: ++ if pattern.search(line): ++ required[header] = (linenum, template) ++ ++ # The policy is that if you #include something in foo.h you don't need to ++ # include it again in foo.cc. Here, we will look at possible includes. ++ # Let's flatten the include_state include_list and copy it into a dictionary. ++ include_dict = dict([item for sublist in include_state.include_list ++ for item in sublist]) ++ ++ # Did we find the header for this file (if any) and successfully load it? ++ header_found = False ++ ++ # Use the absolute path so that matching works properly. ++ abs_filename = FileInfo(filename).FullName() ++ ++ # For Emacs's flymake. ++ # If cpplint is invoked from Emacs's flymake, a temporary file is generated ++ # by flymake and that file name might end with '_flymake.cc'. In that case, ++ # restore original file name here so that the corresponding header file can be ++ # found. ++ # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' ++ # instead of 'foo_flymake.h' ++ abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) ++ ++ # include_dict is modified during iteration, so we iterate over a copy of ++ # the keys. ++ header_keys = include_dict.keys() ++ for header in header_keys: ++ (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) ++ fullpath = common_path + header ++ if same_module and UpdateIncludeState(fullpath, include_dict, io): ++ header_found = True ++ ++ # If we can't find the header file for a .cc, assume it's because we don't ++ # know where to look. In that case we'll give up as we're not sure they ++ # didn't include it in the .h file. ++ # TODO(unknown): Do a better job of finding .h files so we are confident that ++ # not having the .h file means there isn't one. ++ if filename.endswith('.cc') and not header_found: ++ return ++ ++ # All the lines have been processed, report the errors found. ++ for required_header_unstripped in required: ++ template = required[required_header_unstripped][1] ++ if required_header_unstripped.strip('<>"') not in include_dict: ++ error(filename, required[required_header_unstripped][0], ++ 'build/include_what_you_use', 4, ++ 'Add #include ' + required_header_unstripped + ' for ' + template) ++ ++ ++_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') ++ ++ ++def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): ++ """Check that make_pair's template arguments are deduced. ++ ++ G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are ++ specified explicitly, and such use isn't intended in any case. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) ++ if match: ++ error(filename, linenum, 'build/explicit_make_pair', ++ 4, # 4 = high confidence ++ 'For C++11-compatibility, omit template arguments from make_pair' ++ ' OR use pair directly OR if appropriate, construct a pair directly') ++ ++ ++def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error): ++ """Check that default lambda captures are not used. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # A lambda introducer specifies a default capture if it starts with "[=" ++ # or if it starts with "[&" _not_ followed by an identifier. ++ match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line) ++ if match: ++ # Found a potential error, check what comes after the lambda-introducer. ++ # If it's not open parenthesis (for lambda-declarator) or open brace ++ # (for compound-statement), it's not a lambda. ++ line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1))) ++ if pos >= 0 and Match(r'^\s*[{(]', line[pos:]): ++ error(filename, linenum, 'build/c++11', ++ 4, # 4 = high confidence ++ 'Default lambda captures are an unapproved C++ feature.') ++ ++ ++def CheckRedundantVirtual(filename, clean_lines, linenum, error): ++ """Check if line contains a redundant "virtual" function-specifier. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ # Look for "virtual" on current line. ++ line = clean_lines.elided[linenum] ++ virtual = Match(r'^(.*\bvirtual\b)', line) ++ if not virtual: return ++ ++ # Look for the next opening parenthesis. This is the start of the ++ # parameter list (possibly on the next line shortly after virtual). ++ # TODO(unknown): doesn't work if there are virtual functions with ++ # decltype() or other things that use parentheses, but csearch suggests ++ # that this is rare. ++ end_col = -1 ++ end_line = -1 ++ start_col = len(virtual.group(1)) ++ for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): ++ line = clean_lines.elided[start_line][start_col:] ++ parameter_list = Match(r'^([^(]*)\(', line) ++ if parameter_list: ++ # Match parentheses to find the end of the parameter list ++ (_, end_line, end_col) = CloseExpression( ++ clean_lines, start_line, start_col + len(parameter_list.group(1))) ++ break ++ start_col = 0 ++ ++ if end_col < 0: ++ return # Couldn't find end of parameter list, give up ++ ++ # Look for "override" or "final" after the parameter list ++ # (possibly on the next few lines). ++ for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): ++ line = clean_lines.elided[i][end_col:] ++ match = Search(r'\b(override|final)\b', line) ++ if match: ++ error(filename, linenum, 'readability/inheritance', 4, ++ ('"virtual" is redundant since function is ' ++ 'already declared as "%s"' % match.group(1))) ++ ++ # Set end_col to check whole lines after we are done with the ++ # first line. ++ end_col = 0 ++ if Search(r'[^\w]\s*$', line): ++ break ++ ++ ++def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): ++ """Check if line contains a redundant "override" or "final" virt-specifier. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ # Check that at most one of "override" or "final" is present, not both ++ line = clean_lines.elided[linenum] ++ if Search(r'\boverride\b', line) and Search(r'\bfinal\b', line): ++ error(filename, linenum, 'readability/inheritance', 4, ++ ('"override" is redundant since function is ' ++ 'already declared as "final"')) ++ ++ ++ ++ ++# Returns true if we are at a new block, and it is directly ++# inside of a namespace. ++def IsBlockInNameSpace(nesting_state, is_forward_declaration): ++ """Checks that the new block is directly in a namespace. ++ ++ Args: ++ nesting_state: The _NestingState object that contains info about our state. ++ is_forward_declaration: If the class is a forward declared class. ++ Returns: ++ Whether or not the new block is directly in a namespace. ++ """ ++ if is_forward_declaration: ++ if len(nesting_state.stack) >= 1 and ( ++ isinstance(nesting_state.stack[-1], _NamespaceInfo)): ++ return True ++ else: ++ return False ++ ++ return (len(nesting_state.stack) > 1 and ++ nesting_state.stack[-1].check_namespace_indentation and ++ isinstance(nesting_state.stack[-2], _NamespaceInfo)) ++ ++ ++def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, ++ raw_lines_no_comments, linenum): ++ """This method determines if we should apply our namespace indentation check. ++ ++ Args: ++ nesting_state: The current nesting state. ++ is_namespace_indent_item: If we just put a new class on the stack, True. ++ If the top of the stack is not a class, or we did not recently ++ add the class, False. ++ raw_lines_no_comments: The lines without the comments. ++ linenum: The current line number we are processing. ++ ++ Returns: ++ True if we should apply our namespace indentation check. Currently, it ++ only works for classes and namespaces inside of a namespace. ++ """ ++ ++ is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, ++ linenum) ++ ++ if not (is_namespace_indent_item or is_forward_declaration): ++ return False ++ ++ # If we are in a macro, we do not want to check the namespace indentation. ++ if IsMacroDefinition(raw_lines_no_comments, linenum): ++ return False ++ ++ return IsBlockInNameSpace(nesting_state, is_forward_declaration) ++ ++ ++# Call this method if the line is directly inside of a namespace. ++# If the line above is blank (excluding comments) or the start of ++# an inner namespace, it cannot be indented. ++def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, ++ error): ++ line = raw_lines_no_comments[linenum] ++ if Match(r'^\s+', line): ++ error(filename, linenum, 'runtime/indentation_namespace', 4, ++ 'Do not indent within a namespace') ++ ++ ++def ProcessLine(filename, file_extension, clean_lines, line, ++ include_state, function_state, nesting_state, error, ++ extra_check_functions=[]): ++ """Processes a single line in the file. ++ ++ Args: ++ filename: Filename of the file that is being processed. ++ file_extension: The extension (dot not included) of the file. ++ clean_lines: An array of strings, each representing a line of the file, ++ with comments stripped. ++ line: Number of line being processed. ++ include_state: An _IncludeState instance in which the headers are inserted. ++ function_state: A _FunctionState instance which counts function lines, etc. ++ nesting_state: A NestingState instance which maintains information about ++ the current stack of nested blocks being parsed. ++ error: A callable to which errors are reported, which takes 4 arguments: ++ filename, line number, error level, and message ++ extra_check_functions: An array of additional check functions that will be ++ run on each source line. Each function takes 4 ++ arguments: filename, clean_lines, line, error ++ """ ++ raw_lines = clean_lines.raw_lines ++ ParseNolintSuppressions(filename, raw_lines[line], line, error) ++ nesting_state.Update(filename, clean_lines, line, error) ++ CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, ++ error) ++ CheckForMongoPolyfill(filename, clean_lines, line, error) ++ CheckForMongoAtomic(filename, clean_lines, line, error) ++ CheckForMongoVolatile(filename, clean_lines, line, error) ++ CheckForNonMongoAssert(filename, clean_lines, line, error) ++ if nesting_state.InAsmBlock(): return ++ CheckForFunctionLengths(filename, clean_lines, line, function_state, error) ++ CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) ++ CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) ++ CheckLanguage(filename, clean_lines, line, file_extension, include_state, ++ nesting_state, error) ++ CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) ++ CheckForNonStandardConstructs(filename, clean_lines, line, ++ nesting_state, error) ++ CheckVlogArguments(filename, clean_lines, line, error) ++ CheckPosixThreading(filename, clean_lines, line, error) ++ CheckInvalidIncrement(filename, clean_lines, line, error) ++ CheckMakePairUsesDeduction(filename, clean_lines, line, error) ++ CheckDefaultLambdaCaptures(filename, clean_lines, line, error) ++ CheckRedundantVirtual(filename, clean_lines, line, error) ++ CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) ++ for check_fn in extra_check_functions: ++ check_fn(filename, clean_lines, line, error) ++ ++def FlagCxx11Features(filename, clean_lines, linenum, error): ++ """Flag those c++11 features that we only allow in certain places. ++ ++ Args: ++ filename: The name of the current file. ++ clean_lines: A CleansedLines instance containing the file. ++ linenum: The number of the line to check. ++ error: The function to call with any errors found. ++ """ ++ line = clean_lines.elided[linenum] ++ ++ # Flag unapproved C++11 headers. ++ include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) ++ if include and include.group(1) in ('cfenv', ++ 'condition_variable', ++ 'fenv.h', ++ 'future', ++ 'mutex', ++ 'thread', ++ 'chrono', ++ 'ratio', ++ 'regex', ++ 'system_error', ++ ): ++ error(filename, linenum, 'build/c++11', 5, ++ ('<%s> is an unapproved C++11 header.') % include.group(1)) ++ ++ # The only place where we need to worry about C++11 keywords and library ++ # features in preprocessor directives is in macro definitions. ++ if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return ++ ++ # These are classes and free functions. The classes are always ++ # mentioned as std::*, but we only catch the free functions if ++ # they're not found by ADL. They're alphabetical by header. ++ for top_name in ( ++ # type_traits ++ 'alignment_of', ++ 'aligned_union', ++ ++ # utility ++ 'forward', ++ ): ++ if Search(r'\bstd::%s\b' % top_name, line): ++ error(filename, linenum, 'build/c++11', 5, ++ ('std::%s is an unapproved C++11 class or function. Send c-style ' ++ 'an example of where it would make your code more readable, and ' ++ 'they may let you use it.') % top_name) ++ ++ ++def ProcessFileData(filename, file_extension, lines, error, ++ extra_check_functions=[]): ++ """Performs lint checks and reports any errors to the given error function. ++ ++ Args: ++ filename: Filename of the file that is being processed. ++ file_extension: The extension (dot not included) of the file. ++ lines: An array of strings, each representing a line of the file, with the ++ last element being empty if the file is terminated with a newline. ++ error: A callable to which errors are reported, which takes 4 arguments: ++ filename, line number, error level, and message ++ extra_check_functions: An array of additional check functions that will be ++ run on each source line. Each function takes 4 ++ arguments: filename, clean_lines, line, error ++ """ ++ lines = (['// marker so line numbers and indices both start at 1'] + lines + ++ ['// marker so line numbers end in a known way']) ++ ++ include_state = _IncludeState() ++ function_state = _FunctionState() ++ nesting_state = NestingState() ++ ++ ResetNolintSuppressions() ++ ++ CheckForCopyright(filename, lines, error) ++ ++ if file_extension == 'h': ++ CheckForHeaderGuard(filename, lines, error) ++ ++ RemoveMultiLineComments(filename, lines, error) ++ clean_lines = CleansedLines(lines) ++ for line in xrange(clean_lines.NumLines()): ++ ProcessLine(filename, file_extension, clean_lines, line, ++ include_state, function_state, nesting_state, error, ++ extra_check_functions) ++ FlagCxx11Features(filename, clean_lines, line, error) ++ nesting_state.CheckCompletedBlocks(filename, error) ++ ++ CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) ++ ++ # We check here rather than inside ProcessLine so that we see raw ++ # lines rather than "cleaned" lines. ++ CheckForBadCharacters(filename, lines, error) ++ ++ CheckForNewlineAtEOF(filename, lines, error) ++ ++def ProcessConfigOverrides(filename): ++ """ Loads the configuration files and processes the config overrides. ++ ++ Args: ++ filename: The name of the file being processed by the linter. ++ ++ Returns: ++ False if the current |filename| should not be processed further. ++ """ ++ ++ abs_filename = os.path.abspath(filename) ++ cfg_filters = [] ++ keep_looking = True ++ while keep_looking: ++ abs_path, base_name = os.path.split(abs_filename) ++ if not base_name: ++ break # Reached the root directory. ++ ++ cfg_file = os.path.join(abs_path, "CPPLINT.cfg") ++ abs_filename = abs_path ++ if not os.path.isfile(cfg_file): ++ continue ++ ++ try: ++ with open(cfg_file) as file_handle: ++ for line in file_handle: ++ line, _, _ = line.partition('#') # Remove comments. ++ if not line.strip(): ++ continue ++ ++ name, _, val = line.partition('=') ++ name = name.strip() ++ val = val.strip() ++ if name == 'set noparent': ++ keep_looking = False ++ elif name == 'filter': ++ cfg_filters.append(val) ++ elif name == 'exclude_files': ++ # When matching exclude_files pattern, use the base_name of ++ # the current file name or the directory name we are processing. ++ # For example, if we are checking for lint errors in /foo/bar/baz.cc ++ # and we found the .cfg file at /foo/CPPLINT.cfg, then the config ++ # file's "exclude_files" filter is meant to be checked against "bar" ++ # and not "baz" nor "bar/baz.cc". ++ if base_name: ++ pattern = re.compile(val) ++ if pattern.match(base_name): ++ sys.stderr.write('Ignoring "%s": file excluded by "%s". ' ++ 'File path component "%s" matches ' ++ 'pattern "%s"\n' % ++ (filename, cfg_file, base_name, val)) ++ return False ++ elif name == 'linelength': ++ global _line_length ++ try: ++ _line_length = int(val) ++ except ValueError: ++ sys.stderr.write('Line length must be numeric.') ++ else: ++ sys.stderr.write( ++ 'Invalid configuration option (%s) in file %s\n' % ++ (name, cfg_file)) ++ ++ except IOError: ++ sys.stderr.write( ++ "Skipping config file '%s': Can't open for reading\n" % cfg_file) ++ keep_looking = False ++ ++ # Apply all the accumulated filters in reverse order (top-level directory ++ # config options having the least priority). ++ for filter in reversed(cfg_filters): ++ _AddFilters(filter) ++ ++ return True ++ ++ ++def ProcessFile(filename, vlevel, extra_check_functions=[]): ++ """Does google-lint on a single file. ++ ++ Args: ++ filename: The name of the file to parse. ++ ++ vlevel: The level of errors to report. Every error of confidence ++ >= verbose_level will be reported. 0 is a good default. ++ ++ extra_check_functions: An array of additional check functions that will be ++ run on each source line. Each function takes 4 ++ arguments: filename, clean_lines, line, error ++ """ ++ ++ _SetVerboseLevel(vlevel) ++ _BackupFilters() ++ ++ if not ProcessConfigOverrides(filename): ++ _RestoreFilters() ++ return ++ ++ lf_lines = [] ++ crlf_lines = [] ++ try: ++ # Support the UNIX convention of using "-" for stdin. Note that ++ # we are not opening the file with universal newline support ++ # (which codecs doesn't support anyway), so the resulting lines do ++ # contain trailing '\r' characters if we are reading a file that ++ # has CRLF endings. ++ # If after the split a trailing '\r' is present, it is removed ++ # below. ++ if filename == '-': ++ lines = codecs.StreamReaderWriter(sys.stdin, ++ codecs.getreader('utf8'), ++ codecs.getwriter('utf8'), ++ 'replace').read().split('\n') ++ else: ++ lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') ++ ++ # Remove trailing '\r'. ++ # The -1 accounts for the extra trailing blank line we get from split() ++ for linenum in range(len(lines) - 1): ++ if lines[linenum].endswith('\r'): ++ lines[linenum] = lines[linenum].rstrip('\r') ++ crlf_lines.append(linenum + 1) ++ else: ++ lf_lines.append(linenum + 1) ++ ++ except IOError: ++ sys.stderr.write( ++ "Skipping input '%s': Can't open for reading\n" % filename) ++ _RestoreFilters() ++ return ++ ++ # Note, if no dot is found, this will give the entire filename as the ext. ++ file_extension = filename[filename.rfind('.') + 1:] ++ ++ # When reading from stdin, the extension is unknown, so no cpplint tests ++ # should rely on the extension. ++ if filename != '-' and file_extension not in _valid_extensions: ++ sys.stderr.write('Ignoring %s; not a valid file name ' ++ '(%s)\n' % (filename, ', '.join(_valid_extensions))) ++ else: ++ ProcessFileData(filename, file_extension, lines, Error, ++ extra_check_functions) ++ ++ # If end-of-line sequences are a mix of LF and CR-LF, issue ++ # warnings on the lines with CR. ++ # ++ # Don't issue any warnings if all lines are uniformly LF or CR-LF, ++ # since critique can handle these just fine, and the style guide ++ # doesn't dictate a particular end of line sequence. ++ # ++ # We can't depend on os.linesep to determine what the desired ++ # end-of-line sequence should be, since that will return the ++ # server-side end-of-line sequence. ++ if lf_lines and crlf_lines: ++ # Warn on every line with CR. An alternative approach might be to ++ # check whether the file is mostly CRLF or just LF, and warn on the ++ # minority, we bias toward LF here since most tools prefer LF. ++ for linenum in crlf_lines: ++ Error(filename, linenum, 'whitespace/newline', 1, ++ 'Unexpected \\r (^M) found; better to use only \\n') ++ ++ #sys.stderr.write('Done processing %s\n' % filename) ++ _RestoreFilters() ++ ++ ++def PrintUsage(message): ++ """Prints a brief usage string and exits, optionally with an error message. ++ ++ Args: ++ message: The optional error message. ++ """ ++ sys.stderr.write(_USAGE) ++ if message: ++ sys.exit('\nFATAL ERROR: ' + message) ++ else: ++ sys.exit(1) ++ ++ ++def PrintCategories(): ++ """Prints a list of all the error-categories used by error messages. ++ ++ These are the categories used to filter messages via --filter. ++ """ ++ sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) ++ sys.exit(0) ++ ++ ++def ParseArguments(args): ++ """Parses the command line arguments. ++ ++ This may set the output format and verbosity level as side-effects. ++ ++ Args: ++ args: The command line arguments: ++ ++ Returns: ++ The list of filenames to lint. ++ """ ++ try: ++ (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', ++ 'counting=', ++ 'filter=', ++ 'root=', ++ 'linelength=', ++ 'extensions=']) ++ except getopt.GetoptError: ++ PrintUsage('Invalid arguments.') ++ ++ verbosity = _VerboseLevel() ++ output_format = _OutputFormat() ++ filters = '' ++ counting_style = '' ++ ++ for (opt, val) in opts: ++ if opt == '--help': ++ PrintUsage(None) ++ elif opt == '--output': ++ if val not in ('emacs', 'vs7', 'eclipse'): ++ PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') ++ output_format = val ++ elif opt == '--verbose': ++ verbosity = int(val) ++ elif opt == '--filter': ++ filters = val ++ if not filters: ++ PrintCategories() ++ elif opt == '--counting': ++ if val not in ('total', 'toplevel', 'detailed'): ++ PrintUsage('Valid counting options are total, toplevel, and detailed') ++ counting_style = val ++ elif opt == '--root': ++ global _root ++ _root = val ++ elif opt == '--linelength': ++ global _line_length ++ try: ++ _line_length = int(val) ++ except ValueError: ++ PrintUsage('Line length must be digits.') ++ elif opt == '--extensions': ++ global _valid_extensions ++ try: ++ _valid_extensions = set(val.split(',')) ++ except ValueError: ++ PrintUsage('Extensions must be comma seperated list.') ++ ++ if not filenames: ++ PrintUsage('No files were specified.') ++ ++ _SetOutputFormat(output_format) ++ _SetVerboseLevel(verbosity) ++ _SetFilters(filters) ++ _SetCountingStyle(counting_style) ++ ++ return filenames ++ ++ ++def main(): ++ filenames = ParseArguments(sys.argv[1:]) ++ ++ # Change stderr to write with replacement characters so we don't die ++ # if we try to print something containing non-ASCII characters. ++ sys.stderr = codecs.StreamReaderWriter(sys.stderr, ++ codecs.getreader('utf8'), ++ codecs.getwriter('utf8'), ++ 'replace') ++ ++ _cpplint_state.ResetErrorCounts() ++ for filename in filenames: ++ ProcessFile(filename, _cpplint_state.verbose_level) ++ _cpplint_state.PrintErrorCounts() ++ ++ sys.exit(_cpplint_state.error_count > 0) ++ ++ ++if __name__ == '__main__': ++ main() +diff -upNr mongo-r4.0.23.orig/buildscripts/errorcodes.py mongo-r4.0.23/buildscripts/errorcodes.py +--- mongo-r4.0.23.orig/buildscripts/errorcodes.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/errorcodes.py 2021-03-17 01:21:05.952000000 +0000 +@@ -7,12 +7,15 @@ Optionally replaces zero codes in source + + from __future__ import absolute_import + from __future__ import print_function ++from __future__ import unicode_literals + ++import io + import bisect + import os.path + import sys + from collections import defaultdict, namedtuple + from optparse import OptionParser ++from functools import reduce + + # Get relative imports to work when the package is not installed on the PYTHONPATH. + if __name__ == "__main__" and __package__ is None: +@@ -56,7 +59,7 @@ def parse_source_files(callback): + if list_files: + print('scanning file: ' + source_file) + +- with open(source_file) as fh: ++ with open(source_file, encoding="utf-8") as fh: + text = fh.read() + + if not any([zz in text for zz in quick]): +diff -upNr mongo-r4.0.23.orig/buildscripts/eslint.py mongo-r4.0.23/buildscripts/eslint.py +--- mongo-r4.0.23.orig/buildscripts/eslint.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/eslint.py 2021-03-17 01:21:05.952000000 +0000 +@@ -20,7 +20,7 @@ import sys + import tarfile + import tempfile + import threading +-import urllib ++import urllib.request, urllib.parse, urllib.error + from distutils import spawn # pylint: disable=no-name-in-module + from optparse import OptionParser + +@@ -84,7 +84,7 @@ def get_eslint_from_cache(dest_file, pla + + # Download the file + print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) +- urllib.urlretrieve(url, temp_tar_file) ++ urllib.request.urlretrieve(url, temp_tar_file) + + eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) + extract_eslint(temp_tar_file, eslint_distfile) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/binder.py mongo-r4.0.23/buildscripts/idl/idl/binder.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/binder.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/binder.py 2021-03-17 01:21:05.952000000 +0000 +@@ -745,7 +745,7 @@ def _validate_enum_int(ctxt, idl_enum): + min_value = min(int_values_set) + max_value = max(int_values_set) + +- valid_int = {x for x in xrange(min_value, max_value + 1)} ++ valid_int = {x for x in range(min_value, max_value + 1)} + + if valid_int != int_values_set: + ctxt.add_enum_non_continuous_range_error(idl_enum, idl_enum.name) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/binder.py.orig mongo-r4.0.23/buildscripts/idl/idl/binder.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/binder.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/binder.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,822 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++# pylint: disable=too-many-lines ++"""Transform idl.syntax trees from the parser into well-defined idl.ast trees.""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++import re ++from typing import cast, List, Set, Union ++ ++from . import ast ++from . import bson ++from . import common ++from . import cpp_types ++from . import enum_types ++from . import errors ++from . import syntax ++ ++ ++def _validate_single_bson_type(ctxt, idl_type, syntax_type): ++ # type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> bool ++ """Validate bson serialization type is correct for a type.""" ++ bson_type = idl_type.bson_serialization_type[0] ++ ++ # Any and Chain are only valid if they are the only bson types specified ++ if bson_type in ["any", "chain"]: ++ return True ++ ++ if not bson.is_valid_bson_type(bson_type): ++ ctxt.add_bad_bson_type_error(idl_type, syntax_type, idl_type.name, bson_type) ++ return False ++ ++ # Validate bindata_subytpe ++ if bson_type == "bindata": ++ subtype = idl_type.bindata_subtype ++ ++ if subtype is None: ++ subtype = "" ++ ++ if not bson.is_valid_bindata_subtype(subtype): ++ ctxt.add_bad_bson_bindata_subtype_value_error(idl_type, syntax_type, idl_type.name, ++ subtype) ++ elif idl_type.bindata_subtype is not None: ++ ctxt.add_bad_bson_bindata_subtype_error(idl_type, syntax_type, idl_type.name, bson_type) ++ ++ return True ++ ++ ++def _validate_bson_types_list(ctxt, idl_type, syntax_type): ++ # type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> bool ++ """Validate bson serialization type(s) is correct for a type.""" ++ ++ bson_types = idl_type.bson_serialization_type ++ if len(bson_types) == 1: ++ return _validate_single_bson_type(ctxt, idl_type, syntax_type) ++ ++ for bson_type in bson_types: ++ if bson_type in ["any", "chain"]: ++ ctxt.add_bad_any_type_use_error(idl_type, bson_type, syntax_type, idl_type.name) ++ return False ++ ++ if not bson.is_valid_bson_type(bson_type): ++ ctxt.add_bad_bson_type_error(idl_type, syntax_type, idl_type.name, bson_type) ++ return False ++ ++ # V1 restiction: cannot mix bindata into list of types ++ if bson_type == "bindata": ++ ctxt.add_bad_bson_type_error(idl_type, syntax_type, idl_type.name, bson_type) ++ return False ++ ++ # Cannot mix non-scalar types into the list of types ++ if not bson.is_scalar_bson_type(bson_type): ++ ctxt.add_bad_bson_scalar_type_error(idl_type, syntax_type, idl_type.name, bson_type) ++ return False ++ ++ return True ++ ++ ++def _validate_type(ctxt, idl_type): ++ # type: (errors.ParserContext, syntax.Type) -> None ++ """Validate each type is correct.""" ++ ++ # Validate naming restrictions ++ if idl_type.name.startswith("array<"): ++ ctxt.add_array_not_valid_error(idl_type, "type", idl_type.name) ++ ++ _validate_type_properties(ctxt, idl_type, 'type') ++ ++ ++def _validate_cpp_type(ctxt, idl_type, syntax_type): ++ # type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None ++ """Validate the cpp_type is correct.""" ++ ++ # Validate cpp_type ++ # Do not allow StringData, use std::string instead. ++ if "StringData" in idl_type.cpp_type: ++ ctxt.add_no_string_data_error(idl_type, syntax_type, idl_type.name) ++ ++ # We do not support C++ char and float types for style reasons ++ if idl_type.cpp_type in ['char', 'wchar_t', 'char16_t', 'char32_t', 'float']: ++ ctxt.add_bad_cpp_numeric_type_use_error(idl_type, syntax_type, idl_type.name, ++ idl_type.cpp_type) ++ ++ # We do not support C++ builtin integer for style reasons ++ for numeric_word in ['signed', "unsigned", "int", "long", "short"]: ++ if re.search(r'\b%s\b' % (numeric_word), idl_type.cpp_type): ++ ctxt.add_bad_cpp_numeric_type_use_error(idl_type, syntax_type, idl_type.name, ++ idl_type.cpp_type) ++ # Return early so we only throw one error for types like "signed short int" ++ return ++ ++ # Check for std fixed integer types which are allowed ++ if idl_type.cpp_type in ["std::int32_t", "std::int64_t", "std::uint32_t", "std::uint64_t"]: ++ return ++ ++ # Only allow 16-byte arrays since they are for MD5 and UUID ++ if idl_type.cpp_type.replace(" ", "") == "std::array": ++ return ++ ++ # Support vector for variable length BinData. ++ if idl_type.cpp_type == "std::vector": ++ return ++ ++ # Check for std fixed integer types which are not allowed. These are not allowed even if they ++ # have the "std::" prefix. ++ for std_numeric_type in [ ++ "int8_t", "int16_t", "int32_t", "int64_t", "uint8_t", "uint16_t", "uint32_t", "uint64_t" ++ ]: ++ if std_numeric_type in idl_type.cpp_type: ++ ctxt.add_bad_cpp_numeric_type_use_error(idl_type, syntax_type, idl_type.name, ++ idl_type.cpp_type) ++ return ++ ++ ++def _validate_chain_type_properties(ctxt, idl_type, syntax_type): ++ # type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None ++ """Validate a chained type has both a deserializer and serializer.""" ++ assert len( ++ idl_type.bson_serialization_type) == 1 and idl_type.bson_serialization_type[0] == 'chain' ++ ++ if idl_type.deserializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "deserializer") ++ ++ if idl_type.serializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "serializer") ++ ++ ++def _validate_type_properties(ctxt, idl_type, syntax_type): ++ # type: (errors.ParserContext, Union[syntax.Type, ast.Field], unicode) -> None ++ # pylint: disable=too-many-branches ++ """Validate each type or field is correct.""" ++ ++ # Validate bson type restrictions ++ if not _validate_bson_types_list(ctxt, idl_type, syntax_type): ++ return ++ ++ if len(idl_type.bson_serialization_type) == 1: ++ bson_type = idl_type.bson_serialization_type[0] ++ ++ if bson_type == "any": ++ # For 'any', a deserializer is required but the user can try to get away with the default ++ # serialization for their C++ type. ++ if idl_type.deserializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "deserializer") ++ elif bson_type == "chain": ++ _validate_chain_type_properties(ctxt, idl_type, syntax_type) ++ ++ elif bson_type == "string": ++ # Strings support custom serialization unlike other non-object scalar types ++ if idl_type.deserializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "deserializer") ++ ++ elif not bson_type in ["object", "bindata"]: ++ if idl_type.deserializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "deserializer") ++ ++ if idl_type.deserializer is not None and "BSONElement" not in idl_type.deserializer: ++ ctxt.add_not_custom_scalar_serialization_not_supported_error( ++ idl_type, syntax_type, idl_type.name, bson_type) ++ ++ if idl_type.serializer is not None: ++ ctxt.add_not_custom_scalar_serialization_not_supported_error( ++ idl_type, syntax_type, idl_type.name, bson_type) ++ ++ if bson_type == "bindata" and idl_type.default: ++ ctxt.add_bindata_no_default(idl_type, syntax_type, idl_type.name) ++ ++ else: ++ # Now, this is a list of scalar types ++ if idl_type.deserializer is None: ++ ctxt.add_missing_ast_required_field_error(idl_type, syntax_type, idl_type.name, ++ "deserializer") ++ ++ _validate_cpp_type(ctxt, idl_type, syntax_type) ++ ++ ++def _validate_types(ctxt, parsed_spec): ++ # type: (errors.ParserContext, syntax.IDLSpec) -> None ++ """Validate all types are correct.""" ++ ++ for idl_type in parsed_spec.symbols.types: ++ _validate_type(ctxt, idl_type) ++ ++ ++def _is_duplicate_field(ctxt, field_container, fields, ast_field): ++ # type: (errors.ParserContext, unicode, List[ast.Field], ast.Field) -> bool ++ """Return True if there is a naming conflict for a given field.""" ++ ++ # This is normally tested in the parser as part of duplicate detection in a map ++ if ast_field.name in [field.name for field in fields]: ++ for field in fields: ++ if field.name == ast_field.name: ++ duplicate_field = field ++ ++ ctxt.add_duplicate_field_error(ast_field, field_container, ast_field.name, duplicate_field) ++ return True ++ ++ return False ++ ++ ++def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): ++ # type: (errors.ParserContext, syntax.IDLSpec, syntax.Struct, ast.Struct) -> None ++ # pylint: disable=too-many-branches ++ ++ ast_struct.name = struct.name ++ ast_struct.description = struct.description ++ ast_struct.strict = struct.strict ++ ast_struct.immutable = struct.immutable ++ ast_struct.inline_chained_structs = struct.inline_chained_structs ++ ast_struct.generate_comparison_operators = struct.generate_comparison_operators ++ ast_struct.cpp_name = struct.name ++ if struct.cpp_name: ++ ast_struct.cpp_name = struct.cpp_name ++ ++ # Validate naming restrictions ++ if ast_struct.name.startswith("array<"): ++ ctxt.add_array_not_valid_error(ast_struct, "struct", ast_struct.name) ++ ++ # Merge chained types as chained fields ++ if struct.chained_types: ++ if ast_struct.strict: ++ ctxt.add_chained_type_no_strict_error(ast_struct, ast_struct.name) ++ ++ for chained_type in struct.chained_types: ++ ast_field = _bind_chained_type(ctxt, parsed_spec, ast_struct, chained_type) ++ if ast_field and not _is_duplicate_field(ctxt, chained_type.name, ast_struct.fields, ++ ast_field): ++ ast_struct.fields.append(ast_field) ++ ++ # Merge chained structs as a chained struct and ignored fields ++ for chained_struct in struct.chained_structs or []: ++ _bind_chained_struct(ctxt, parsed_spec, ast_struct, chained_struct) ++ ++ # Parse the fields last so that they are serialized after chained stuff. ++ for field in struct.fields or []: ++ ast_field = _bind_field(ctxt, parsed_spec, field) ++ if ast_field: ++ if ast_field.supports_doc_sequence and not isinstance(ast_struct, ast.Command): ++ # Doc sequences are only supported in commands at the moment ++ ctxt.add_bad_struct_field_as_doc_sequence_error(ast_struct, ast_struct.name, ++ ast_field.name) ++ ++ if ast_field.non_const_getter and struct.immutable: ++ ctxt.add_bad_field_non_const_getter_in_immutable_struct_error( ++ ast_struct, ast_struct.name, ast_field.name) ++ ++ if not _is_duplicate_field(ctxt, ast_struct.name, ast_struct.fields, ast_field): ++ ast_struct.fields.append(ast_field) ++ ++ # Fill out the field comparison_order property as needed ++ if ast_struct.generate_comparison_operators and ast_struct.fields: ++ # If the user did not specify an ordering of fields, then number all fields in ++ # declared field. ++ use_default_order = True ++ comparison_orders = set() # type: Set[int] ++ ++ for ast_field in ast_struct.fields: ++ if not ast_field.comparison_order == -1: ++ use_default_order = False ++ if ast_field.comparison_order in comparison_orders: ++ ctxt.add_duplicate_comparison_order_field_error(ast_struct, ast_struct.name, ++ ast_field.comparison_order) ++ ++ comparison_orders.add(ast_field.comparison_order) ++ ++ if use_default_order: ++ pos = 0 ++ for ast_field in ast_struct.fields: ++ ast_field.comparison_order = pos ++ pos += 1 ++ ++ ++def _bind_struct(ctxt, parsed_spec, struct): ++ # type: (errors.ParserContext, syntax.IDLSpec, syntax.Struct) -> ast.Struct ++ """ ++ Bind a struct. ++ ++ - Validating a struct and fields. ++ - Create the idl.ast version from the idl.syntax tree. ++ """ ++ ++ ast_struct = ast.Struct(struct.file_name, struct.line, struct.column) ++ ++ _bind_struct_common(ctxt, parsed_spec, struct, ast_struct) ++ ++ return ast_struct ++ ++ ++def _inject_hidden_command_fields(command): ++ # type: (syntax.Command) -> None ++ """Inject hidden fields to aid deserialization/serialization for OpMsg parsing of commands.""" ++ ++ # Inject a "$db" which we can decode during command parsing ++ db_field = syntax.Field(command.file_name, command.line, command.column) ++ db_field.name = "$db" ++ db_field.type = "string" # This comes from basic_types.idl ++ db_field.cpp_name = "dbName" ++ db_field.serialize_op_msg_request_only = True ++ ++ # Commands that require namespaces do not need to have db defaulted in the constructor ++ if command.namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: ++ db_field.constructed = True ++ ++ command.fields.append(db_field) ++ ++ ++def _bind_command_type(ctxt, parsed_spec, command): ++ # type: (errors.ParserContext, syntax.IDLSpec, syntax.Command) -> ast.Field ++ """Bind the type field in a command as the first field.""" ++ # pylint: disable=too-many-branches,too-many-statements ++ ast_field = ast.Field(command.file_name, command.line, command.column) ++ ast_field.name = command.name ++ ast_field.description = command.description ++ ast_field.optional = False ++ ast_field.supports_doc_sequence = False ++ ast_field.serialize_op_msg_request_only = False ++ ast_field.constructed = False ++ ++ ast_field.cpp_name = "CommandParameter" ++ ++ # Validate naming restrictions ++ if ast_field.name.startswith("array<"): ++ ctxt.add_array_not_valid_error(ast_field, "field", ast_field.name) ++ ++ # Resolve the command type as a field ++ syntax_symbol = parsed_spec.symbols.resolve_field_type(ctxt, command, command.name, ++ command.type) ++ if syntax_symbol is None: ++ return None ++ ++ if isinstance(syntax_symbol, syntax.Command): ++ ctxt.add_bad_command_as_field_error(ast_field, command.type) ++ return None ++ ++ assert not isinstance(syntax_symbol, syntax.Enum) ++ ++ # If the field type is an array, mark the AST version as such. ++ if syntax.parse_array_type(command.type): ++ ast_field.array = True ++ ++ # Copy over only the needed information if this a struct or a type ++ if isinstance(syntax_symbol, syntax.Struct): ++ struct = cast(syntax.Struct, syntax_symbol) ++ cpp_name = struct.name ++ if struct.cpp_name: ++ cpp_name = struct.cpp_name ++ ast_field.struct_type = common.qualify_cpp_name(struct.cpp_namespace, cpp_name) ++ ast_field.bson_serialization_type = ["object"] ++ ++ _validate_field_of_type_struct(ctxt, ast_field) ++ else: ++ # Produce the union of type information for the type and this field. ++ idltype = cast(syntax.Type, syntax_symbol) ++ ++ # Copy over the type fields first ++ ast_field.cpp_type = idltype.cpp_type ++ ast_field.bson_serialization_type = idltype.bson_serialization_type ++ ast_field.bindata_subtype = idltype.bindata_subtype ++ ast_field.serializer = _normalize_method_name(idltype.cpp_type, idltype.serializer) ++ ast_field.deserializer = _normalize_method_name(idltype.cpp_type, idltype.deserializer) ++ ast_field.default = idltype.default ++ ++ # Validate merged type ++ _validate_type_properties(ctxt, ast_field, "command.type") ++ ++ # Validate merged type ++ _validate_field_properties(ctxt, ast_field) ++ ++ return ast_field ++ ++ ++def _bind_command(ctxt, parsed_spec, command): ++ # type: (errors.ParserContext, syntax.IDLSpec, syntax.Command) -> ast.Command ++ """ ++ Bind a command. ++ ++ - Validating a command and fields. ++ - Create the idl.ast version from the idl.syntax tree. ++ """ ++ ++ ast_command = ast.Command(command.file_name, command.line, command.column) ++ ++ # Inject special fields used for command parsing ++ _inject_hidden_command_fields(command) ++ ++ _bind_struct_common(ctxt, parsed_spec, command, ast_command) ++ ++ ast_command.namespace = command.namespace ++ ++ if command.type: ++ ast_command.command_field = _bind_command_type(ctxt, parsed_spec, command) ++ ++ if [field for field in ast_command.fields if field.name == ast_command.name]: ++ ctxt.add_bad_command_name_duplicates_field(ast_command, ast_command.name) ++ ++ return ast_command ++ ++ ++def _validate_ignored_field(ctxt, field): ++ # type: (errors.ParserContext, syntax.Field) -> None ++ """Validate that for ignored fields, no other properties are set.""" ++ if field.optional: ++ ctxt.add_ignored_field_must_be_empty_error(field, field.name, "optional") ++ if field.default is not None: ++ ctxt.add_ignored_field_must_be_empty_error(field, field.name, "default") ++ ++ ++def _validate_field_of_type_struct(ctxt, field): ++ # type: (errors.ParserContext, Union[syntax.Field, ast.Field]) -> None ++ """Validate that for fields with a type of struct, no other properties are set.""" ++ if field.default is not None: ++ ctxt.add_struct_field_must_be_empty_error(field, field.name, "default") ++ ++ ++def _validate_array_type(ctxt, syntax_symbol, field): ++ # type: (errors.ParserContext, Union[syntax.Command, syntax.Enum, syntax.Struct, syntax.Type], syntax.Field) -> None ++ """Validate this an array of plain objects or a struct.""" ++ if isinstance(syntax_symbol, syntax.Enum): ++ ctxt.add_array_enum_error(field, field.name) ++ ++ if field.default or (isinstance(syntax_symbol, syntax.Type) and syntax_symbol.default): ++ ctxt.add_array_no_default_error(field, field.name) ++ ++ ++def _validate_field_properties(ctxt, ast_field): ++ # type: (errors.ParserContext, ast.Field) -> None ++ """Validate field specific rules.""" ++ ++ if ast_field.default and ast_field.optional: ++ ctxt.add_bad_field_default_and_optional(ast_field, ast_field.name) ++ ++ # A "chain" type should never appear as a field. ++ if ast_field.bson_serialization_type == ['chain']: ++ ctxt.add_bad_array_of_chain(ast_field, ast_field.name) ++ ++ ++def _validate_doc_sequence_field(ctxt, ast_field): ++ # type: (errors.ParserContext, ast.Field) -> None ++ """Validate the doc_sequence is an array of plain objects.""" ++ if not ast_field.supports_doc_sequence: ++ return ++ ++ assert ast_field.array ++ ++ # The only allowed BSON type for a doc_sequence field is "object" ++ if ast_field.bson_serialization_type != ['object']: ++ ctxt.add_bad_non_object_as_doc_sequence_error(ast_field, ast_field.name) ++ ++ ++def _normalize_method_name(cpp_type_name, cpp_method_name): ++ # type: (unicode, unicode) -> unicode ++ """Normalize the method name to be fully-qualified with the type name.""" ++ # Default deserializer ++ if not cpp_method_name: ++ return cpp_method_name ++ ++ # Global function ++ if cpp_method_name.startswith('::'): ++ return cpp_method_name ++ ++ # Method is full qualified already ++ if cpp_method_name.startswith(cpp_type_name): ++ return cpp_method_name ++ ++ # Get the unqualified type name ++ type_name = cpp_type_name.split("::")[-1] ++ ++ # Method is prefixed with just the type name ++ if cpp_method_name.startswith(type_name): ++ return '::'.join(cpp_type_name.split('::')[0:-1]) + "::" + cpp_method_name ++ ++ return cpp_method_name ++ ++ ++def _bind_field(ctxt, parsed_spec, field): ++ # type: (errors.ParserContext, syntax.IDLSpec, syntax.Field) -> ast.Field ++ """ ++ Bind a field from the idl.syntax tree. ++ ++ - Create the idl.ast version from the idl.syntax tree. ++ - Validate the resulting type is correct. ++ """ ++ # pylint: disable=too-many-branches,too-many-statements ++ ast_field = ast.Field(field.file_name, field.line, field.column) ++ ast_field.name = field.name ++ ast_field.description = field.description ++ ast_field.optional = field.optional ++ ast_field.supports_doc_sequence = field.supports_doc_sequence ++ ast_field.serialize_op_msg_request_only = field.serialize_op_msg_request_only ++ ast_field.constructed = field.constructed ++ ast_field.comparison_order = field.comparison_order ++ ast_field.non_const_getter = field.non_const_getter ++ ++ ast_field.cpp_name = field.name ++ if field.cpp_name: ++ ast_field.cpp_name = field.cpp_name ++ ++ # Validate naming restrictions ++ if ast_field.name.startswith("array<"): ++ ctxt.add_array_not_valid_error(ast_field, "field", ast_field.name) ++ ++ if field.ignore: ++ ast_field.ignore = field.ignore ++ _validate_ignored_field(ctxt, field) ++ return ast_field ++ ++ syntax_symbol = parsed_spec.symbols.resolve_field_type(ctxt, field, field.name, field.type) ++ if syntax_symbol is None: ++ return None ++ ++ if isinstance(syntax_symbol, syntax.Command): ++ ctxt.add_bad_command_as_field_error(ast_field, field.type) ++ return None ++ ++ # If the field type is an array, mark the AST version as such. ++ if syntax.parse_array_type(field.type): ++ ast_field.array = True ++ ++ _validate_array_type(ctxt, syntax_symbol, field) ++ elif field.supports_doc_sequence: ++ # Doc sequences are only supported for arrays ++ ctxt.add_bad_non_array_as_doc_sequence_error(syntax_symbol, syntax_symbol.name, ++ ast_field.name) ++ return None ++ ++ # Copy over only the needed information if this a struct or a type ++ if isinstance(syntax_symbol, syntax.Struct): ++ struct = cast(syntax.Struct, syntax_symbol) ++ cpp_name = struct.name ++ if struct.cpp_name: ++ cpp_name = struct.cpp_name ++ ast_field.struct_type = common.qualify_cpp_name(struct.cpp_namespace, cpp_name) ++ ast_field.bson_serialization_type = ["object"] ++ ++ _validate_field_of_type_struct(ctxt, field) ++ elif isinstance(syntax_symbol, syntax.Enum): ++ enum_type_info = enum_types.get_type_info(cast(syntax.Enum, syntax_symbol)) ++ ++ ast_field.enum_type = True ++ ast_field.default = field.default ++ ast_field.cpp_type = enum_type_info.get_qualified_cpp_type_name() ++ ast_field.bson_serialization_type = enum_type_info.get_bson_types() ++ ast_field.serializer = enum_type_info.get_enum_serializer_name() ++ ast_field.deserializer = enum_type_info.get_enum_deserializer_name() ++ else: ++ # Produce the union of type information for the type and this field. ++ idltype = cast(syntax.Type, syntax_symbol) ++ ++ # Copy over the type fields first ++ ast_field.cpp_type = idltype.cpp_type ++ ast_field.bson_serialization_type = idltype.bson_serialization_type ++ ast_field.bindata_subtype = idltype.bindata_subtype ++ ast_field.serializer = _normalize_method_name(idltype.cpp_type, idltype.serializer) ++ ast_field.deserializer = _normalize_method_name(idltype.cpp_type, idltype.deserializer) ++ ast_field.default = idltype.default ++ ++ if field.default: ++ ast_field.default = field.default ++ ++ # Validate merged type ++ _validate_type_properties(ctxt, ast_field, "field") ++ ++ # Validate merged type ++ _validate_field_properties(ctxt, ast_field) ++ ++ # Validation doc_sequence types ++ _validate_doc_sequence_field(ctxt, ast_field) ++ ++ return ast_field ++ ++ ++def _bind_chained_type(ctxt, parsed_spec, location, chained_type): ++ # type: (errors.ParserContext, syntax.IDLSpec, common.SourceLocation, syntax.ChainedType) -> ast.Field ++ """Bind the specified chained type.""" ++ syntax_symbol = parsed_spec.symbols.resolve_field_type(ctxt, location, chained_type.name, ++ chained_type.name) ++ if not syntax_symbol: ++ return None ++ ++ if not isinstance(syntax_symbol, syntax.Type): ++ ctxt.add_chained_type_not_found_error(location, chained_type.name) ++ return None ++ ++ idltype = cast(syntax.Type, syntax_symbol) ++ ++ if len(idltype.bson_serialization_type) != 1 or idltype.bson_serialization_type[0] != 'chain': ++ ctxt.add_chained_type_wrong_type_error(location, chained_type.name, ++ idltype.bson_serialization_type[0]) ++ return None ++ ++ ast_field = ast.Field(location.file_name, location.line, location.column) ++ ast_field.name = idltype.name ++ ast_field.cpp_name = chained_type.cpp_name ++ ast_field.description = idltype.description ++ ast_field.chained = True ++ ++ ast_field.cpp_type = idltype.cpp_type ++ ast_field.bson_serialization_type = idltype.bson_serialization_type ++ ast_field.serializer = idltype.serializer ++ ast_field.deserializer = idltype.deserializer ++ ++ return ast_field ++ ++ ++def _bind_chained_struct(ctxt, parsed_spec, ast_struct, chained_struct): ++ # type: (errors.ParserContext, syntax.IDLSpec, ast.Struct, syntax.ChainedStruct) -> None ++ """Bind the specified chained struct.""" ++ syntax_symbol = parsed_spec.symbols.resolve_field_type(ctxt, ast_struct, chained_struct.name, ++ chained_struct.name) ++ ++ if not syntax_symbol: ++ return ++ ++ if not isinstance(syntax_symbol, syntax.Struct) or isinstance(syntax_symbol, syntax.Command): ++ ctxt.add_chained_struct_not_found_error(ast_struct, chained_struct.name) ++ return ++ ++ struct = cast(syntax.Struct, syntax_symbol) ++ ++ # chained struct cannot be strict unless it is inlined ++ if struct.strict and not ast_struct.inline_chained_structs: ++ ctxt.add_chained_nested_struct_no_strict_error(ast_struct, ast_struct.name, ++ chained_struct.name) ++ ++ if struct.chained_types or struct.chained_structs: ++ ctxt.add_chained_nested_struct_no_nested_error(ast_struct, ast_struct.name, ++ chained_struct.name) ++ ++ # Configure a field for the chained struct. ++ ast_chained_field = ast.Field(ast_struct.file_name, ast_struct.line, ast_struct.column) ++ ast_chained_field.name = struct.name ++ ast_chained_field.cpp_name = chained_struct.cpp_name ++ ast_chained_field.description = struct.description ++ cpp_name = struct.name ++ if struct.cpp_name: ++ cpp_name = struct.cpp_name ++ ast_chained_field.struct_type = cpp_name ++ ast_chained_field.bson_serialization_type = ["object"] ++ ++ ast_chained_field.chained = True ++ ++ if not _is_duplicate_field(ctxt, chained_struct.name, ast_struct.fields, ast_chained_field): ++ ast_struct.fields.append(ast_chained_field) ++ else: ++ return ++ ++ # Merge all the fields from resolved struct into this ast struct. ++ for field in struct.fields or []: ++ ast_field = _bind_field(ctxt, parsed_spec, field) ++ if ast_field and not _is_duplicate_field(ctxt, chained_struct.name, ast_struct.fields, ++ ast_field): ++ ++ if ast_struct.inline_chained_structs: ++ ast_field.chained_struct_field = ast_chained_field ++ else: ++ # For non-inlined structs, mark them as ignore ++ ast_field.ignore = True ++ ++ ast_struct.fields.append(ast_field) ++ ++ ++def _bind_globals(parsed_spec): ++ # type: (syntax.IDLSpec) -> ast.Global ++ """Bind the globals object from the idl.syntax tree into the idl.ast tree by doing a deep copy.""" ++ if parsed_spec.globals: ++ ast_global = ast.Global(parsed_spec.globals.file_name, parsed_spec.globals.line, ++ parsed_spec.globals.column) ++ ast_global.cpp_namespace = parsed_spec.globals.cpp_namespace ++ ast_global.cpp_includes = parsed_spec.globals.cpp_includes ++ else: ++ ast_global = ast.Global("", 0, 0) ++ ++ # If no namespace has been set, default it do "mongo" ++ ast_global.cpp_namespace = "mongo" ++ ++ return ast_global ++ ++ ++def _validate_enum_int(ctxt, idl_enum): ++ # type: (errors.ParserContext, syntax.Enum) -> None ++ """Validate an integer enumeration.""" ++ ++ # Check they are all ints ++ int_values_set = set() # type: Set[int] ++ ++ for enum_value in idl_enum.values: ++ try: ++ int_values_set.add(int(enum_value.value)) ++ except ValueError as value_error: ++ ctxt.add_enum_value_not_int_error(idl_enum, idl_enum.name, enum_value.value, ++ str(value_error)) ++ return ++ ++ # Check the values are continuous so they can be static_cast. ++ min_value = min(int_values_set) ++ max_value = max(int_values_set) ++ ++ valid_int = {x for x in xrange(min_value, max_value + 1)} ++ ++ if valid_int != int_values_set: ++ ctxt.add_enum_non_continuous_range_error(idl_enum, idl_enum.name) ++ ++ ++def _bind_enum(ctxt, idl_enum): ++ # type: (errors.ParserContext, syntax.Enum) -> ast.Enum ++ """ ++ Bind an enum. ++ ++ - Validating an enum and values. ++ - Create the idl.ast version from the idl.syntax tree. ++ """ ++ ++ ast_enum = ast.Enum(idl_enum.file_name, idl_enum.line, idl_enum.column) ++ ast_enum.name = idl_enum.name ++ ast_enum.description = idl_enum.description ++ ast_enum.type = idl_enum.type ++ ast_enum.cpp_namespace = idl_enum.cpp_namespace ++ ++ enum_type_info = enum_types.get_type_info(idl_enum) ++ if not enum_type_info: ++ ctxt.add_enum_bad_type_error(idl_enum, idl_enum.name, idl_enum.type) ++ return None ++ ++ for enum_value in idl_enum.values: ++ ast_enum_value = ast.EnumValue(enum_value.file_name, enum_value.line, enum_value.column) ++ ast_enum_value.name = enum_value.name ++ ast_enum_value.value = enum_value.value ++ ast_enum.values.append(ast_enum_value) ++ ++ values_set = set() # type: Set[unicode] ++ for enum_value in idl_enum.values: ++ values_set.add(enum_value.value) ++ ++ # Check the values are unique ++ if len(idl_enum.values) != len(values_set): ++ ctxt.add_enum_value_not_unique_error(idl_enum, idl_enum.name) ++ ++ if ast_enum.type == 'int': ++ _validate_enum_int(ctxt, idl_enum) ++ ++ return ast_enum ++ ++ ++def bind(parsed_spec): ++ # type: (syntax.IDLSpec) -> ast.IDLBoundSpec ++ """Read an idl.syntax, create an idl.ast tree, and validate the final IDL Specification.""" ++ ++ ctxt = errors.ParserContext("unknown", errors.ParserErrorCollection()) ++ ++ bound_spec = ast.IDLAST() ++ ++ bound_spec.globals = _bind_globals(parsed_spec) ++ ++ _validate_types(ctxt, parsed_spec) ++ ++ # Check enums before structs to ensure they are valid ++ for idl_enum in parsed_spec.symbols.enums: ++ if not idl_enum.imported: ++ bound_spec.enums.append(_bind_enum(ctxt, idl_enum)) ++ ++ for command in parsed_spec.symbols.commands: ++ if not command.imported: ++ bound_spec.commands.append(_bind_command(ctxt, parsed_spec, command)) ++ ++ for struct in parsed_spec.symbols.structs: ++ if not struct.imported: ++ bound_spec.structs.append(_bind_struct(ctxt, parsed_spec, struct)) ++ ++ if ctxt.errors.has_errors(): ++ return ast.IDLBoundSpec(None, ctxt.errors) ++ ++ return ast.IDLBoundSpec(bound_spec, None) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/bson.py mongo-r4.0.23/buildscripts/idl/idl/bson.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/bson.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/bson.py 2021-03-17 01:21:05.952000000 +0000 +@@ -100,7 +100,7 @@ def cpp_bson_type_name(name): + def list_valid_types(): + # type: () -> List[unicode] + """Return a list of supported bson types.""" +- return [a for a in _BSON_TYPE_INFORMATION.iterkeys()] ++ return [a for a in _BSON_TYPE_INFORMATION.keys()] + + + def is_valid_bindata_subtype(name): +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/bson.py.orig mongo-r4.0.23/buildscripts/idl/idl/bson.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/bson.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/bson.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,116 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++""" ++BSON Type Information. ++ ++Utilities for validating bson types, etc. ++""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from typing import Dict, List ++ ++# Dictionary of BSON type Information ++# scalar: True if the type is not an array or object ++# bson_type_enum: The BSONType enum value for the given type ++_BSON_TYPE_INFORMATION = { ++ "double": {'scalar': True, 'bson_type_enum': 'NumberDouble'}, ++ "string": {'scalar': True, 'bson_type_enum': 'String'}, ++ "object": {'scalar': False, 'bson_type_enum': 'Object'}, ++ # TODO: add support: "array" : { 'scalar' : False, 'bson_type_enum' : 'Array'}, ++ "bindata": {'scalar': True, 'bson_type_enum': 'BinData'}, ++ "undefined": {'scalar': True, 'bson_type_enum': 'Undefined'}, ++ "objectid": {'scalar': True, 'bson_type_enum': 'jstOID'}, ++ "bool": {'scalar': True, 'bson_type_enum': 'Bool'}, ++ "date": {'scalar': True, 'bson_type_enum': 'Date'}, ++ "null": {'scalar': True, 'bson_type_enum': 'jstNULL'}, ++ "regex": {'scalar': True, 'bson_type_enum': 'RegEx'}, ++ "int": {'scalar': True, 'bson_type_enum': 'NumberInt'}, ++ "timestamp": {'scalar': True, 'bson_type_enum': 'bsonTimestamp'}, ++ "long": {'scalar': True, 'bson_type_enum': 'NumberLong'}, ++ "decimal": {'scalar': True, 'bson_type_enum': 'NumberDecimal'}, ++} ++ ++# Dictionary of BinData subtype type Information ++# scalar: True if the type is not an array or object ++# bindata_enum: The BinDataType enum value for the given type ++_BINDATA_SUBTYPE = { ++ "generic": {'scalar': True, 'bindata_enum': 'BinDataGeneral'}, ++ "function": {'scalar': True, 'bindata_enum': 'Function'}, ++ # Also simply known as type 2, deprecated, and requires special handling ++ #"binary": { ++ # 'scalar': False, ++ # 'bindata_enum': 'ByteArrayDeprecated' ++ #}, ++ # Deprecated ++ # "uuid_old": { ++ # 'scalar': False, ++ # 'bindata_enum': 'bdtUUID' ++ # }, ++ "uuid": {'scalar': True, 'bindata_enum': 'newUUID'}, ++ "md5": {'scalar': True, 'bindata_enum': 'MD5Type'}, ++} ++ ++ ++def is_valid_bson_type(name): ++ # type: (unicode) -> bool ++ """Return True if this is a valid bson type.""" ++ return name in _BSON_TYPE_INFORMATION ++ ++ ++def is_scalar_bson_type(name): ++ # type: (unicode) -> bool ++ """Return True if this bson type is a scalar.""" ++ assert is_valid_bson_type(name) ++ return _BSON_TYPE_INFORMATION[name]['scalar'] # type: ignore ++ ++ ++def cpp_bson_type_name(name): ++ # type: (unicode) -> unicode ++ """Return the C++ type name for a bson type.""" ++ assert is_valid_bson_type(name) ++ return _BSON_TYPE_INFORMATION[name]['bson_type_enum'] # type: ignore ++ ++ ++def list_valid_types(): ++ # type: () -> List[unicode] ++ """Return a list of supported bson types.""" ++ return [a for a in _BSON_TYPE_INFORMATION.iterkeys()] ++ ++ ++def is_valid_bindata_subtype(name): ++ # type: (unicode) -> bool ++ """Return True if this bindata subtype is valid.""" ++ return name in _BINDATA_SUBTYPE ++ ++ ++def cpp_bindata_subtype_type_name(name): ++ # type: (unicode) -> unicode ++ """Return the C++ type name for a bindata subtype.""" ++ assert is_valid_bindata_subtype(name) ++ return _BINDATA_SUBTYPE[name]['bindata_enum'] # type: ignore +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/cpp_types.py mongo-r4.0.23/buildscripts/idl/idl/cpp_types.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/cpp_types.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/cpp_types.py 2021-03-17 01:21:05.952000000 +0000 +@@ -41,6 +41,7 @@ from . import writer + + _STD_ARRAY_UINT8_16 = 'std::array' + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + + def is_primitive_scalar_type(cpp_type): + # type: (unicode) -> bool +@@ -88,11 +89,9 @@ def _qualify_array_type(cpp_type): + return "std::vector<%s>" % (cpp_type) + + +-class CppTypeBase(object): ++class CppTypeBase(ABC): + """Base type for C++ Type information.""" + +- __metaclass__ = ABCMeta +- + def __init__(self, field): + # type: (ast.Field) -> None + """Construct a CppTypeBase.""" +@@ -534,11 +533,9 @@ def get_cpp_type(field): + return cpp_type_info + + +-class BsonCppTypeBase(object): ++class BsonCppTypeBase(ABC): + """Base type for custom C++ support for BSON Types information.""" + +- __metaclass__ = ABCMeta +- + def __init__(self, field): + # type: (ast.Field) -> None + """Construct a BsonCppTypeBase.""" +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/cpp_types.py.orig mongo-r4.0.23/buildscripts/idl/idl/cpp_types.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/cpp_types.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/cpp_types.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,691 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++"""IDL C++ Code Generator.""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from abc import ABCMeta, abstractmethod ++import string ++import textwrap ++from typing import Any, Optional ++ ++from . import ast ++from . import bson ++from . import common ++from . import writer ++ ++_STD_ARRAY_UINT8_16 = 'std::array' ++ ++ ++def is_primitive_scalar_type(cpp_type): ++ # type: (unicode) -> bool ++ """ ++ Return True if a cpp_type is a primitive scalar type. ++ ++ Primitive scalar types need to have a default value to prevent warnings from Coverity. ++ """ ++ cpp_type = cpp_type.replace(' ', '') ++ return cpp_type in [ ++ 'bool', 'double', 'std::int32_t', 'std::uint32_t', 'std::uint64_t', 'std::int64_t' ++ ] ++ ++ ++def get_primitive_scalar_type_default_value(cpp_type): ++ # type: (unicode) -> unicode ++ """ ++ Return a default value for a primitive scalar type. ++ ++ Assumes the IDL generated code verifies the user sets the value before serialization. ++ """ ++ # pylint: disable=invalid-name ++ assert is_primitive_scalar_type(cpp_type) ++ if cpp_type == 'bool': ++ return 'false' ++ return '-1' ++ ++ ++def _is_primitive_type(cpp_type): ++ # type: (unicode) -> bool ++ """Return True if a cpp_type is a primitive type and should not be returned as reference.""" ++ cpp_type = cpp_type.replace(' ', '') ++ return is_primitive_scalar_type(cpp_type) or cpp_type == _STD_ARRAY_UINT8_16 ++ ++ ++def _qualify_optional_type(cpp_type): ++ # type: (unicode) -> unicode ++ """Qualify the type as optional.""" ++ return 'boost::optional<%s>' % (cpp_type) ++ ++ ++def _qualify_array_type(cpp_type): ++ # type: (unicode) -> unicode ++ """Qualify the type if the field is an array.""" ++ return "std::vector<%s>" % (cpp_type) ++ ++ ++class CppTypeBase(object): ++ """Base type for C++ Type information.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def __init__(self, field): ++ # type: (ast.Field) -> None ++ """Construct a CppTypeBase.""" ++ self._field = field ++ ++ @abstractmethod ++ def get_type_name(self): ++ # type: () -> unicode ++ """Get the C++ type name for a field.""" ++ pass ++ ++ @abstractmethod ++ def get_storage_type(self): ++ # type: () -> unicode ++ """Get the C++ type name for the storage of class member for a field.""" ++ pass ++ ++ @abstractmethod ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ """Get the C++ type name for the getter/setter parameter for a field.""" ++ pass ++ ++ @abstractmethod ++ def is_const_type(self): ++ # type: () -> bool ++ """Return True if the type should be returned by const.""" ++ pass ++ ++ @abstractmethod ++ def return_by_reference(self): ++ # type: () -> bool ++ """Return True if the type should be returned by reference.""" ++ pass ++ ++ @abstractmethod ++ def disable_xvalue(self): ++ # type: () -> bool ++ """Return True if the type should have the xvalue getter disabled.""" ++ pass ++ ++ @abstractmethod ++ def is_view_type(self): ++ # type: () -> bool ++ """Return True if the C++ is returned as a view type from an IDL class.""" ++ pass ++ ++ @abstractmethod ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ """Get the body of the getter.""" ++ pass ++ ++ @abstractmethod ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ """Get the body of the setter.""" ++ pass ++ ++ @abstractmethod ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ """Get the expression to transform the input expression into the getter type.""" ++ pass ++ ++ @abstractmethod ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ """Get the expression to transform the input expression into the setter type.""" ++ pass ++ ++ ++class _CppTypeBasic(CppTypeBase): ++ """Default class for C++ Type information. Does not handle view types.""" ++ ++ def get_type_name(self): ++ # type: () -> unicode ++ if self._field.struct_type: ++ cpp_type = common.title_case(self._field.struct_type) ++ else: ++ cpp_type = self._field.cpp_type ++ ++ return cpp_type ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return self.get_type_name() ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return self.get_type_name() ++ ++ def is_const_type(self): ++ # type: () -> bool ++ # Enum types are never const since they are mapped to primitive types, and coverity warns. ++ if self._field.enum_type: ++ return False ++ ++ type_name = self.get_type_name().replace(' ', '') ++ ++ # If it is not a primitive type, then it is const. ++ if not _is_primitive_type(type_name): ++ return True ++ ++ # Arrays of bytes should also be const though. ++ if type_name == _STD_ARRAY_UINT8_16: ++ return True ++ ++ return False ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ return not _is_primitive_type(self.get_type_name()) and not self._field.enum_type ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return False ++ ++ def is_view_type(self): ++ # type: () -> bool ++ return False ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args('return ${member_name};', member_name=member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args('${member_name} = std::move(value);', member_name=member_name) ++ ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return None ++ ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return None ++ ++ ++class _CppTypeView(CppTypeBase): ++ """Base type for C++ View Types information.""" ++ ++ def __init__(self, field, storage_type, view_type): ++ # type: (ast.Field, unicode, unicode) -> None ++ self._storage_type = storage_type ++ self._view_type = view_type ++ super(_CppTypeView, self).__init__(field) ++ ++ def get_type_name(self): ++ # type: () -> unicode ++ return self._storage_type ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return self._storage_type ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return self._view_type ++ ++ def is_const_type(self): ++ # type: () -> bool ++ return True ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ return False ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return True ++ ++ def is_view_type(self): ++ # type: () -> bool ++ return True ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args('return ${member_name};', member_name=member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args('${member_name} = ${value};', member_name=member_name, ++ value=self.get_transform_to_storage_type("value")) ++ ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return None ++ ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return common.template_args( ++ '${expression}.toString()', ++ expression=expression, ++ ) ++ ++ ++class _CppTypeVector(CppTypeBase): ++ """Base type for C++ Std::Vector Types information.""" ++ ++ def get_type_name(self): ++ # type: () -> unicode ++ return 'std::vector' ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return self.get_type_name() ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return 'ConstDataRange' ++ ++ def is_const_type(self): ++ # type: () -> bool ++ return True ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ return False ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return True ++ ++ def is_view_type(self): ++ # type: () -> bool ++ return True ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args( ++ 'return ConstDataRange(reinterpret_cast(${member_name}.data()), ${member_name}.size());', ++ member_name=member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return common.template_args('${member_name} = ${value};', member_name=member_name, ++ value=self.get_transform_to_storage_type("value")) ++ ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return common.template_args('makeCDR(${expression});', expression=expression) ++ ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return common.template_args( ++ 'std::vector(reinterpret_cast(${expression}.data()), ' + ++ 'reinterpret_cast(${expression}.data()) + ${expression}.length())', ++ expression=expression) ++ ++ ++class _CppTypeDelegating(CppTypeBase): ++ """Delegates all method calls a nested instance of CppTypeBase. Used to build other classes.""" ++ ++ def __init__(self, base, field): ++ # type: (CppTypeBase, ast.Field) -> None ++ self._base = base ++ super(_CppTypeDelegating, self).__init__(field) ++ ++ def get_type_name(self): ++ # type: () -> unicode ++ return self._base.get_type_name() ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return self._base.get_storage_type() ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return self._base.get_getter_setter_type() ++ ++ def is_const_type(self): ++ # type: () -> bool ++ return True ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ return self._base.return_by_reference() ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return self._base.disable_xvalue() ++ ++ def is_view_type(self): ++ # type: () -> bool ++ return self._base.is_view_type() ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return self._base.get_getter_body(member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ return self._base.get_setter_body(member_name) ++ ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return self._base.get_transform_to_getter_type(expression) ++ ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ return self._base.get_transform_to_storage_type(expression) ++ ++ ++class _CppTypeArray(_CppTypeDelegating): ++ """C++ Array type for wrapping a base C++ Type information.""" ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return _qualify_array_type(self._base.get_storage_type()) ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return _qualify_array_type(self._base.get_getter_setter_type()) ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ if self._base.is_view_type(): ++ return False ++ return True ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return True ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ convert = self.get_transform_to_getter_type(member_name) ++ if convert: ++ return common.template_args('return ${convert};', convert=convert) ++ return self._base.get_getter_body(member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ convert = self.get_transform_to_storage_type("value") ++ if convert: ++ return common.template_args('${member_name} = ${convert};', member_name=member_name, ++ convert=convert) ++ return self._base.get_setter_body(member_name) ++ ++ def get_transform_to_getter_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ if self._base.get_storage_type() != self._base.get_getter_setter_type(): ++ return common.template_args( ++ 'transformVector(${expression})', ++ expression=expression, ++ ) ++ return None ++ ++ def get_transform_to_storage_type(self, expression): ++ # type: (unicode) -> Optional[unicode] ++ if self._base.get_storage_type() != self._base.get_getter_setter_type(): ++ return common.template_args( ++ 'transformVector(${expression})', ++ expression=expression, ++ ) ++ return None ++ ++ ++class _CppTypeOptional(_CppTypeDelegating): ++ """Base type for Optional C++ Type information which wraps C++ types.""" ++ ++ def get_storage_type(self): ++ # type: () -> unicode ++ return _qualify_optional_type(self._base.get_storage_type()) ++ ++ def get_getter_setter_type(self): ++ # type: () -> unicode ++ return _qualify_optional_type(self._base.get_getter_setter_type()) ++ ++ def disable_xvalue(self): ++ # type: () -> bool ++ return True ++ ++ def return_by_reference(self): ++ # type: () -> bool ++ if self._base.is_view_type(): ++ return False ++ return self._base.return_by_reference() ++ ++ def get_getter_body(self, member_name): ++ # type: (unicode) -> unicode ++ base_expression = common.template_args("${member_name}.get()", member_name=member_name) ++ ++ convert = self._base.get_transform_to_getter_type(base_expression) ++ if convert: ++ # We need to convert between two different types of optional and yet provide ++ # the ability for the user specifiy an uninitialized optional. This occurs ++ # for vector and vector paired together. ++ return common.template_args( ++ textwrap.dedent("""\ ++ if (${member_name}.is_initialized()) { ++ return ${convert}; ++ } else { ++ return boost::none; ++ } ++ """), member_name=member_name, convert=convert) ++ elif self.is_view_type(): ++ # For optionals around view types, do an explicit construction ++ return common.template_args('return ${param_type}{${member_name}};', ++ param_type=self.get_getter_setter_type(), ++ member_name=member_name) ++ return common.template_args('return ${member_name};', member_name=member_name) ++ ++ def get_setter_body(self, member_name): ++ # type: (unicode) -> unicode ++ convert = self._base.get_transform_to_storage_type("value.get()") ++ if convert: ++ return common.template_args( ++ textwrap.dedent("""\ ++ if (value.is_initialized()) { ++ ${member_name} = ${convert}; ++ } else { ++ ${member_name} = boost::none; ++ } ++ """), member_name=member_name, convert=convert) ++ return self._base.get_setter_body(member_name) ++ ++ ++def get_cpp_type(field): ++ # type: (ast.Field) -> CppTypeBase ++ """Get the C++ Type information for the given field.""" ++ ++ cpp_type_info = None # type: Any ++ ++ if field.cpp_type == 'std::string': ++ cpp_type_info = _CppTypeView(field, 'std::string', 'StringData') ++ elif field.cpp_type == 'std::vector': ++ cpp_type_info = _CppTypeVector(field) ++ else: ++ cpp_type_info = _CppTypeBasic(field) ++ ++ if field.array: ++ cpp_type_info = _CppTypeArray(cpp_type_info, field) ++ ++ if field.optional: ++ cpp_type_info = _CppTypeOptional(cpp_type_info, field) ++ ++ return cpp_type_info ++ ++ ++class BsonCppTypeBase(object): ++ """Base type for custom C++ support for BSON Types information.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def __init__(self, field): ++ # type: (ast.Field) -> None ++ """Construct a BsonCppTypeBase.""" ++ self._field = field ++ ++ @abstractmethod ++ def gen_deserializer_expression(self, indented_writer, object_instance): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ """Generate code with the text writer and return an expression to deserialize the type.""" ++ pass ++ ++ @abstractmethod ++ def has_serializer(self): ++ # type: () -> bool ++ """Return True if this class generate a serializer for the given field.""" ++ pass ++ ++ @abstractmethod ++ def gen_serializer_expression(self, indented_writer, expression): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ """Generate code with the text writer and return an expression to serialize the type.""" ++ pass ++ ++ ++def _call_method_or_global_function(expression, method_name): ++ # type: (unicode, unicode) -> unicode ++ """ ++ Given a fully-qualified method name, call it correctly. ++ ++ A function is prefixed with "::" and use it to indicate a function instead of a method. It is ++ not treated as a global C++ function though. This notion of functions is designed to support ++ enum deserializers/serializers which are not methods. ++ """ ++ short_method_name = writer.get_method_name(method_name) ++ if writer.is_function(method_name): ++ return common.template_args('${method_name}(${expression})', expression=expression, ++ method_name=short_method_name) ++ ++ return common.template_args('${expression}.${method_name}()', expression=expression, ++ method_name=short_method_name) ++ ++ ++class _CommonBsonCppTypeBase(BsonCppTypeBase): ++ """Custom C++ support for basic BSON types.""" ++ ++ def __init__(self, field, deserialize_method_name): ++ # type: (ast.Field, unicode) -> None ++ self._deserialize_method_name = deserialize_method_name ++ super(_CommonBsonCppTypeBase, self).__init__(field) ++ ++ def gen_deserializer_expression(self, indented_writer, object_instance): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ return common.template_args('${object_instance}.${method_name}()', ++ object_instance=object_instance, ++ method_name=self._deserialize_method_name) ++ ++ def has_serializer(self): ++ # type: () -> bool ++ return self._field.serializer is not None ++ ++ def gen_serializer_expression(self, indented_writer, expression): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ return _call_method_or_global_function(expression, self._field.serializer) ++ ++ ++class _ObjectBsonCppTypeBase(BsonCppTypeBase): ++ """Custom C++ support for object BSON types.""" ++ ++ def gen_deserializer_expression(self, indented_writer, object_instance): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ if self._field.deserializer: ++ # Call a method like: Class::method(const BSONObj& value) ++ indented_writer.write_line( ++ common.template_args('const BSONObj localObject = ${object_instance}.Obj();', ++ object_instance=object_instance)) ++ return "localObject" ++ ++ # Just pass the BSONObj through without trying to parse it. ++ return common.template_args('${object_instance}.Obj()', object_instance=object_instance) ++ ++ def has_serializer(self): ++ # type: () -> bool ++ return self._field.serializer is not None ++ ++ def gen_serializer_expression(self, indented_writer, expression): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ method_name = writer.get_method_name(self._field.serializer) ++ indented_writer.write_line( ++ common.template_args('const BSONObj localObject = ${expression}.${method_name}();', ++ expression=expression, method_name=method_name)) ++ return "localObject" ++ ++ ++class _BinDataBsonCppTypeBase(BsonCppTypeBase): ++ """Custom C++ support for all binData BSON types.""" ++ ++ def gen_deserializer_expression(self, indented_writer, object_instance): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ if self._field.bindata_subtype == 'uuid': ++ return common.template_args('${object_instance}.uuid()', ++ object_instance=object_instance) ++ return common.template_args('${object_instance}._binDataVector()', ++ object_instance=object_instance) ++ ++ def has_serializer(self): ++ # type: () -> bool ++ return True ++ ++ def gen_serializer_expression(self, indented_writer, expression): ++ # type: (writer.IndentedTextWriter, unicode) -> unicode ++ if self._field.serializer: ++ method_name = writer.get_method_name(self._field.serializer) ++ indented_writer.write_line( ++ common.template_args('ConstDataRange tempCDR = ${expression}.${method_name}();', ++ expression=expression, method_name=method_name)) ++ else: ++ indented_writer.write_line( ++ common.template_args('ConstDataRange tempCDR = makeCDR(${expression});', ++ expression=expression)) ++ ++ return common.template_args( ++ 'BSONBinData(tempCDR.data(), tempCDR.length(), ${bindata_subtype})', ++ bindata_subtype=bson.cpp_bindata_subtype_type_name(self._field.bindata_subtype)) ++ ++ ++# For some fields, we want to support custom serialization but defer most of the serialization to ++# the core BSONElement class. This means that callers need to only process a string, a vector of ++# bytes, or a integer, not a BSONElement or BSONObj. ++def get_bson_cpp_type(field): ++ # type: (ast.Field) -> Optional[BsonCppTypeBase] ++ """Get a class that provides custom serialization for the given BSON type.""" ++ ++ # Does not support list of types ++ if len(field.bson_serialization_type) > 1: ++ return None ++ ++ if field.bson_serialization_type[0] == 'string': ++ return _CommonBsonCppTypeBase(field, "valueStringData") ++ ++ if field.bson_serialization_type[0] == 'object': ++ return _ObjectBsonCppTypeBase(field) ++ ++ if field.bson_serialization_type[0] == 'bindata': ++ return _BinDataBsonCppTypeBase(field) ++ ++ if field.bson_serialization_type[0] == 'int': ++ return _CommonBsonCppTypeBase(field, "_numberInt") ++ ++ # Unsupported type ++ return None +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/enum_types.py mongo-r4.0.23/buildscripts/idl/idl/enum_types.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/enum_types.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/enum_types.py 2021-03-17 01:21:05.952000000 +0000 +@@ -42,11 +42,11 @@ from . import common + from . import syntax + from . import writer + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + +-class EnumTypeInfoBase(object): +- """Base type for enumeration type information.""" + +- __metaclass__ = ABCMeta ++class EnumTypeInfoBase(ABC): ++ """Base type for enumeration type information.""" + + def __init__(self, idl_enum): + # type: (Union[syntax.Enum,ast.Enum]) -> None +@@ -128,8 +128,6 @@ class EnumTypeInfoBase(object): + class _EnumTypeInt(EnumTypeInfoBase): + """Type information for integer enumerations.""" + +- __metaclass__ = ABCMeta +- + def get_cpp_type_name(self): + # type: () -> unicode + return common.title_case(self._enum.name) +@@ -203,8 +201,6 @@ def _get_constant_enum_name(idl_enum, en + class _EnumTypeString(EnumTypeInfoBase): + """Type information for string enumerations.""" + +- __metaclass__ = ABCMeta +- + def get_cpp_type_name(self): + # type: () -> unicode + return common.template_args("${enum_name}Enum", enum_name=common.title_case( +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/enum_types.py.orig mongo-r4.0.23/buildscripts/idl/idl/enum_types.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/enum_types.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/enum_types.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,291 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++""" ++IDL Enum type information. ++ ++Support the code generation for enums ++""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from abc import ABCMeta, abstractmethod ++import textwrap ++from typing import cast, List, Optional, Union ++ ++from . import ast ++from . import common ++from . import syntax ++from . import writer ++ ++ ++class EnumTypeInfoBase(object): ++ """Base type for enumeration type information.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def __init__(self, idl_enum): ++ # type: (Union[syntax.Enum,ast.Enum]) -> None ++ """Construct a EnumTypeInfoBase.""" ++ self._enum = idl_enum ++ ++ def get_qualified_cpp_type_name(self): ++ # type: () -> unicode ++ """Get the fully qualified C++ type name for an enum.""" ++ return common.qualify_cpp_name(self._enum.cpp_namespace, self.get_cpp_type_name()) ++ ++ @abstractmethod ++ def get_cpp_type_name(self): ++ # type: () -> unicode ++ """Get the C++ type name for an enum.""" ++ pass ++ ++ @abstractmethod ++ def get_bson_types(self): ++ # type: () -> List[unicode] ++ """Get the BSON type names for an enum.""" ++ pass ++ ++ def _get_enum_deserializer_name(self): ++ # type: () -> unicode ++ """Return the name of deserializer function without prefix.""" ++ return common.template_args("${enum_name}_parse", enum_name=common.title_case( ++ self._enum.name)) ++ ++ def get_enum_deserializer_name(self): ++ # type: () -> unicode ++ """Return the name of deserializer function with non-method prefix.""" ++ return "::" + common.qualify_cpp_name(self._enum.cpp_namespace, ++ self._get_enum_deserializer_name()) ++ ++ def _get_enum_serializer_name(self): ++ # type: () -> unicode ++ """Return the name of serializer function without prefix.""" ++ return common.template_args("${enum_name}_serializer", enum_name=common.title_case( ++ self._enum.name)) ++ ++ def get_enum_serializer_name(self): ++ # type: () -> unicode ++ """Return the name of serializer function with non-method prefix.""" ++ return "::" + common.qualify_cpp_name(self._enum.cpp_namespace, ++ self._get_enum_serializer_name()) ++ ++ @abstractmethod ++ def get_cpp_value_assignment(self, enum_value): ++ # type: (ast.EnumValue) -> unicode ++ """Get the textual representation of the enum value, includes equal sign.""" ++ pass ++ ++ @abstractmethod ++ def get_deserializer_declaration(self): ++ # type: () -> unicode ++ """Get the deserializer function declaration minus trailing semicolon.""" ++ pass ++ ++ @abstractmethod ++ def gen_deserializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the deserializer function definition.""" ++ pass ++ ++ @abstractmethod ++ def get_serializer_declaration(self): ++ # type: () -> unicode ++ """Get the serializer function declaration minus trailing semicolon.""" ++ pass ++ ++ @abstractmethod ++ def gen_serializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the serializer function definition.""" ++ pass ++ ++ ++class _EnumTypeInt(EnumTypeInfoBase): ++ """Type information for integer enumerations.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def get_cpp_type_name(self): ++ # type: () -> unicode ++ return common.title_case(self._enum.name) ++ ++ def get_bson_types(self): ++ # type: () -> List[unicode] ++ return [self._enum.type] ++ ++ def get_cpp_value_assignment(self, enum_value): ++ # type: (ast.EnumValue) -> unicode ++ return " = %s" % (enum_value.value) ++ ++ def get_deserializer_declaration(self): ++ # type: () -> unicode ++ return common.template_args( ++ "${enum_name} ${function_name}(const IDLParserErrorContext& ctxt, std::int32_t value)", ++ enum_name=self.get_cpp_type_name(), function_name=self._get_enum_deserializer_name()) ++ ++ def gen_deserializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ enum_values = sorted(cast(ast.Enum, self._enum).values, key=lambda ev: int(ev.value)) ++ ++ template_params = { ++ 'enum_name': self.get_cpp_type_name(), ++ 'function_name': self.get_deserializer_declaration(), ++ 'min_value': enum_values[0].name, ++ 'max_value': enum_values[-1].name, ++ } ++ ++ with writer.TemplateContext(indented_writer, template_params): ++ with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"): ++ indented_writer.write_template( ++ textwrap.dedent(""" ++ if (!(value >= static_cast::type>( ++ ${enum_name}::${min_value}) && ++ value <= static_cast::type>( ++ ${enum_name}::${max_value}))) { ++ ctxt.throwBadEnumValue(value); ++ } else { ++ return static_cast<${enum_name}>(value); ++ } ++ """)) ++ ++ def get_serializer_declaration(self): ++ # type: () -> unicode ++ """Get the serializer function declaration minus trailing semicolon.""" ++ return common.template_args("std::int32_t ${function_name}(${enum_name} value)", ++ enum_name=self.get_cpp_type_name(), ++ function_name=self._get_enum_serializer_name()) ++ ++ def gen_serializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the serializer function definition.""" ++ template_params = { ++ 'enum_name': self.get_cpp_type_name(), ++ 'function_name': self.get_serializer_declaration(), ++ } ++ ++ with writer.TemplateContext(indented_writer, template_params): ++ with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"): ++ indented_writer.write_template('return static_cast(value);') ++ ++ ++def _get_constant_enum_name(idl_enum, enum_value): ++ # type: (Union[syntax.Enum,ast.Enum], Union[syntax.EnumValue,ast.EnumValue]) -> unicode ++ """Return the C++ name for a string constant of string enum value.""" ++ return common.template_args('k${enum_name}_${name}', enum_name=common.title_case(idl_enum.name), ++ name=enum_value.name) ++ ++ ++class _EnumTypeString(EnumTypeInfoBase): ++ """Type information for string enumerations.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def get_cpp_type_name(self): ++ # type: () -> unicode ++ return common.template_args("${enum_name}Enum", enum_name=common.title_case( ++ self._enum.name)) ++ ++ def get_bson_types(self): ++ # type: () -> List[unicode] ++ return [self._enum.type] ++ ++ def get_cpp_value_assignment(self, enum_value): ++ # type: (ast.EnumValue) -> unicode ++ return '' ++ ++ def get_deserializer_declaration(self): ++ # type: () -> unicode ++ return common.template_args( ++ "${enum_name} ${function_name}(const IDLParserErrorContext& ctxt, StringData value)", ++ enum_name=self.get_cpp_type_name(), function_name=self._get_enum_deserializer_name()) ++ ++ def gen_deserializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ template_params = { ++ 'enum_name': self.get_cpp_type_name(), ++ 'function_name': self.get_deserializer_declaration(), ++ } ++ ++ # Generate an anonymous namespace full of string constants ++ # ++ with writer.NamespaceScopeBlock(indented_writer, ['']): ++ for enum_value in self._enum.values: ++ indented_writer.write_line( ++ common.template_args('constexpr StringData ${constant_name} = "${value}"_sd;', ++ constant_name=_get_constant_enum_name( ++ self._enum, enum_value), value=enum_value.value)) ++ indented_writer.write_empty_line() ++ ++ with writer.TemplateContext(indented_writer, template_params): ++ with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"): ++ for enum_value in self._enum.values: ++ predicate = 'if (value == %s) {' % ( ++ _get_constant_enum_name(self._enum, enum_value)) ++ with writer.IndentedScopedBlock(indented_writer, predicate, "}"): ++ indented_writer.write_template('return ${enum_name}::%s;' % ++ (enum_value.name)) ++ ++ indented_writer.write_line("ctxt.throwBadEnumValue(value);") ++ ++ def get_serializer_declaration(self): ++ # type: () -> unicode ++ """Get the serializer function declaration minus trailing semicolon.""" ++ return common.template_args("StringData ${function_name}(${enum_name} value)", ++ enum_name=self.get_cpp_type_name(), ++ function_name=self._get_enum_serializer_name()) ++ ++ def gen_serializer_definition(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the serializer function definition.""" ++ template_params = { ++ 'enum_name': self.get_cpp_type_name(), ++ 'function_name': self.get_serializer_declaration(), ++ } ++ ++ with writer.TemplateContext(indented_writer, template_params): ++ with writer.IndentedScopedBlock(indented_writer, "${function_name} {", "}"): ++ for enum_value in self._enum.values: ++ with writer.IndentedScopedBlock(indented_writer, ++ 'if (value == ${enum_name}::%s) {' % ++ (enum_value.name), "}"): ++ indented_writer.write_line( ++ 'return %s;' % (_get_constant_enum_name(self._enum, enum_value))) ++ ++ indented_writer.write_line('MONGO_UNREACHABLE;') ++ indented_writer.write_line('return StringData();') ++ ++ ++def get_type_info(idl_enum): ++ # type: (Union[syntax.Enum,ast.Enum]) -> Optional[EnumTypeInfoBase] ++ """Get the type information for a given enumeration type, return None if not supported.""" ++ if idl_enum.type == 'int': ++ return _EnumTypeInt(idl_enum) ++ elif idl_enum.type == 'string': ++ return _EnumTypeString(idl_enum) ++ ++ return None +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/generator.py mongo-r4.0.23/buildscripts/idl/idl/generator.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/generator.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/generator.py 2021-03-17 01:21:05.952000000 +0000 +@@ -46,6 +46,7 @@ from . import enum_types + from . import struct_types + from . import writer + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + + def _get_field_member_name(field): + # type: (ast.Field) -> unicode +@@ -135,11 +136,9 @@ def _get_all_fields(struct): + return sorted([field for field in all_fields], key=lambda f: f.cpp_name) + + +-class _FieldUsageCheckerBase(object): ++class _FieldUsageCheckerBase(ABC): + """Check for duplicate fields, and required fields as needed.""" + +- __metaclass__ = ABCMeta +- + def __init__(self, indented_writer): + # type: (writer.IndentedTextWriter) -> None + """Create a field usage checker.""" +@@ -1604,8 +1603,8 @@ def _generate_header(spec, file_name): + str_value = generate_header_str(spec) + + # Generate structs +- with io.open(file_name, mode='wb') as file_handle: +- file_handle.write(str_value.encode()) ++ with io.open(file_name, mode='w') as file_handle: ++ file_handle.write(str_value) + + + def generate_source_str(spec, target_arch, header_file_name): +@@ -1627,8 +1626,8 @@ def _generate_source(spec, target_arch, + str_value = generate_source_str(spec, target_arch, header_file_name) + + # Generate structs +- with io.open(file_name, mode='wb') as file_handle: +- file_handle.write(str_value.encode()) ++ with io.open(file_name, mode='w') as file_handle: ++ file_handle.write(str_value) + + + def generate_code(spec, target_arch, output_base_dir, header_file_name, source_file_name): +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/generator.py.orig mongo-r4.0.23/buildscripts/idl/idl/generator.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/generator.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/generator.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,1649 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++# pylint: disable=too-many-lines ++"""IDL C++ Code Generator.""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from abc import ABCMeta, abstractmethod ++import io ++import os ++import string ++import sys ++import textwrap ++from typing import cast, List, Mapping, Union ++ ++from . import ast ++from . import bson ++from . import common ++from . import cpp_types ++from . import enum_types ++from . import struct_types ++from . import writer ++ ++ ++def _get_field_member_name(field): ++ # type: (ast.Field) -> unicode ++ """Get the C++ class member name for a field.""" ++ return '_%s' % (common.camel_case(field.cpp_name)) ++ ++ ++def _get_field_member_setter_name(field): ++ # type: (ast.Field) -> unicode ++ """Get the C++ class setter name for a field.""" ++ return "set%s" % (common.title_case(field.cpp_name)) ++ ++ ++def _get_field_member_getter_name(field): ++ # type: (ast.Field) -> unicode ++ """Get the C++ class getter name for a field.""" ++ return "get%s" % (common.title_case(field.cpp_name)) ++ ++ ++def _get_has_field_member_name(field): ++ # type: (ast.Field) -> unicode ++ """Get the C++ class member name for bool 'has' member field.""" ++ return '_has%s' % (common.title_case(field.cpp_name)) ++ ++ ++def _is_required_serializer_field(field): ++ # type: (ast.Field) -> bool ++ """ ++ Get whether we require this field to have a value set before serialization. ++ ++ Fields that must be set before serialization are fields without default values, that are not ++ optional, and are not chained. ++ """ ++ return not field.ignore and not field.optional and not field.default and not field.chained and not field.chained_struct_field ++ ++ ++def _get_field_constant_name(field): ++ # type: (ast.Field) -> unicode ++ """Get the C++ string constant name for a field.""" ++ return common.template_args('k${constant_name}FieldName', constant_name=common.title_case( ++ field.cpp_name)) ++ ++ ++def _access_member(field): ++ # type: (ast.Field) -> unicode ++ """Get the declaration to access a member for a field.""" ++ member_name = _get_field_member_name(field) ++ ++ if not field.optional: ++ return '%s' % (member_name) ++ ++ # optional types need a method call to access their values ++ return '%s.get()' % (member_name) ++ ++ ++def _get_bson_type_check(bson_element, ctxt_name, field): ++ # type: (unicode, unicode, ast.Field) -> unicode ++ """Get the C++ bson type check for a field.""" ++ bson_types = field.bson_serialization_type ++ if len(bson_types) == 1: ++ if bson_types[0] in ['any', 'chain']: ++ # Skip BSON validation for 'any' types since they are required to validate the ++ # BSONElement. ++ # Skip BSON validation for 'chain' types since they process the raw BSONObject the ++ # encapsulating IDL struct parser is passed. ++ return None ++ ++ if not bson_types[0] == 'bindata': ++ return '%s.checkAndAssertType(%s, %s)' % (ctxt_name, bson_element, ++ bson.cpp_bson_type_name(bson_types[0])) ++ return '%s.checkAndAssertBinDataType(%s, %s)' % ( ++ ctxt_name, bson_element, bson.cpp_bindata_subtype_type_name(field.bindata_subtype)) ++ else: ++ type_list = '{%s}' % (', '.join([bson.cpp_bson_type_name(b) for b in bson_types])) ++ return '%s.checkAndAssertTypes(%s, %s)' % (ctxt_name, bson_element, type_list) ++ ++ ++def _get_all_fields(struct): ++ # type: (ast.Struct) -> List[ast.Field] ++ """Get a list of all the fields, including the command field.""" ++ all_fields = [] ++ if isinstance(struct, ast.Command) and struct.command_field: ++ all_fields.append(struct.command_field) ++ ++ all_fields += struct.fields ++ ++ return sorted([field for field in all_fields], key=lambda f: f.cpp_name) ++ ++ ++class _FieldUsageCheckerBase(object): ++ """Check for duplicate fields, and required fields as needed.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def __init__(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Create a field usage checker.""" ++ self._writer = indented_writer # type: writer.IndentedTextWriter ++ self._fields = [] # type: List[ast.Field] ++ ++ @abstractmethod ++ def add_store(self, field_name): ++ # type: (unicode) -> None ++ """Create the C++ field store initialization code.""" ++ pass ++ ++ @abstractmethod ++ def add(self, field, bson_element_variable): ++ # type: (ast.Field, unicode) -> None ++ """Add a field to track.""" ++ pass ++ ++ @abstractmethod ++ def add_final_checks(self): ++ # type: () -> None ++ """Output the code to check for missing fields.""" ++ pass ++ ++ ++class _SlowFieldUsageChecker(_FieldUsageCheckerBase): ++ """ ++ Check for duplicate fields, and required fields as needed. ++ ++ Detects duplicate extra fields. ++ Generates code with a C++ std::set to maintain a set of fields seen while parsing a BSON ++ document. The std::set has O(N lg N) lookup, and allocates memory in the heap. ++ """ ++ ++ def __init__(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ super(_SlowFieldUsageChecker, self).__init__(indented_writer) ++ ++ self._writer.write_line('std::set usedFields;') ++ ++ def add_store(self, field_name): ++ # type: (unicode) -> None ++ self._writer.write_line('auto push_result = usedFields.insert(%s);' % (field_name)) ++ with writer.IndentedScopedBlock(self._writer, ++ 'if (MONGO_unlikely(push_result.second == false)) {', '}'): ++ self._writer.write_line('ctxt.throwDuplicateField(%s);' % (field_name)) ++ ++ def add(self, field, bson_element_variable): ++ # type: (ast.Field, unicode) -> None ++ if not field in self._fields: ++ self._fields.append(field) ++ ++ def add_final_checks(self): ++ # type: () -> None ++ for field in self._fields: ++ if (not field.optional) and (not field.ignore) and (not field.chained): ++ pred = 'if (MONGO_unlikely(usedFields.find(%s) == usedFields.end())) {' % \ ++ (_get_field_constant_name(field)) ++ with writer.IndentedScopedBlock(self._writer, pred, '}'): ++ if field.default: ++ if field.enum_type: ++ self._writer.write_line('%s = %s::%s;' % ++ (_get_field_member_name(field), field.cpp_type, ++ field.default)) ++ else: ++ self._writer.write_line('%s = %s;' % (_get_field_member_name(field), ++ field.default)) ++ else: ++ self._writer.write_line('ctxt.throwMissingField(%s);' % ++ (_get_field_constant_name(field))) ++ ++ ++def _gen_field_usage_constant(field): ++ # type: (ast.Field) -> unicode ++ """Get the name for a bitset constant in field usage checking.""" ++ return "k%sBit" % (common.title_case(field.cpp_name)) ++ ++ ++class _FastFieldUsageChecker(_FieldUsageCheckerBase): ++ """ ++ Check for duplicate fields, and required fields as needed. ++ ++ Does not detect duplicate extra fields. Only works for strict parsers. ++ Generates code with a C++ std::bitset to maintain a record each field seen while parsing a ++ document. The std::bitset has O(1) lookup, and allocates a single int or similar on the stack. ++ """ ++ ++ def __init__(self, indented_writer, fields): ++ # type: (writer.IndentedTextWriter, List[ast.Field]) -> None ++ super(_FastFieldUsageChecker, self).__init__(indented_writer) ++ ++ self._writer.write_line('std::bitset<%d> usedFields;' % (len(fields))) ++ ++ bit_id = 0 ++ for field in fields: ++ if field.chained: ++ continue ++ ++ self._writer.write_line('const size_t %s = %d;' % (_gen_field_usage_constant(field), ++ bit_id)) ++ bit_id += 1 ++ ++ def add_store(self, field_name): ++ # type: (unicode) -> None ++ """Create the C++ field store initialization code.""" ++ pass ++ ++ def add(self, field, bson_element_variable): ++ # type: (ast.Field, unicode) -> None ++ """Add a field to track.""" ++ if not field in self._fields: ++ self._fields.append(field) ++ ++ with writer.IndentedScopedBlock(self._writer, 'if (MONGO_unlikely(usedFields[%s])) {' % ++ (_gen_field_usage_constant(field)), '}'): ++ self._writer.write_line('ctxt.throwDuplicateField(%s);' % (bson_element_variable)) ++ self._writer.write_empty_line() ++ ++ self._writer.write_line('usedFields.set(%s);' % (_gen_field_usage_constant(field))) ++ self._writer.write_empty_line() ++ ++ def add_final_checks(self): ++ # type: () -> None ++ """Output the code to check for missing fields.""" ++ with writer.IndentedScopedBlock(self._writer, 'if (MONGO_unlikely(!usedFields.all())) {', ++ '}'): ++ for field in self._fields: ++ if (not field.optional) and (not field.ignore): ++ with writer.IndentedScopedBlock(self._writer, 'if (!usedFields[%s]) {' % ++ (_gen_field_usage_constant(field)), '}'): ++ if field.default: ++ if field.chained_struct_field: ++ self._writer.write_line( ++ '%s.%s(%s);' % ++ (_get_field_member_name(field.chained_struct_field), ++ _get_field_member_setter_name(field), field.default)) ++ elif field.enum_type: ++ self._writer.write_line('%s = %s::%s;' % ++ (_get_field_member_name(field), ++ field.cpp_type, field.default)) ++ else: ++ self._writer.write_line('%s = %s;' % (_get_field_member_name(field), ++ field.default)) ++ else: ++ self._writer.write_line('ctxt.throwMissingField(%s);' % ++ (_get_field_constant_name(field))) ++ ++ ++def _get_field_usage_checker(indented_writer, struct): ++ # type: (writer.IndentedTextWriter, ast.Struct) -> _FieldUsageCheckerBase ++ ++ # Only use the fast field usage checker if we never expect extra fields that we need to ignore ++ # but still wish to do duplicate detection on. ++ if struct.strict: ++ return _FastFieldUsageChecker(indented_writer, struct.fields) ++ ++ return _SlowFieldUsageChecker(indented_writer) ++ ++ ++class _CppFileWriterBase(object): ++ """ ++ C++ File writer. ++ ++ Encapsulates low level knowledge of how to print a C++ file. ++ Relies on caller to orchestrate calls correctly though. ++ """ ++ ++ def __init__(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Create a C++ code writer.""" ++ self._writer = indented_writer # type: writer.IndentedTextWriter ++ ++ def write_unindented_line(self, msg): ++ # type: (unicode) -> None ++ """Write an unindented line to the stream.""" ++ self._writer.write_unindented_line(msg) ++ ++ def write_empty_line(self): ++ # type: () -> None ++ """Write an empty line to the stream.""" ++ self._writer.write_empty_line() ++ ++ def gen_file_header(self): ++ # type: () -> None ++ """Generate a file header saying the file is generated.""" ++ self._writer.write_unindented_line( ++ textwrap.dedent("""\ ++ /** ++ * WARNING: This is a generated file. Do not modify. ++ * ++ * Source: %s ++ */ ++ """ % (" ".join(sys.argv)))) ++ ++ def gen_system_include(self, include): ++ # type: (unicode) -> None ++ """Generate a system C++ include line.""" ++ self._writer.write_unindented_line('#include <%s>' % (include)) ++ ++ def gen_include(self, include): ++ # type: (unicode) -> None ++ """Generate a non-system C++ include line.""" ++ self._writer.write_unindented_line('#include "%s"' % (include)) ++ ++ def gen_namespace_block(self, namespace): ++ # type: (unicode) -> writer.NamespaceScopeBlock ++ """Generate a namespace block.""" ++ namespace_list = namespace.split("::") ++ ++ return writer.NamespaceScopeBlock(self._writer, namespace_list) ++ ++ def gen_description_comment(self, description): ++ # type: (unicode) -> None ++ """Generate a multiline comment with the description from the IDL.""" ++ self._writer.write_line( ++ textwrap.dedent("""\ ++ /** ++ * %s ++ */""" % (description))) ++ ++ def _with_template(self, template_params): ++ # type: (Mapping[unicode,unicode]) -> writer.TemplateContext ++ """Generate a template context for the current parameters.""" ++ return writer.TemplateContext(self._writer, template_params) ++ ++ def _block(self, opening, closing): ++ # type: (unicode, unicode) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock] ++ """Generate an indented block if opening is not empty.""" ++ if not opening: ++ return writer.EmptyBlock() ++ ++ return writer.IndentedScopedBlock(self._writer, opening, closing) ++ ++ def _predicate(self, check_str, use_else_if=False): ++ # type: (unicode, bool) -> Union[writer.IndentedScopedBlock,writer.EmptyBlock] ++ """ ++ Generate an if block if the condition is not-empty. ++ ++ Generate 'else if' instead of use_else_if is True. ++ """ ++ if not check_str: ++ return writer.EmptyBlock() ++ ++ conditional = 'if' ++ if use_else_if: ++ conditional = 'else if' ++ ++ return writer.IndentedScopedBlock(self._writer, '%s (%s) {' % (conditional, check_str), '}') ++ ++ ++class _CppHeaderFileWriter(_CppFileWriterBase): ++ """C++ .h File writer.""" ++ ++ def gen_class_declaration_block(self, class_name): ++ # type: (unicode) -> writer.IndentedScopedBlock ++ """Generate a class declaration block.""" ++ return writer.IndentedScopedBlock(self._writer, ++ 'class %s {' % common.title_case(class_name), '};') ++ ++ def gen_class_constructors(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the declarations for the class constructors.""" ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ self._writer.write_line(struct_type_info.get_constructor_method().get_declaration()) ++ ++ def gen_serializer_methods(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate a serializer method declarations.""" ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ parse_method = struct_type_info.get_deserializer_static_method() ++ if parse_method: ++ self._writer.write_line(parse_method.get_declaration()) ++ ++ parse_method = struct_type_info.get_op_msg_request_deserializer_static_method() ++ if parse_method: ++ self._writer.write_line(parse_method.get_declaration()) ++ ++ self._writer.write_line(struct_type_info.get_serializer_method().get_declaration()) ++ ++ parse_method = struct_type_info.get_op_msg_request_serializer_method() ++ if parse_method: ++ self._writer.write_line(parse_method.get_declaration()) ++ ++ self._writer.write_line(struct_type_info.get_to_bson_method().get_declaration()) ++ ++ self._writer.write_empty_line() ++ ++ def gen_protected_serializer_methods(self, struct): ++ # type: (ast.Struct) -> None ++ # pylint: disable=invalid-name ++ """Generate protected serializer method declarations.""" ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ parse_method = struct_type_info.get_deserializer_method() ++ if parse_method: ++ self._writer.write_line(parse_method.get_declaration()) ++ ++ parse_method = struct_type_info.get_op_msg_request_deserializer_method() ++ if parse_method: ++ self._writer.write_line(parse_method.get_declaration()) ++ ++ self._writer.write_empty_line() ++ ++ def gen_getter(self, struct, field): ++ # type: (ast.Struct, ast.Field) -> None ++ """Generate the C++ getter definition for a field.""" ++ cpp_type_info = cpp_types.get_cpp_type(field) ++ param_type = cpp_type_info.get_getter_setter_type() ++ member_name = _get_field_member_name(field) ++ ++ if cpp_type_info.return_by_reference(): ++ param_type += "&" ++ ++ template_params = { ++ 'method_name': _get_field_member_getter_name(field), ++ 'param_type': param_type, ++ 'body': cpp_type_info.get_getter_body(member_name), ++ 'const_type': 'const ' if cpp_type_info.is_const_type() else '', ++ } ++ ++ # Generate a getter that disables xvalue for view types (i.e. StringData), constructed ++ # optional types, and non-primitive types. ++ with self._with_template(template_params): ++ ++ if field.chained_struct_field: ++ self._writer.write_template( ++ '${const_type} ${param_type} ${method_name}() const { return %s.%s(); }' % ++ ((_get_field_member_name(field.chained_struct_field), ++ _get_field_member_getter_name(field)))) ++ ++ elif cpp_type_info.disable_xvalue(): ++ self._writer.write_template( ++ 'const ${param_type} ${method_name}() const& { ${body} }') ++ self._writer.write_template('void ${method_name}() && = delete;') ++ ++ elif field.struct_type: ++ # Support mutable accessors ++ self._writer.write_template( ++ 'const ${param_type} ${method_name}() const { ${body} }') ++ ++ if not struct.immutable: ++ self._writer.write_template('${param_type} ${method_name}() { ${body} }') ++ else: ++ self._writer.write_template( ++ '${const_type}${param_type} ${method_name}() const { ${body} }') ++ ++ if field.non_const_getter: ++ self._writer.write_template('${param_type} ${method_name}() { ${body} }') ++ ++ def gen_setter(self, field): ++ # type: (ast.Field) -> None ++ """Generate the C++ setter definition for a field.""" ++ cpp_type_info = cpp_types.get_cpp_type(field) ++ param_type = cpp_type_info.get_getter_setter_type() ++ member_name = _get_field_member_name(field) ++ ++ post_body = '' ++ if _is_required_serializer_field(field): ++ post_body = '%s = true;' % (_get_has_field_member_name(field)) ++ ++ template_params = { ++ 'method_name': _get_field_member_setter_name(field), ++ 'member_name': member_name, ++ 'param_type': param_type, ++ 'body': cpp_type_info.get_setter_body(member_name), ++ 'post_body': post_body, ++ } ++ ++ with self._with_template(template_params): ++ self._writer.write_template( ++ 'void ${method_name}(${param_type} value) & ' + '{ ${body} ${post_body} }') ++ ++ self._writer.write_empty_line() ++ ++ def gen_member(self, field): ++ # type: (ast.Field) -> None ++ """Generate the C++ class member definition for a field.""" ++ cpp_type_info = cpp_types.get_cpp_type(field) ++ member_type = cpp_type_info.get_storage_type() ++ member_name = _get_field_member_name(field) ++ ++ if field.default and not field.constructed: ++ if field.enum_type: ++ self._writer.write_line('%s %s{%s::%s};' % (member_type, member_name, ++ field.cpp_type, field.default)) ++ else: ++ self._writer.write_line('%s %s{%s};' % (member_type, member_name, field.default)) ++ else: ++ self._writer.write_line('%s %s;' % (member_type, member_name)) ++ ++ def gen_serializer_member(self, field): ++ # type: (ast.Field) -> None ++ """Generate the C++ class bool has_ member definition for a field.""" ++ has_member_name = _get_has_field_member_name(field) ++ ++ # Use a bitfield to save space ++ self._writer.write_line('bool %s : 1;' % (has_member_name)) ++ ++ def gen_string_constants_declarations(self, struct): ++ # type: (ast.Struct) -> None ++ # pylint: disable=invalid-name ++ """Generate a StringData constant for field name.""" ++ ++ for field in _get_all_fields(struct): ++ self._writer.write_line( ++ common.template_args('static constexpr auto ${constant_name} = "${field_name}"_sd;', ++ constant_name=_get_field_constant_name(field), ++ field_name=field.name)) ++ ++ if isinstance(struct, ast.Command): ++ self._writer.write_line( ++ common.template_args('static constexpr auto kCommandName = "${struct_name}"_sd;', ++ struct_name=struct.name)) ++ ++ def gen_enum_functions(self, idl_enum): ++ # type: (ast.Enum) -> None ++ """Generate the declaration for an enum's supporting functions.""" ++ enum_type_info = enum_types.get_type_info(idl_enum) ++ ++ self._writer.write_line("%s;" % (enum_type_info.get_deserializer_declaration())) ++ ++ self._writer.write_line("%s;" % (enum_type_info.get_serializer_declaration())) ++ ++ def gen_enum_declaration(self, idl_enum): ++ # type: (ast.Enum) -> None ++ """Generate the declaration for an enum.""" ++ enum_type_info = enum_types.get_type_info(idl_enum) ++ ++ with self._block('enum class %s : std::int32_t {' % (enum_type_info.get_cpp_type_name()), ++ '};'): ++ for enum_value in idl_enum.values: ++ self._writer.write_line( ++ common.template_args('${name} ${value},', name=enum_value.name, ++ value=enum_type_info.get_cpp_value_assignment(enum_value))) ++ ++ def gen_op_msg_request_methods(self, command): ++ # type: (ast.Command) -> None ++ """Generate the methods for a command.""" ++ if command.command_field: ++ self.gen_getter(command, command.command_field) ++ else: ++ struct_type_info = struct_types.get_struct_info(command) ++ struct_type_info.gen_getter_method(self._writer) ++ ++ self._writer.write_empty_line() ++ ++ def gen_op_msg_request_member(self, command): ++ # type: (ast.Command) -> None ++ """Generate the class members for a command.""" ++ if command.command_field: ++ self.gen_member(command.command_field) ++ else: ++ struct_type_info = struct_types.get_struct_info(command) ++ struct_type_info.gen_member(self._writer) ++ ++ self._writer.write_empty_line() ++ ++ def gen_known_fields_declaration(self): ++ # type: () -> None ++ """Generate a known fields vector for a command.""" ++ self._writer.write_line("static const std::vector _knownFields;") ++ self.write_empty_line() ++ ++ def gen_comparison_operators_declarations(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate comparison operators declarations for the type.""" ++ # pylint: disable=invalid-name ++ ++ sorted_fields = sorted([ ++ field for field in struct.fields if (not field.ignore) and field.comparison_order != -1 ++ ], key=lambda f: f.comparison_order) ++ fields = [_get_field_member_name(field) for field in sorted_fields] ++ ++ with self._block("auto relationalTie() const {", "}"): ++ self._writer.write_line('return std::tie(%s);' % (', '.join(fields))) ++ ++ for rel_op in ['==', '!=', '<', '>', '<=', '>=']: ++ self.write_empty_line() ++ decl = common.template_args( ++ "friend bool operator${rel_op}(const ${class_name}& left, const ${class_name}& right) {", ++ rel_op=rel_op, class_name=common.title_case(struct.name)) ++ ++ with self._block(decl, "}"): ++ self._writer.write_line('return left.relationalTie() %s right.relationalTie();' % ++ (rel_op)) ++ ++ self.write_empty_line() ++ ++ def generate(self, spec): ++ # type: (ast.IDLAST) -> None ++ """Generate the C++ header to a stream.""" ++ # pylint: disable=too-many-branches,too-many-statements ++ self.gen_file_header() ++ ++ self._writer.write_unindented_line('#pragma once') ++ self.write_empty_line() ++ ++ # Generate system includes first ++ header_list = [ ++ 'algorithm', ++ 'boost/optional.hpp', ++ 'cstdint', ++ 'string', ++ 'tuple', ++ 'vector', ++ ] ++ ++ header_list.sort() ++ ++ for include in header_list: ++ self.gen_system_include(include) ++ ++ self.write_empty_line() ++ ++ # Generate user includes second ++ header_list = [ ++ 'mongo/base/string_data.h', ++ 'mongo/base/data_range.h', ++ 'mongo/bson/bsonobj.h', ++ 'mongo/bson/bsonobjbuilder.h', ++ 'mongo/idl/idl_parser.h', ++ 'mongo/rpc/op_msg.h', ++ ] + spec.globals.cpp_includes ++ ++ header_list.sort() ++ ++ for include in header_list: ++ self.gen_include(include) ++ ++ self.write_empty_line() ++ ++ # Generate namesapce ++ with self.gen_namespace_block(spec.globals.cpp_namespace): ++ self.write_empty_line() ++ ++ for idl_enum in spec.enums: ++ self.gen_description_comment(idl_enum.description) ++ self.gen_enum_declaration(idl_enum) ++ self._writer.write_empty_line() ++ ++ self.gen_enum_functions(idl_enum) ++ self._writer.write_empty_line() ++ ++ spec_and_structs = spec.structs ++ spec_and_structs += spec.commands ++ ++ for struct in spec_and_structs: ++ self.gen_description_comment(struct.description) ++ with self.gen_class_declaration_block(struct.cpp_name): ++ self.write_unindented_line('public:') ++ ++ # Generate a sorted list of string constants ++ self.gen_string_constants_declarations(struct) ++ self.write_empty_line() ++ ++ # Write constructor ++ self.gen_class_constructors(struct) ++ self.write_empty_line() ++ ++ # Write serialization ++ self.gen_serializer_methods(struct) ++ ++ if isinstance(struct, ast.Command): ++ self.gen_op_msg_request_methods(struct) ++ ++ # Write getters & setters ++ for field in struct.fields: ++ if not field.ignore: ++ if field.description: ++ self.gen_description_comment(field.description) ++ self.gen_getter(struct, field) ++ if not struct.immutable and not field.chained_struct_field: ++ self.gen_setter(field) ++ ++ if struct.generate_comparison_operators: ++ self.gen_comparison_operators_declarations(struct) ++ ++ self.write_unindented_line('protected:') ++ self.gen_protected_serializer_methods(struct) ++ ++ self.write_unindented_line('private:') ++ ++ # Write command member variables ++ if isinstance(struct, ast.Command): ++ self.gen_known_fields_declaration() ++ self.write_empty_line() ++ ++ self.gen_op_msg_request_member(struct) ++ ++ # Write member variables ++ for field in struct.fields: ++ if not field.ignore and not field.chained_struct_field: ++ self.gen_member(field) ++ ++ # Write serializer member variables ++ # Note: we write these out second to ensure the bit fields can be packed by ++ # the compiler. ++ for field in struct.fields: ++ if _is_required_serializer_field(field): ++ self.gen_serializer_member(field) ++ ++ self.write_empty_line() ++ ++ ++class _CppSourceFileWriter(_CppFileWriterBase): ++ """C++ .cpp File writer.""" ++ ++ def __init__(self, indented_writer, target_arch): ++ # type: (writer.IndentedTextWriter, unicode) -> None ++ """Create a C++ .cpp file code writer.""" ++ self._target_arch = target_arch ++ super(_CppSourceFileWriter, self).__init__(indented_writer) ++ ++ def _gen_field_deserializer_expression(self, element_name, field): ++ # type: (unicode, ast.Field) -> unicode ++ # pylint: disable=invalid-name ++ """ ++ Generate the C++ deserializer piece for a field. ++ ++ Writes multiple lines into the generated file. ++ Returns the final statement to access the deserialized value. ++ """ ++ ++ if field.struct_type: ++ self._writer.write_line('IDLParserErrorContext tempContext(%s, &ctxt);' % ++ (_get_field_constant_name(field))) ++ self._writer.write_line('const auto localObject = %s.Obj();' % (element_name)) ++ return '%s::parse(tempContext, localObject)' % (common.title_case(field.struct_type)) ++ elif field.deserializer and 'BSONElement::' in field.deserializer: ++ method_name = writer.get_method_name(field.deserializer) ++ return '%s.%s()' % (element_name, method_name) ++ ++ # Custom method, call the method on object. ++ bson_cpp_type = cpp_types.get_bson_cpp_type(field) ++ ++ if bson_cpp_type: ++ # Call a static class method with the signature: ++ # Class Class::method(StringData value) ++ # or ++ # Class::method(const BSONObj& value) ++ expression = bson_cpp_type.gen_deserializer_expression(self._writer, element_name) ++ if field.deserializer: ++ method_name = writer.get_method_name_from_qualified_method_name(field.deserializer) ++ ++ # For fields which are enums, pass a IDLParserErrorContext ++ if field.enum_type: ++ self._writer.write_line('IDLParserErrorContext tempContext(%s, &ctxt);' % ++ (_get_field_constant_name(field))) ++ return common.template_args("${method_name}(tempContext, ${expression})", ++ method_name=method_name, expression=expression) ++ return common.template_args("${method_name}(${expression})", ++ method_name=method_name, expression=expression) ++ ++ # BSONObjects are allowed to be pass through without deserialization ++ assert field.bson_serialization_type == ['object'] ++ return expression ++ ++ # Call a static class method with the signature: ++ # Class Class::method(const BSONElement& value) ++ method_name = writer.get_method_name_from_qualified_method_name(field.deserializer) ++ ++ return '%s(%s)' % (method_name, element_name) ++ ++ def _gen_array_deserializer(self, field, bson_element): ++ # type: (ast.Field, unicode) -> None ++ """Generate the C++ deserializer piece for an array field.""" ++ cpp_type_info = cpp_types.get_cpp_type(field) ++ cpp_type = cpp_type_info.get_type_name() ++ ++ self._writer.write_line('std::uint32_t expectedFieldNumber{0};') ++ self._writer.write_line('const IDLParserErrorContext arrayCtxt(%s, &ctxt);' % ++ (_get_field_constant_name(field))) ++ self._writer.write_line('std::vector<%s> values;' % (cpp_type)) ++ self._writer.write_empty_line() ++ ++ self._writer.write_line('const BSONObj arrayObject = %s.Obj();' % (bson_element)) ++ ++ with self._block('for (const auto& arrayElement : arrayObject) {', '}'): ++ ++ self._writer.write_line( ++ 'const auto arrayFieldName = arrayElement.fieldNameStringData();') ++ self._writer.write_line('std::uint32_t fieldNumber;') ++ self._writer.write_empty_line() ++ ++ # Check the array field names are integers ++ self._writer.write_line( ++ 'Status status = parseNumberFromString(arrayFieldName, &fieldNumber);') ++ with self._predicate('status.isOK()'): ++ ++ # Check that the array field names are sequential ++ with self._predicate('fieldNumber != expectedFieldNumber'): ++ self._writer.write_line('arrayCtxt.throwBadArrayFieldNumberSequence(' + ++ 'fieldNumber, expectedFieldNumber);') ++ self._writer.write_empty_line() ++ ++ with self._predicate(_get_bson_type_check('arrayElement', 'arrayCtxt', field)): ++ array_value = self._gen_field_deserializer_expression('arrayElement', field) ++ ++ # HACK - SERVER-32431 ++ # GCC 5.4.0 on s390x has a code gen bug, work around it by not using std::move ++ if self._target_arch == "s390x": ++ self._writer.write_line('auto localValue = %s;' % (array_value)) ++ self._writer.write_line('values.push_back(localValue);') ++ else: ++ self._writer.write_line('values.emplace_back(%s);' % (array_value)) ++ ++ with self._block('else {', '}'): ++ self._writer.write_line('arrayCtxt.throwBadArrayFieldNumberValue(arrayFieldName);') ++ ++ self._writer.write_line('++expectedFieldNumber;') ++ ++ if field.chained_struct_field: ++ self._writer.write_line('%s.%s(std::move(values));' % ++ (_get_field_member_name(field.chained_struct_field), ++ _get_field_member_setter_name(field))) ++ else: ++ self._writer.write_line('%s = std::move(values);' % (_get_field_member_name(field))) ++ ++ def gen_field_deserializer(self, field, bson_object, bson_element): ++ # type: (ast.Field, unicode, unicode) -> None ++ """Generate the C++ deserializer piece for a field.""" ++ if field.array: ++ self._gen_array_deserializer(field, bson_element) ++ return ++ ++ if field.chained: ++ # Do not generate a predicate check since we always call these deserializers. ++ ++ if field.struct_type: ++ # Do not generate a new parser context, reuse the current one since we are not ++ # entering a nested document. ++ expression = '%s::parse(ctxt, %s)' % (common.title_case(field.struct_type), ++ bson_object) ++ else: ++ method_name = writer.get_method_name_from_qualified_method_name(field.deserializer) ++ expression = "%s(%s)" % (method_name, bson_object) ++ ++ self._writer.write_line('%s = %s;' % (_get_field_member_name(field), expression)) ++ else: ++ # May be an empty block if the type is 'any' ++ predicate = _get_bson_type_check(bson_element, 'ctxt', field) ++ if predicate: ++ predicate = "MONGO_likely(%s)" % (predicate) ++ with self._predicate(predicate): ++ object_value = self._gen_field_deserializer_expression(bson_element, field) ++ if field.chained_struct_field: ++ self._writer.write_line('%s.%s(%s);' % ++ (_get_field_member_name(field.chained_struct_field), ++ _get_field_member_setter_name(field), object_value)) ++ else: ++ self._writer.write_line('%s = %s;' % (_get_field_member_name(field), ++ object_value)) ++ ++ def gen_doc_sequence_deserializer(self, field): ++ # type: (ast.Field) -> None ++ """Generate the C++ deserializer piece for a C++ mongo::OpMsg::DocumentSequence.""" ++ cpp_type_info = cpp_types.get_cpp_type(field) ++ cpp_type = cpp_type_info.get_type_name() ++ ++ self._writer.write_line('std::vector<%s> values;' % (cpp_type)) ++ self._writer.write_empty_line() ++ ++ # TODO: add support for sequence length checks, today we allow an empty document sequence ++ # because we do not give a way for IDL specifications to specify if they allow empty ++ # sequences or require non-empty sequences. ++ ++ with self._block('for (const BSONObj& sequenceObject : sequence.objs) {', '}'): ++ ++ # Either we are deserializing BSON Objects or IDL structs ++ if field.struct_type: ++ self._writer.write_line('IDLParserErrorContext tempContext(%s, &ctxt);' % ++ (_get_field_constant_name(field))) ++ array_value = '%s::parse(tempContext, sequenceObject)' % ( ++ common.title_case(field.struct_type)) ++ else: ++ assert field.bson_serialization_type == ['object'] ++ if field.deserializer: ++ array_value = '%s(sequenceObject)' % (field.deserializer) ++ else: ++ array_value = "sequenceObject" ++ ++ self._writer.write_line('values.emplace_back(%s);' % (array_value)) ++ ++ self._writer.write_line('%s = std::move(values);' % (_get_field_member_name(field))) ++ ++ def gen_op_msg_request_namespace_check(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate a namespace check for a command.""" ++ # pylint: disable=invalid-name ++ if not isinstance(struct, ast.Command): ++ return ++ ++ with self._predicate("firstFieldFound == false"): ++ # Get the Command element if we need it for later in the deserializer to get the ++ # namespace ++ if struct.namespace != common.COMMAND_NAMESPACE_IGNORED: ++ self._writer.write_line('commandElement = element;') ++ ++ self._writer.write_line('firstFieldFound = true;') ++ self._writer.write_line('continue;') ++ ++ def gen_constructors(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the C++ constructor definition.""" ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ constructor = struct_type_info.get_constructor_method() ++ ++ initializers = ['_%s(std::move(%s))' % (arg.name, arg.name) for arg in constructor.args] ++ ++ # Serialize non-has fields first ++ # Initialize int and other primitive fields to -1 to prevent Coverity warnings. ++ for field in struct.fields: ++ needs_init = field.cpp_type and not field.array and cpp_types.is_primitive_scalar_type( ++ field.cpp_type) ++ if _is_required_serializer_field(field) and needs_init: ++ initializers.append( ++ '%s(%s)' % (_get_field_member_name(field), ++ cpp_types.get_primitive_scalar_type_default_value(field.cpp_type))) ++ ++ # Serialize the _dbName field second ++ initializes_db_name = False ++ if [arg for arg in constructor.args if arg.name == 'nss']: ++ initializers.append('_dbName(nss.db().toString())') ++ initializes_db_name = True ++ ++ # Serialize has fields third ++ # Add _has{FIELD} bool members to ensure fields are set before serialization. ++ for field in struct.fields: ++ if _is_required_serializer_field(field) and not (field.name == "$db" ++ and initializes_db_name): ++ initializers.append('%s(false)' % _get_has_field_member_name(field)) ++ ++ if initializes_db_name: ++ initializers.append('_hasDbName(true)') ++ ++ initializers_str = '' ++ if initializers: ++ initializers_str = ': ' + ', '.join(initializers) ++ ++ with self._block('%s %s {' % (constructor.get_definition(), initializers_str), '}'): ++ self._writer.write_line('// Used for initialization only') ++ ++ def _gen_command_deserializer(self, struct, bson_object): ++ # type: (ast.Struct, unicode) -> None ++ """Generate the command field deserializer.""" ++ ++ if isinstance(struct, ast.Command) and struct.command_field: ++ with self._block('{', '}'): ++ self.gen_field_deserializer(struct.command_field, bson_object, "commandElement") ++ else: ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ # Generate namespace check now that "$db" has been read or defaulted ++ struct_type_info.gen_namespace_check(self._writer, "_dbName", "commandElement") ++ ++ def _gen_fields_deserializer_common(self, struct, bson_object): ++ # type: (ast.Struct, unicode) -> _FieldUsageCheckerBase ++ """Generate the C++ code to deserialize list of fields.""" ++ # pylint: disable=too-many-branches ++ field_usage_check = _get_field_usage_checker(self._writer, struct) ++ if isinstance(struct, ast.Command): ++ self._writer.write_line('BSONElement commandElement;') ++ self._writer.write_line('bool firstFieldFound = false;') ++ ++ self._writer.write_empty_line() ++ ++ with self._block('for (const auto& element :%s) {' % (bson_object), '}'): ++ ++ self._writer.write_line('const auto fieldName = element.fieldNameStringData();') ++ self._writer.write_empty_line() ++ ++ if isinstance(struct, ast.Command): ++ with self._predicate("firstFieldFound == false"): ++ # Get the Command element if we need it for later in the deserializer to get the ++ # namespace ++ if struct.namespace != common.COMMAND_NAMESPACE_IGNORED: ++ self._writer.write_line('commandElement = element;') ++ ++ self._writer.write_line('firstFieldFound = true;') ++ self._writer.write_line('continue;') ++ ++ field_usage_check.add_store("fieldName") ++ self._writer.write_empty_line() ++ ++ first_field = True ++ for field in struct.fields: ++ # Do not parse chained fields as fields since they are actually chained types. ++ if field.chained and not field.chained_struct_field: ++ continue ++ ++ field_predicate = 'fieldName == %s' % (_get_field_constant_name(field)) ++ ++ with self._predicate(field_predicate, not first_field): ++ field_usage_check.add(field, "element") ++ ++ if field.ignore: ++ self._writer.write_line('// ignore field') ++ else: ++ if _is_required_serializer_field(field): ++ self._writer.write_line('%s = true;' % ++ (_get_has_field_member_name(field))) ++ ++ self.gen_field_deserializer(field, bson_object, "element") ++ ++ if first_field: ++ first_field = False ++ ++ # End of for fields ++ # Generate strict check for extranous fields ++ if struct.strict: ++ with self._block('else {', '}'): ++ # For commands, check if this a well known command field that the IDL parser ++ # should ignore regardless of strict mode. ++ command_predicate = None ++ if isinstance(struct, ast.Command): ++ command_predicate = "!mongo::isGenericArgument(fieldName)" ++ ++ with self._predicate(command_predicate): ++ self._writer.write_line('ctxt.throwUnknownField(fieldName);') ++ ++ # Parse chained structs if not inlined ++ # Parse chained types always here ++ for field in struct.fields: ++ if not field.chained or \ ++ (field.chained and field.struct_type and struct.inline_chained_structs): ++ continue ++ ++ # Simply generate deserializers since these are all 'any' types ++ self.gen_field_deserializer(field, bson_object, "element") ++ self._writer.write_empty_line() ++ ++ self._writer.write_empty_line() ++ ++ return field_usage_check ++ ++ def get_bson_deserializer_static_common(self, struct, static_method_info, method_info): ++ # type: (ast.Struct, struct_types.MethodInfo, struct_types.MethodInfo) -> None ++ """Generate the C++ deserializer static method.""" ++ # pylint: disable=invalid-name ++ func_def = static_method_info.get_definition() ++ ++ with self._block('%s {' % (func_def), '}'): ++ if isinstance(struct, ++ ast.Command) and struct.namespace != common.COMMAND_NAMESPACE_IGNORED: ++ if struct.namespace == common.COMMAND_NAMESPACE_TYPE: ++ cpp_type_info = cpp_types.get_cpp_type(struct.command_field) ++ ++ if struct.command_field.cpp_type and cpp_types.is_primitive_scalar_type( ++ struct.command_field.cpp_type): ++ self._writer.write_line('%s localCmdType(%s);' % ++ (cpp_type_info.get_storage_type(), ++ cpp_types.get_primitive_scalar_type_default_value( ++ struct.command_field.cpp_type))) ++ else: ++ self._writer.write_line('%s localCmdType;' % ++ (cpp_type_info.get_storage_type())) ++ self._writer.write_line('%s object(localCmdType);' % ++ (common.title_case(struct.cpp_name))) ++ elif struct.namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: ++ self._writer.write_line('NamespaceString localNS;') ++ self._writer.write_line('%s object(localNS);' % ++ (common.title_case(struct.cpp_name))) ++ else: ++ self._writer.write_line('%s object;' % common.title_case(struct.cpp_name)) ++ ++ self._writer.write_line(method_info.get_call('object')) ++ self._writer.write_line('return object;') ++ ++ def gen_bson_deserializer_methods(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the C++ deserializer method definitions.""" ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ self.get_bson_deserializer_static_common(struct, ++ struct_type_info.get_deserializer_static_method(), ++ struct_type_info.get_deserializer_method()) ++ ++ func_def = struct_type_info.get_deserializer_method().get_definition() ++ with self._block('%s {' % (func_def), '}'): ++ ++ # Deserialize all the fields ++ field_usage_check = self._gen_fields_deserializer_common(struct, "bsonObject") ++ ++ # Check for required fields ++ field_usage_check.add_final_checks() ++ self._writer.write_empty_line() ++ ++ self._gen_command_deserializer(struct, "bsonObject") ++ ++ def gen_op_msg_request_deserializer_methods(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the C++ deserializer method definitions from OpMsgRequest.""" ++ # pylint: disable=invalid-name ++ # Commands that have concatentate_with_db namespaces require db name as a parameter ++ if not isinstance(struct, ast.Command): ++ return ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ self.get_bson_deserializer_static_common( ++ struct, struct_type_info.get_op_msg_request_deserializer_static_method(), ++ struct_type_info.get_op_msg_request_deserializer_method()) ++ ++ func_def = struct_type_info.get_op_msg_request_deserializer_method().get_definition() ++ with self._block('%s {' % (func_def), '}'): ++ ++ # Deserialize all the fields ++ field_usage_check = self._gen_fields_deserializer_common(struct, "request.body") ++ ++ # Iterate through the document sequences if we have any ++ has_doc_sequence = len( ++ [field for field in struct.fields if field.supports_doc_sequence]) ++ if has_doc_sequence: ++ with self._block('for (auto&& sequence : request.sequences) {', '}'): ++ field_usage_check.add_store("sequence.name") ++ self._writer.write_empty_line() ++ ++ first_field = True ++ for field in struct.fields: ++ # Only parse document sequence fields here ++ if not field.supports_doc_sequence: ++ continue ++ ++ field_predicate = 'sequence.name == %s' % (_get_field_constant_name(field)) ++ ++ with self._predicate(field_predicate, not first_field): ++ field_usage_check.add(field, "sequence.name") ++ ++ if _is_required_serializer_field(field): ++ self._writer.write_line('%s = true;' % ++ (_get_has_field_member_name(field))) ++ ++ self.gen_doc_sequence_deserializer(field) ++ ++ if first_field: ++ first_field = False ++ ++ # End of for fields ++ # Generate strict check for extranous fields ++ if struct.strict: ++ with self._block('else {', '}'): ++ self._writer.write_line('ctxt.throwUnknownField(sequence.name);') ++ self._writer.write_empty_line() ++ ++ # Check for required fields ++ field_usage_check.add_final_checks() ++ self._writer.write_empty_line() ++ ++ self._gen_command_deserializer(struct, "request.body") ++ ++ def _gen_serializer_method_custom(self, field): ++ # type: (ast.Field) -> None ++ """Generate the serialize method definition for a custom type.""" ++ ++ # Generate custom serialization ++ template_params = { ++ 'field_name': _get_field_constant_name(field), ++ 'access_member': _access_member(field), ++ } ++ ++ with self._with_template(template_params): ++ # Is this a scalar bson C++ type? ++ bson_cpp_type = cpp_types.get_bson_cpp_type(field) ++ ++ # Object types need to go through the generic custom serialization code below ++ if bson_cpp_type and bson_cpp_type.has_serializer(): ++ if field.array: ++ self._writer.write_template( ++ 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') ++ with self._block('for (const auto& item : ${access_member}) {', '}'): ++ expression = bson_cpp_type.gen_serializer_expression(self._writer, 'item') ++ template_params['expression'] = expression ++ self._writer.write_template('arrayBuilder.append(${expression});') ++ else: ++ expression = bson_cpp_type.gen_serializer_expression( ++ self._writer, _access_member(field)) ++ template_params['expression'] = expression ++ self._writer.write_template('builder->append(${field_name}, ${expression});') ++ ++ elif field.bson_serialization_type[0] == 'any': ++ # Any types are special ++ # Array variants - we pass an array builder ++ # Non-array variants - we pass the field name they should use, and a BSONObjBuilder. ++ method_name = writer.get_method_name(field.serializer) ++ template_params['method_name'] = method_name ++ ++ if field.array: ++ self._writer.write_template( ++ 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') ++ with self._block('for (const auto& item : ${access_member}) {', '}'): ++ # Call a method like class::method(BSONArrayBuilder*) ++ self._writer.write_template('item.${method_name}(&arrayBuilder);') ++ else: ++ if writer.is_function(field.serializer): ++ # Call a method like method(value, StringData, BSONObjBuilder*) ++ self._writer.write_template( ++ '${method_name}(${access_member}, ${field_name}, builder);') ++ else: ++ # Call a method like class::method(StringData, BSONObjBuilder*) ++ self._writer.write_template( ++ '${access_member}.${method_name}(${field_name}, builder);') ++ ++ else: ++ method_name = writer.get_method_name(field.serializer) ++ template_params['method_name'] = method_name ++ ++ if field.array: ++ self._writer.write_template( ++ 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') ++ with self._block('for (const auto& item : ${access_member}) {', '}'): ++ self._writer.write_line( ++ 'BSONObjBuilder subObjBuilder(arrayBuilder.subobjStart());') ++ self._writer.write_template('item.${method_name}(&subObjBuilder);') ++ else: ++ self._writer.write_template('${access_member}.${method_name}(builder);') ++ ++ def _gen_serializer_method_struct(self, field): ++ # type: (ast.Field) -> None ++ """Generate the serialize method definition for a struct type.""" ++ ++ template_params = { ++ 'field_name': _get_field_constant_name(field), ++ 'access_member': _access_member(field), ++ } ++ ++ with self._with_template(template_params): ++ ++ if field.chained: ++ # Just directly call the serializer for chained structs without opening up a nested ++ # document. ++ self._writer.write_template('${access_member}.serialize(builder);') ++ elif field.array: ++ self._writer.write_template( ++ 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') ++ with self._block('for (const auto& item : ${access_member}) {', '}'): ++ self._writer.write_line( ++ 'BSONObjBuilder subObjBuilder(arrayBuilder.subobjStart());') ++ self._writer.write_line('item.serialize(&subObjBuilder);') ++ else: ++ self._writer.write_template( ++ 'BSONObjBuilder subObjBuilder(builder->subobjStart(${field_name}));') ++ self._writer.write_template('${access_member}.serialize(&subObjBuilder);') ++ ++ def _gen_serializer_method_common(self, field): ++ # type: (ast.Field) -> None ++ """Generate the serialize method definition.""" ++ member_name = _get_field_member_name(field) ++ ++ # Is this a scalar bson C++ type? ++ bson_cpp_type = cpp_types.get_bson_cpp_type(field) ++ ++ needs_custom_serializer = field.serializer or (bson_cpp_type ++ and bson_cpp_type.has_serializer()) ++ ++ optional_block_start = None ++ if field.optional: ++ optional_block_start = 'if (%s.is_initialized()) {' % (member_name) ++ elif field.struct_type or needs_custom_serializer or field.array: ++ # Introduce a new scope for required nested object serialization. ++ optional_block_start = '{' ++ ++ with self._block(optional_block_start, '}'): ++ ++ if not field.struct_type: ++ if needs_custom_serializer: ++ self._gen_serializer_method_custom(field) ++ else: ++ # Generate default serialization using BSONObjBuilder::append ++ # Note: BSONObjBuilder::append has overrides for std::vector also ++ self._writer.write_line( ++ 'builder->append(%s, %s);' % (_get_field_constant_name(field), ++ _access_member(field))) ++ else: ++ self._gen_serializer_method_struct(field) ++ ++ def _gen_serializer_methods_common(self, struct, is_op_msg_request): ++ # type: (ast.Struct, bool) -> None ++ """Generate the serialize method definition.""" ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ # Check all required fields have been specified ++ required_fields = [ ++ _get_has_field_member_name(field) for field in struct.fields ++ if _is_required_serializer_field(field) ++ ] ++ ++ if required_fields: ++ assert_fields_set = ' && '.join(required_fields) ++ self._writer.write_line('invariant(%s);' % assert_fields_set) ++ self._writer.write_empty_line() ++ ++ # Serialize the namespace as the first field ++ if isinstance(struct, ast.Command): ++ if struct.command_field: ++ self._gen_serializer_method_common(struct.command_field) ++ else: ++ struct_type_info = struct_types.get_struct_info(struct) ++ struct_type_info.gen_serializer(self._writer) ++ ++ for field in struct.fields: ++ # If fields are meant to be ignored during deserialization, there is no need to ++ # serialize. Ignored fields have no backing storage. ++ if field.ignore: ++ continue ++ ++ if field.chained_struct_field: ++ continue ++ ++ # The $db injected field should only be inject when serializing to OpMsgRequest. In the ++ # BSON case, it will be injected in the generic command layer. ++ if field.serialize_op_msg_request_only and not is_op_msg_request: ++ continue ++ ++ # Serialize fields that can be document sequence as document sequences so as not to ++ # generate the BSON body >= 16 MB. ++ if field.supports_doc_sequence and is_op_msg_request: ++ continue ++ ++ self._gen_serializer_method_common(field) ++ ++ # Add a blank line after each block ++ self._writer.write_empty_line() ++ ++ # Append passthrough elements ++ if isinstance(struct, ast.Command): ++ self._writer.write_line( ++ "IDLParserErrorContext::appendGenericCommandArguments(commandPassthroughFields, _knownFields, builder);" ++ ) ++ self._writer.write_empty_line() ++ ++ def gen_bson_serializer_method(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the serialize method definition.""" ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ with self._block('%s {' % (struct_type_info.get_serializer_method().get_definition()), '}'): ++ self._gen_serializer_methods_common(struct, False) ++ ++ def gen_to_bson_serializer_method(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the toBSON method definition.""" ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ with self._block('%s {' % (struct_type_info.get_to_bson_method().get_definition()), '}'): ++ self._writer.write_line('BSONObjBuilder builder;') ++ self._writer.write_line(struct_type_info.get_serializer_method().get_call(None).replace( ++ "builder", "&builder")) ++ self._writer.write_line('return builder.obj();') ++ ++ def _gen_doc_sequence_serializer(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the serialize method portion for fields which can be document sequence.""" ++ ++ for field in struct.fields: ++ if not field.supports_doc_sequence: ++ continue ++ ++ member_name = _get_field_member_name(field) ++ ++ optional_block_start = '{' ++ if field.optional: ++ optional_block_start = 'if (%s.is_initialized()) {' % (member_name) ++ ++ with self._block(optional_block_start, '}'): ++ self._writer.write_line('OpMsg::DocumentSequence documentSequence;') ++ self._writer.write_template('documentSequence.name = %s.toString();' % ++ (_get_field_constant_name(field))) ++ ++ with self._block('for (const auto& item : %s) {' % (_access_member(field)), '}'): ++ ++ if not field.struct_type: ++ if field.serializer: ++ self._writer.write_line('documentSequence.objs.push_back(item.%s());' % ++ (writer.get_method_name(field.serializer))) ++ else: ++ self._writer.write_line('documentSequence.objs.push_back(item);') ++ else: ++ self._writer.write_line('BSONObjBuilder builder;') ++ self._writer.write_line('item.serialize(&builder);') ++ self._writer.write_line('documentSequence.objs.push_back(builder.obj());') ++ ++ self._writer.write_template('request.sequences.emplace_back(documentSequence);') ++ ++ # Add a blank line after each block ++ self._writer.write_empty_line() ++ ++ def gen_op_msg_request_serializer_method(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the serialzer method definition for OpMsgRequest.""" ++ # pylint: disable=invalid-name ++ if not isinstance(struct, ast.Command): ++ return ++ ++ struct_type_info = struct_types.get_struct_info(struct) ++ ++ with self._block('%s {' % ++ (struct_type_info.get_op_msg_request_serializer_method().get_definition()), ++ '}'): ++ self._writer.write_line('BSONObjBuilder localBuilder;') ++ ++ with self._block('{', '}'): ++ self._writer.write_line('BSONObjBuilder* builder = &localBuilder;') ++ ++ self._gen_serializer_methods_common(struct, True) ++ ++ self._writer.write_line('OpMsgRequest request;') ++ self._writer.write_line('request.body = localBuilder.obj();') ++ ++ self._gen_doc_sequence_serializer(struct) ++ ++ self._writer.write_line('return request;') ++ ++ def gen_string_constants_definitions(self, struct): ++ # type: (ast.Struct) -> None ++ # pylint: disable=invalid-name ++ """Generate a StringData constant for field name in the cpp file.""" ++ ++ for field in _get_all_fields(struct): ++ self._writer.write_line( ++ common.template_args('constexpr StringData ${class_name}::${constant_name};', ++ class_name=common.title_case(struct.cpp_name), ++ constant_name=_get_field_constant_name(field))) ++ ++ if isinstance(struct, ast.Command): ++ self._writer.write_line( ++ common.template_args('constexpr StringData ${class_name}::kCommandName;', ++ class_name=common.title_case(struct.cpp_name))) ++ ++ def gen_enum_definition(self, idl_enum): ++ # type: (ast.Enum) -> None ++ """Generate the definitions for an enum's supporting functions.""" ++ enum_type_info = enum_types.get_type_info(idl_enum) ++ ++ enum_type_info.gen_deserializer_definition(self._writer) ++ self._writer.write_empty_line() ++ ++ enum_type_info.gen_serializer_definition(self._writer) ++ self._writer.write_empty_line() ++ ++ def gen_known_fields_declaration(self, struct): ++ # type: (ast.Struct) -> None ++ """Generate the known fields declaration.""" ++ if not isinstance(struct, ast.Command): ++ return ++ ++ block_name = common.template_args( ++ 'const std::vector ${class_name}::_knownFields {', ++ class_name=common.title_case(struct.cpp_name)) ++ with self._block(block_name, "};"): ++ sorted_fields = sorted([field for field in struct.fields], key=lambda f: f.cpp_name) ++ ++ for field in sorted_fields: ++ self._writer.write_line( ++ common.template_args( ++ '${class_name}::${constant_name},', class_name=common.title_case( ++ struct.cpp_name), constant_name=_get_field_constant_name(field))) ++ ++ self._writer.write_line( ++ common.template_args('${class_name}::kCommandName,', class_name=common.title_case( ++ struct.cpp_name))) ++ ++ def generate(self, spec, header_file_name): ++ # type: (ast.IDLAST, unicode) -> None ++ """Generate the C++ header to a stream.""" ++ self.gen_file_header() ++ ++ # Include platform/basic.h ++ self.gen_include("mongo/platform/basic.h") ++ self.write_empty_line() ++ ++ # Generate include for generated header first ++ self.gen_include(header_file_name) ++ self.write_empty_line() ++ ++ # Generate system includes second ++ header_list = [ ++ 'bitset', ++ 'set', ++ ] ++ ++ for include in header_list: ++ self.gen_system_include(include) ++ ++ self.write_empty_line() ++ ++ # Generate mongo includes third ++ header_list = [ ++ 'mongo/bson/bsonobjbuilder.h', ++ 'mongo/db/command_generic_argument.h', ++ 'mongo/db/commands.h', ++ ] ++ header_list.sort() ++ ++ for include in header_list: ++ self.gen_include(include) ++ ++ self.write_empty_line() ++ ++ # Generate namesapce ++ with self.gen_namespace_block(spec.globals.cpp_namespace): ++ self.write_empty_line() ++ ++ for idl_enum in spec.enums: ++ self.gen_description_comment(idl_enum.description) ++ self.gen_enum_definition(idl_enum) ++ ++ for struct in spec.structs: ++ self.gen_string_constants_definitions(struct) ++ self.write_empty_line() ++ ++ # Write known fields declaration for command ++ self.gen_known_fields_declaration(struct) ++ self.write_empty_line() ++ ++ # Write constructor ++ self.gen_constructors(struct) ++ self.write_empty_line() ++ ++ # Write deserializers ++ self.gen_bson_deserializer_methods(struct) ++ self.write_empty_line() ++ ++ self.gen_op_msg_request_deserializer_methods(struct) ++ self.write_empty_line() ++ ++ # Write serializer ++ self.gen_bson_serializer_method(struct) ++ self.write_empty_line() ++ ++ # Write OpMsgRequest serializer ++ self.gen_op_msg_request_serializer_method(struct) ++ self.write_empty_line() ++ ++ # Write toBSON ++ self.gen_to_bson_serializer_method(struct) ++ self.write_empty_line() ++ ++ ++def generate_header_str(spec): ++ # type: (ast.IDLAST) -> unicode ++ """Generate a C++ header in-memory.""" ++ stream = io.StringIO() ++ text_writer = writer.IndentedTextWriter(stream) ++ ++ header = _CppHeaderFileWriter(text_writer) ++ ++ header.generate(spec) ++ ++ return stream.getvalue() ++ ++ ++def _generate_header(spec, file_name): ++ # type: (ast.IDLAST, unicode) -> None ++ """Generate a C++ header.""" ++ ++ str_value = generate_header_str(spec) ++ ++ # Generate structs ++ with io.open(file_name, mode='wb') as file_handle: ++ file_handle.write(str_value.encode()) ++ ++ ++def generate_source_str(spec, target_arch, header_file_name): ++ # type: (ast.IDLAST, unicode, unicode) -> unicode ++ """Generate a C++ source file in-memory.""" ++ stream = io.StringIO() ++ text_writer = writer.IndentedTextWriter(stream) ++ ++ source = _CppSourceFileWriter(text_writer, target_arch) ++ ++ source.generate(spec, header_file_name) ++ ++ return stream.getvalue() ++ ++ ++def _generate_source(spec, target_arch, file_name, header_file_name): ++ # type: (ast.IDLAST, unicode, unicode, unicode) -> None ++ """Generate a C++ source file.""" ++ str_value = generate_source_str(spec, target_arch, header_file_name) ++ ++ # Generate structs ++ with io.open(file_name, mode='wb') as file_handle: ++ file_handle.write(str_value.encode()) ++ ++ ++def generate_code(spec, target_arch, output_base_dir, header_file_name, source_file_name): ++ # type: (ast.IDLAST, unicode, unicode, unicode, unicode) -> None ++ """Generate a C++ header and source file from an idl.ast tree.""" ++ ++ _generate_header(spec, header_file_name) ++ ++ if output_base_dir: ++ include_h_file_name = os.path.relpath( ++ os.path.normpath(header_file_name), os.path.normpath(output_base_dir)) ++ else: ++ include_h_file_name = os.path.abspath(os.path.normpath(header_file_name)) ++ ++ # Normalize to POSIX style for consistency across Windows and POSIX. ++ include_h_file_name = include_h_file_name.replace("\\", "/") ++ ++ _generate_source(spec, target_arch, source_file_name, include_h_file_name) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/parser.py mongo-r4.0.23/buildscripts/idl/idl/parser.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/parser.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/parser.py 2021-03-17 01:21:05.952000000 +0000 +@@ -44,6 +44,7 @@ from . import cpp_types + from . import errors + from . import syntax + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + + class _RuleDesc(object): + """ +@@ -562,11 +563,9 @@ def _parse(stream, error_file_name): + return syntax.IDLParsedSpec(spec, None) + + +-class ImportResolverBase(object): ++class ImportResolverBase(ABC): + """Base class for resolving imported files.""" + +- __metaclass__ = ABCMeta +- + def __init__(self): + # type: () -> None + """Construct a ImportResolver.""" +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/parser.py.orig mongo-r4.0.23/buildscripts/idl/idl/parser.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/parser.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/parser.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,669 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++""" ++IDL Parser. ++ ++Converts a YAML document to an idl.syntax tree. ++Only validates the document is syntatically correct, not semantically. ++""" ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from abc import ABCMeta, abstractmethod ++import io ++from typing import Any, Callable, Dict, List, Set, Tuple, Union ++import yaml ++from yaml import nodes ++ ++from . import common ++from . import cpp_types ++from . import errors ++from . import syntax ++ ++ ++class _RuleDesc(object): ++ """ ++ Describe a simple parser rule for the generic YAML node parser. ++ ++ node_type is either (scalar, bool_scalar, int_scalar, scalar_or_sequence, sequence, or mapping) ++ - bool_scalar - means a scalar node which is a valid bool, populates a bool ++ - int_scalar - means a scalar node which is a valid non-negative int, populates a int ++ - scalar_or_sequence - means a scalar or sequence node, populates a list ++ - sequence - a sequence node, populates a list ++ - mapping - a mapping node, calls another parser ++ mapping_parser_func is only called when parsing a mapping yaml node ++ """ ++ ++ # TODO: after porting to Python 3, use an enum ++ REQUIRED = 1 ++ OPTIONAL = 2 ++ ++ def __init__(self, node_type, required=OPTIONAL, mapping_parser_func=None): ++ # type: (unicode, int, Callable[[errors.ParserContext,yaml.nodes.MappingNode], Any]) -> None ++ """Construct a parser rule description.""" ++ assert required == _RuleDesc.REQUIRED or required == _RuleDesc.OPTIONAL ++ ++ self.node_type = node_type # type: unicode ++ self.required = required # type: int ++ self.mapping_parser_func = mapping_parser_func # type: Callable[[errors.ParserContext,yaml.nodes.MappingNode], Any] ++ ++ ++def _generic_parser( ++ ctxt, # type: errors.ParserContext ++ node, # type: Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode] ++ syntax_node_name, # type: unicode ++ syntax_node, # type: Any ++ mapping_rules # type: Dict[unicode, _RuleDesc] ++): # type: (...) -> None ++ # pylint: disable=too-many-branches ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in node.value: ++ ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ if first_name in mapping_rules: ++ rule_desc = mapping_rules[first_name] ++ ++ if rule_desc.node_type == "scalar": ++ if ctxt.is_scalar_node(second_node, first_name): ++ syntax_node.__dict__[first_name] = second_node.value ++ elif rule_desc.node_type == "bool_scalar": ++ if ctxt.is_scalar_bool_node(second_node, first_name): ++ syntax_node.__dict__[first_name] = ctxt.get_bool(second_node) ++ elif rule_desc.node_type == "int_scalar": ++ if ctxt.is_scalar_non_negative_int_node(second_node, first_name): ++ syntax_node.__dict__[first_name] = ctxt.get_non_negative_int(second_node) ++ elif rule_desc.node_type == "scalar_or_sequence": ++ if ctxt.is_scalar_sequence_or_scalar_node(second_node, first_name): ++ syntax_node.__dict__[first_name] = ctxt.get_list(second_node) ++ elif rule_desc.node_type == "sequence": ++ if ctxt.is_scalar_sequence(second_node, first_name): ++ syntax_node.__dict__[first_name] = ctxt.get_list(second_node) ++ elif rule_desc.node_type == "mapping": ++ if ctxt.is_mapping_node(second_node, first_name): ++ syntax_node.__dict__[first_name] = rule_desc.mapping_parser_func( ++ ctxt, second_node) ++ else: ++ raise errors.IDLError("Unknown node_type '%s' for parser rule" % ++ (rule_desc.node_type)) ++ else: ++ ctxt.add_unknown_node_error(first_node, syntax_node_name) ++ ++ field_name_set.add(first_name) ++ ++ # Check for any missing required fields ++ for name, rule_desc in mapping_rules.items(): ++ if not rule_desc.required == _RuleDesc.REQUIRED: ++ continue ++ ++ # A bool is never "None" like other types, it simply defaults to "false". ++ # It means "if bool is None" will always return false and there is no support for required ++ # 'bool' at this time. ++ if not rule_desc.node_type == 'bool_scalar': ++ if syntax_node.__dict__[name] is None: ++ ctxt.add_missing_required_field_error(node, syntax_node_name, name) ++ else: ++ raise errors.IDLError("Unknown node_type '%s' for parser required rule" % ++ (rule_desc.node_type)) ++ ++ ++def _parse_mapping( ++ ctxt, # type: errors.ParserContext ++ spec, # type: syntax.IDLSpec ++ node, # type: Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode] ++ syntax_node_name, # type: unicode ++ func # type: Callable[[errors.ParserContext,syntax.IDLSpec,unicode,Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]], None] ++): # type: (...) -> None ++ """Parse a top-level mapping section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, syntax_node_name): ++ return ++ ++ for [first_node, second_node] in node.value: ++ ++ first_name = first_node.value ++ ++ func(ctxt, spec, first_name, second_node) ++ ++ ++def _parse_global(ctxt, spec, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse a global section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, "global"): ++ return ++ ++ idlglobal = syntax.Global(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ ++ _generic_parser(ctxt, node, "global", idlglobal, { ++ "cpp_namespace": _RuleDesc("scalar"), ++ "cpp_includes": _RuleDesc("scalar_or_sequence"), ++ }) ++ ++ spec.globals = idlglobal ++ ++ ++def _parse_imports(ctxt, spec, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse an imports section in the IDL file.""" ++ if not ctxt.is_scalar_sequence(node, "imports"): ++ return ++ ++ imports = syntax.Import(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ imports.imports = ctxt.get_list(node) ++ spec.imports = imports ++ ++ ++def _parse_type(ctxt, spec, name, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse a type section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, "type"): ++ return ++ ++ idltype = syntax.Type(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ idltype.name = name ++ ++ _generic_parser( ++ ctxt, node, "type", idltype, { ++ "description": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "cpp_type": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "bson_serialization_type": _RuleDesc('scalar_or_sequence', _RuleDesc.REQUIRED), ++ "bindata_subtype": _RuleDesc('scalar'), ++ "serializer": _RuleDesc('scalar'), ++ "deserializer": _RuleDesc('scalar'), ++ "default": _RuleDesc('scalar'), ++ }) ++ ++ spec.symbols.add_type(ctxt, idltype) ++ ++ ++def _parse_field(ctxt, name, node): ++ # type: (errors.ParserContext, str, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> syntax.Field ++ """Parse a field in a struct/command in the IDL file.""" ++ field = syntax.Field(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ field.name = name ++ ++ _generic_parser( ++ ctxt, node, "field", field, { ++ "description": _RuleDesc('scalar'), ++ "cpp_name": _RuleDesc('scalar'), ++ "type": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "ignore": _RuleDesc("bool_scalar"), ++ "optional": _RuleDesc("bool_scalar"), ++ "default": _RuleDesc('scalar'), ++ "supports_doc_sequence": _RuleDesc("bool_scalar"), ++ "comparison_order": _RuleDesc("int_scalar"), ++ "non_const_getter": _RuleDesc("bool_scalar"), ++ }) ++ ++ return field ++ ++ ++def _parse_fields(ctxt, node): ++ # type: (errors.ParserContext, yaml.nodes.MappingNode) -> List[syntax.Field] ++ """Parse a fields section in a struct in the IDL file.""" ++ ++ fields = [] ++ ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in node.value: ++ ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ # Simple Type ++ if second_node.id == "scalar": ++ field = syntax.Field(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ field.name = first_name ++ field.type = second_node.value ++ fields.append(field) ++ else: ++ field = _parse_field(ctxt, first_name, second_node) ++ fields.append(field) ++ ++ field_name_set.add(first_name) ++ ++ return fields ++ ++ ++def _parse_chained_type(ctxt, name, node): ++ # type: (errors.ParserContext, str, yaml.nodes.MappingNode) -> syntax.ChainedType ++ """Parse a chained type in a struct in the IDL file.""" ++ chain = syntax.ChainedType(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ chain.name = name ++ ++ _generic_parser(ctxt, node, "chain", chain, { ++ "cpp_name": _RuleDesc('scalar'), ++ }) ++ ++ return chain ++ ++ ++def _parse_chained_types(ctxt, node): ++ # type: (errors.ParserContext, yaml.nodes.MappingNode) -> List[syntax.ChainedType] ++ """Parse a chained types section in a struct in the IDL file.""" ++ chained_items = [] ++ ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in node.value: ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ # Simple Scalar ++ if second_node.id == "scalar": ++ chain = syntax.ChainedType(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ chain.name = first_name ++ chain.cpp_name = second_node.value ++ chained_items.append(chain) ++ else: ++ chain = _parse_chained_type(ctxt, first_name, second_node) ++ chained_items.append(chain) ++ ++ field_name_set.add(first_name) ++ ++ return chained_items ++ ++ ++def _parse_chained_struct(ctxt, name, node): ++ # type: (errors.ParserContext, str, yaml.nodes.MappingNode) -> syntax.ChainedStruct ++ """Parse a chained struct in a struct in the IDL file.""" ++ chain = syntax.ChainedStruct(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ chain.name = name ++ ++ _generic_parser(ctxt, node, "chain", chain, { ++ "cpp_name": _RuleDesc('scalar'), ++ }) ++ ++ return chain ++ ++ ++def _parse_chained_structs(ctxt, node): ++ # type: (errors.ParserContext, yaml.nodes.MappingNode) -> List[syntax.ChainedStruct] ++ """Parse a chained structs in a struct in the IDL file.""" ++ chained_items = [] ++ ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in node.value: ++ ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ # Simple Scalar ++ if second_node.id == "scalar": ++ chain = syntax.ChainedStruct(ctxt.file_name, node.start_mark.line, ++ node.start_mark.column) ++ chain.name = first_name ++ chain.cpp_name = second_node.value ++ chained_items.append(chain) ++ else: ++ chain = _parse_chained_struct(ctxt, first_name, second_node) ++ chained_items.append(chain) ++ ++ field_name_set.add(first_name) ++ ++ return chained_items ++ ++ ++def _parse_struct(ctxt, spec, name, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse a struct section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, "struct"): ++ return ++ ++ struct = syntax.Struct(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ struct.name = name ++ ++ _generic_parser( ++ ctxt, node, "struct", struct, { ++ "description": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "fields": _RuleDesc('mapping', mapping_parser_func=_parse_fields), ++ "chained_types": _RuleDesc('mapping', mapping_parser_func=_parse_chained_types), ++ "chained_structs": _RuleDesc('mapping', mapping_parser_func=_parse_chained_structs), ++ "strict": _RuleDesc("bool_scalar"), ++ "inline_chained_structs": _RuleDesc("bool_scalar"), ++ "immutable": _RuleDesc('bool_scalar'), ++ "generate_comparison_operators": _RuleDesc("bool_scalar"), ++ }) ++ ++ # TODO: SHOULD WE ALLOW STRUCTS ONLY WITH CHAINED STUFF and no fields??? ++ if struct.fields is None and struct.chained_types is None and struct.chained_structs is None: ++ ctxt.add_empty_struct_error(node, struct.name) ++ ++ spec.symbols.add_struct(ctxt, struct) ++ ++ ++def _parse_enum_values(ctxt, node): ++ # type: (errors.ParserContext, yaml.nodes.MappingNode) -> List[syntax.EnumValue] ++ """Parse a values section in an enum in the IDL file.""" ++ ++ enum_values = [] ++ ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in node.value: ++ ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ # Simple Type ++ if ctxt.is_scalar_node(second_node, first_name): ++ enum_value = syntax.EnumValue(ctxt.file_name, node.start_mark.line, ++ node.start_mark.column) ++ enum_value.name = first_name ++ enum_value.value = second_node.value ++ enum_values.append(enum_value) ++ ++ field_name_set.add(first_name) ++ ++ return enum_values ++ ++ ++def _parse_enum(ctxt, spec, name, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse an enum section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, "struct"): ++ return ++ ++ idl_enum = syntax.Enum(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ idl_enum.name = name ++ ++ _generic_parser( ++ ctxt, node, "enum", idl_enum, { ++ "description": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "type": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "values": _RuleDesc('mapping', mapping_parser_func=_parse_enum_values), ++ }) ++ ++ if idl_enum.values is None: ++ ctxt.add_empty_enum_error(node, idl_enum.name) ++ ++ spec.symbols.add_enum(ctxt, idl_enum) ++ ++ ++def _parse_command(ctxt, spec, name, node): ++ # type: (errors.ParserContext, syntax.IDLSpec, unicode, Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> None ++ """Parse a command section in the IDL file.""" ++ if not ctxt.is_mapping_node(node, "command"): ++ return ++ ++ command = syntax.Command(ctxt.file_name, node.start_mark.line, node.start_mark.column) ++ command.name = name ++ ++ _generic_parser( ++ ctxt, node, "command", command, { ++ "description": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "chained_types": _RuleDesc('mapping', mapping_parser_func=_parse_chained_types), ++ "chained_structs": _RuleDesc('mapping', mapping_parser_func=_parse_chained_structs), ++ "fields": _RuleDesc('mapping', mapping_parser_func=_parse_fields), ++ "namespace": _RuleDesc('scalar', _RuleDesc.REQUIRED), ++ "cpp_name": _RuleDesc('scalar'), ++ "type": _RuleDesc('scalar'), ++ "strict": _RuleDesc("bool_scalar"), ++ "inline_chained_structs": _RuleDesc("bool_scalar"), ++ "immutable": _RuleDesc('bool_scalar'), ++ "generate_comparison_operators": _RuleDesc("bool_scalar"), ++ }) ++ ++ # TODO: support the first argument as UUID depending on outcome of Catalog Versioning changes. ++ valid_commands = [ ++ common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB, common.COMMAND_NAMESPACE_IGNORED, ++ common.COMMAND_NAMESPACE_TYPE ++ ] ++ ++ if command.namespace: ++ if command.namespace not in valid_commands: ++ ctxt.add_bad_command_namespace_error(command, command.name, command.namespace, ++ valid_commands) ++ ++ # type property must be specified for a namespace = type ++ if command.namespace == common.COMMAND_NAMESPACE_TYPE and not command.type: ++ ctxt.add_missing_required_field_error(node, "command", "type") ++ ++ if command.namespace != common.COMMAND_NAMESPACE_TYPE and command.type: ++ ctxt.add_extranous_command_type(command, command.name) ++ ++ # Commands may only have the first parameter, ensure the fields property is an empty array. ++ if not command.fields: ++ command.fields = [] ++ ++ spec.symbols.add_command(ctxt, command) ++ ++ ++def _prefix_with_namespace(cpp_namespace, cpp_name): ++ # type: (unicode, unicode) -> unicode ++ """Preface a C++ type name with a namespace if not already qualified or a primitive type.""" ++ if "::" in cpp_name or cpp_types.is_primitive_scalar_type(cpp_name): ++ return cpp_name ++ ++ return cpp_namespace + "::" + cpp_name ++ ++ ++def _propagate_globals(spec): ++ # type: (syntax.IDLSpec) -> None ++ """Propagate the globals information to each type and struct as needed.""" ++ if not spec.globals or not spec.globals.cpp_namespace: ++ return ++ ++ cpp_namespace = spec.globals.cpp_namespace ++ ++ for struct in spec.symbols.structs: ++ struct.cpp_namespace = cpp_namespace ++ ++ for command in spec.symbols.commands: ++ command.cpp_namespace = cpp_namespace ++ ++ for idlenum in spec.symbols.enums: ++ idlenum.cpp_namespace = cpp_namespace ++ ++ for idltype in spec.symbols.types: ++ idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type) ++ ++ ++def _parse(stream, error_file_name): ++ # type: (Any, unicode) -> syntax.IDLParsedSpec ++ """ ++ Parse a YAML document into an idl.syntax tree. ++ ++ stream: is a io.Stream. ++ error_file_name: just a file name for error messages to use. ++ """ ++ # pylint: disable=too-many-branches ++ ++ # This will raise an exception if the YAML parse fails ++ root_node = yaml.compose(stream) ++ ++ ctxt = errors.ParserContext(error_file_name, errors.ParserErrorCollection()) ++ ++ spec = syntax.IDLSpec() ++ ++ # If the document is empty, we are done ++ if not root_node: ++ return syntax.IDLParsedSpec(spec, None) ++ ++ if not root_node.id == "mapping": ++ raise errors.IDLError( ++ "Expected a YAML mapping node as root node of IDL document, got '%s' instead" % ++ root_node.id) ++ ++ field_name_set = set() # type: Set[str] ++ ++ for [first_node, second_node] in root_node.value: ++ ++ first_name = first_node.value ++ ++ if first_name in field_name_set: ++ ctxt.add_duplicate_error(first_node, first_name) ++ continue ++ ++ if first_name == "global": ++ _parse_global(ctxt, spec, second_node) ++ elif first_name == "imports": ++ _parse_imports(ctxt, spec, second_node) ++ elif first_name == "enums": ++ _parse_mapping(ctxt, spec, second_node, 'enums', _parse_enum) ++ elif first_name == "types": ++ _parse_mapping(ctxt, spec, second_node, 'types', _parse_type) ++ elif first_name == "structs": ++ _parse_mapping(ctxt, spec, second_node, 'structs', _parse_struct) ++ elif first_name == "commands": ++ _parse_mapping(ctxt, spec, second_node, 'commands', _parse_command) ++ else: ++ ctxt.add_unknown_root_node_error(first_node) ++ ++ field_name_set.add(first_name) ++ ++ if ctxt.errors.has_errors(): ++ return syntax.IDLParsedSpec(None, ctxt.errors) ++ ++ _propagate_globals(spec) ++ ++ return syntax.IDLParsedSpec(spec, None) ++ ++ ++class ImportResolverBase(object): ++ """Base class for resolving imported files.""" ++ ++ __metaclass__ = ABCMeta ++ ++ def __init__(self): ++ # type: () -> None ++ """Construct a ImportResolver.""" ++ pass ++ ++ @abstractmethod ++ def resolve(self, base_file, imported_file_name): ++ # type: (unicode, unicode) -> unicode ++ """Return the complete path to an imported file name.""" ++ pass ++ ++ @abstractmethod ++ def open(self, resolved_file_name): ++ # type: (unicode) -> Any ++ """Return an io.Stream for the requested file.""" ++ pass ++ ++ ++def parse(stream, input_file_name, resolver): ++ # type: (Any, unicode, ImportResolverBase) -> syntax.IDLParsedSpec ++ """ ++ Parse a YAML document into an idl.syntax tree. ++ ++ stream: is a io.Stream. ++ input_file_name: a file name for error messages to use, and to help resolve imported files. ++ """ ++ # pylint: disable=too-many-locals ++ ++ root_doc = _parse(stream, input_file_name) ++ ++ if root_doc.errors: ++ return root_doc ++ ++ imports = [] # type: List[Tuple[common.SourceLocation, unicode, unicode]] ++ needs_include = [] # type: List[unicode] ++ if root_doc.spec.imports: ++ imports = [(root_doc.spec.imports, input_file_name, import_file_name) ++ for import_file_name in root_doc.spec.imports.imports] ++ ++ resolved_file_names = [] # type: List[unicode] ++ ++ ctxt = errors.ParserContext(input_file_name, errors.ParserErrorCollection()) ++ ++ # Process imports in a breadth-first search ++ while imports: ++ file_import_tuple = imports[0] ++ imports = imports[1:] ++ ++ import_location = file_import_tuple[0] ++ base_file_name = file_import_tuple[1] ++ imported_file_name = file_import_tuple[2] ++ ++ # Check for already resolved file ++ resolved_file_name = resolver.resolve(base_file_name, imported_file_name) ++ if not resolved_file_name: ++ ctxt.add_cannot_find_import(import_location, imported_file_name) ++ return syntax.IDLParsedSpec(None, ctxt.errors) ++ ++ if resolved_file_name in resolved_file_names: ++ continue ++ ++ resolved_file_names.append(resolved_file_name) ++ ++ # Parse imported file ++ with resolver.open(resolved_file_name) as file_stream: ++ parsed_doc = _parse(file_stream, resolved_file_name) ++ ++ # Check for errors ++ if parsed_doc.errors: ++ return parsed_doc ++ ++ # We need to generate includes for imported IDL files which have structs ++ if base_file_name == input_file_name and parsed_doc.spec.symbols.structs: ++ needs_include.append(imported_file_name) ++ ++ # Add other imported files to the list of files to parse ++ if parsed_doc.spec.imports: ++ imports += [(parsed_doc.spec.imports, resolved_file_name, import_file_name) ++ for import_file_name in parsed_doc.spec.imports.imports] ++ ++ # Merge cpp_includes as needed ++ if parsed_doc.spec.globals and parsed_doc.spec.globals.cpp_includes: ++ root_doc.spec.globals.cpp_includes = list( ++ set(root_doc.spec.globals.cpp_includes + parsed_doc.spec.globals.cpp_includes)) ++ ++ # Merge symbol tables together ++ root_doc.spec.symbols.add_imported_symbol_table(ctxt, parsed_doc.spec.symbols) ++ if ctxt.errors.has_errors(): ++ return syntax.IDLParsedSpec(None, ctxt.errors) ++ ++ # Resolve the direct imports which contain structs for root document so they can be translated ++ # into include file paths in generated code. ++ for needs_include_name in needs_include: ++ resolved_file_name = resolver.resolve(base_file_name, needs_include_name) ++ root_doc.spec.imports.resolved_imports.append(resolved_file_name) ++ ++ if root_doc.spec.imports: ++ root_doc.spec.imports.dependencies = resolved_file_names ++ ++ return root_doc +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/struct_types.py mongo-r4.0.23/buildscripts/idl/idl/struct_types.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/struct_types.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/struct_types.py 2021-03-17 01:21:05.952000000 +0000 +@@ -37,6 +37,7 @@ from . import common + from . import cpp_types + from . import writer + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + + class ArgumentInfo(object): + """Class that encapsulates information about an argument to a method.""" +@@ -127,11 +128,9 @@ class MethodInfo(object): + args=args) + + +-class StructTypeInfoBase(object): ++class StructTypeInfoBase(ABC): + """Base class for struct and command code generation.""" + +- __metaclass__ = ABCMeta +- + @abstractmethod + def get_constructor_method(self): + # type: () -> MethodInfo +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/struct_types.py.orig mongo-r4.0.23/buildscripts/idl/idl/struct_types.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/struct_types.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/struct_types.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,466 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++"""Provide code generation information for structs and commands in a polymorphic way.""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++from abc import ABCMeta, abstractmethod ++from typing import Optional, List ++ ++from . import ast ++from . import common ++from . import cpp_types ++from . import writer ++ ++ ++class ArgumentInfo(object): ++ """Class that encapsulates information about an argument to a method.""" ++ ++ def __init__(self, arg): ++ # type: (unicode) -> None ++ """Create a instance of the ArgumentInfo class by parsing the argument string.""" ++ parts = arg.split(' ') ++ self.type = ' '.join(parts[0:-1]) ++ self.name = parts[-1] ++ ++ def __str__(self): ++ # type: () -> str ++ """Return a formatted argument string.""" ++ return "%s %s" % (self.type, self.name) # type: ignore ++ ++ ++class MethodInfo(object): ++ """Class that encapslates information about a method and how to declare, define, and call it.""" ++ ++ def __init__(self, class_name, method_name, args, return_type=None, static=False, const=False, ++ explicit=False): ++ # type: (unicode, unicode, List[unicode], unicode, bool, bool, bool) -> None ++ # pylint: disable=too-many-arguments ++ """Create a MethodInfo instance.""" ++ self.class_name = class_name ++ self.method_name = method_name ++ self.args = [ArgumentInfo(arg) for arg in args] ++ self.return_type = return_type ++ self.static = static ++ self.const = const ++ self.explicit = explicit ++ ++ def get_declaration(self): ++ # type: () -> unicode ++ """Get a declaration for a method.""" ++ pre_modifiers = '' ++ post_modifiers = '' ++ return_type_str = '' ++ ++ if self.static: ++ pre_modifiers = 'static ' ++ ++ if self.const: ++ post_modifiers = ' const' ++ ++ if self.explicit: ++ pre_modifiers += 'explicit ' ++ ++ if self.return_type: ++ return_type_str = self.return_type + ' ' ++ ++ return common.template_args( ++ "${pre_modifiers}${return_type}${method_name}(${args})${post_modifiers};", ++ pre_modifiers=pre_modifiers, return_type=return_type_str, method_name=self.method_name, ++ args=', '.join([str(arg) for arg in self.args]), post_modifiers=post_modifiers) ++ ++ def get_definition(self): ++ # type: () -> unicode ++ """Get a definition for a method.""" ++ pre_modifiers = '' ++ post_modifiers = '' ++ return_type_str = '' ++ ++ if self.const: ++ post_modifiers = ' const' ++ ++ if self.return_type: ++ return_type_str = self.return_type + ' ' ++ ++ return common.template_args( ++ "${pre_modifiers}${return_type}${class_name}::${method_name}(${args})${post_modifiers}", ++ pre_modifiers=pre_modifiers, return_type=return_type_str, class_name=self.class_name, ++ method_name=self.method_name, args=', '.join( ++ [str(arg) for arg in self.args]), post_modifiers=post_modifiers) ++ ++ def get_call(self, obj): ++ # type: (Optional[unicode]) -> unicode ++ """Generate a simply call to the method using the defined args list.""" ++ ++ args = ', '.join([arg.name for arg in self.args]) ++ ++ if obj: ++ return common.template_args("${obj}.${method_name}(${args});", obj=obj, ++ method_name=self.method_name, args=args) ++ ++ return common.template_args("${method_name}(${args});", method_name=self.method_name, ++ args=args) ++ ++ ++class StructTypeInfoBase(object): ++ """Base class for struct and command code generation.""" ++ ++ __metaclass__ = ABCMeta ++ ++ @abstractmethod ++ def get_constructor_method(self): ++ # type: () -> MethodInfo ++ """Get the constructor method for a struct.""" ++ pass ++ ++ @abstractmethod ++ def get_serializer_method(self): ++ # type: () -> MethodInfo ++ """Get the serializer method for a struct.""" ++ pass ++ ++ @abstractmethod ++ def get_to_bson_method(self): ++ # type: () -> MethodInfo ++ """Get the to_bson method for a struct.""" ++ pass ++ ++ @abstractmethod ++ def get_deserializer_static_method(self): ++ # type: () -> MethodInfo ++ """Get the public static deserializer method for a struct.""" ++ pass ++ ++ @abstractmethod ++ def get_deserializer_method(self): ++ # type: () -> MethodInfo ++ """Get the protected deserializer method for a struct.""" ++ pass ++ ++ @abstractmethod ++ def get_op_msg_request_serializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ """Get the OpMsg serializer method for a struct.""" ++ # pylint: disable=invalid-name ++ pass ++ ++ @abstractmethod ++ def get_op_msg_request_deserializer_static_method(self): ++ # type: () -> Optional[MethodInfo] ++ """Get the public static OpMsg deserializer method for a struct.""" ++ # pylint: disable=invalid-name ++ pass ++ ++ @abstractmethod ++ def get_op_msg_request_deserializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ """Get the protected OpMsg deserializer method for a struct.""" ++ # pylint: disable=invalid-name ++ pass ++ ++ @abstractmethod ++ def gen_getter_method(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the additional methods for a class.""" ++ pass ++ ++ @abstractmethod ++ def gen_member(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Generate the additional members for a class.""" ++ pass ++ ++ @abstractmethod ++ def gen_serializer(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ """Serialize the first field of a Command.""" ++ pass ++ ++ @abstractmethod ++ def gen_namespace_check(self, indented_writer, db_name, element): ++ # type: (writer.IndentedTextWriter, unicode, unicode) -> None ++ """Generate the namespace check predicate for a command.""" ++ pass ++ ++ ++class _StructTypeInfo(StructTypeInfoBase): ++ """Class for struct code generation.""" ++ ++ def __init__(self, struct): ++ # type: (ast.Struct) -> None ++ """Create a _StructTypeInfo instance.""" ++ self._struct = struct ++ ++ def get_constructor_method(self): ++ # type: () -> MethodInfo ++ class_name = common.title_case(self._struct.cpp_name) ++ return MethodInfo(class_name, class_name, []) ++ ++ def get_deserializer_static_method(self): ++ # type: () -> MethodInfo ++ class_name = common.title_case(self._struct.cpp_name) ++ return MethodInfo(class_name, 'parse', ++ ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], ++ class_name, static=True) ++ ++ def get_deserializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'parseProtected', ++ ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') ++ ++ def get_serializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'serialize', ['BSONObjBuilder* builder'], ++ 'void', const=True) ++ ++ def get_to_bson_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'toBSON', [], 'BSONObj', const=True) ++ ++ def get_op_msg_request_serializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ return None ++ ++ def get_op_msg_request_deserializer_static_method(self): ++ # type: () -> Optional[MethodInfo] ++ return None ++ ++ def get_op_msg_request_deserializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ return None ++ ++ def gen_getter_method(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ pass ++ ++ def gen_member(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ pass ++ ++ def gen_serializer(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ pass ++ ++ def gen_namespace_check(self, indented_writer, db_name, element): ++ # type: (writer.IndentedTextWriter, unicode, unicode) -> None ++ pass ++ ++ ++class _CommandBaseTypeInfo(_StructTypeInfo): ++ """Base class for command code generation.""" ++ ++ def __init__(self, command): ++ # type: (ast.Command) -> None ++ """Create a _CommandBaseTypeInfo instance.""" ++ self._command = command ++ ++ super(_CommandBaseTypeInfo, self).__init__(command) ++ ++ def get_op_msg_request_serializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'serialize', ++ ['const BSONObj& commandPassthroughFields'], 'OpMsgRequest', const=True) ++ ++ def get_op_msg_request_deserializer_static_method(self): ++ # type: () -> Optional[MethodInfo] ++ class_name = common.title_case(self._struct.cpp_name) ++ return MethodInfo(class_name, 'parse', ++ ['const IDLParserErrorContext& ctxt', 'const OpMsgRequest& request'], ++ class_name, static=True) ++ ++ def get_op_msg_request_deserializer_method(self): ++ # type: () -> Optional[MethodInfo] ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'parseProtected', ++ ['const IDLParserErrorContext& ctxt', 'const OpMsgRequest& request'], 'void') ++ ++ ++class _IgnoredCommandTypeInfo(_CommandBaseTypeInfo): ++ """Class for command code generation.""" ++ ++ def __init__(self, command): ++ # type: (ast.Command) -> None ++ """Create a _IgnoredCommandTypeInfo instance.""" ++ self._command = command ++ ++ super(_IgnoredCommandTypeInfo, self).__init__(command) ++ ++ def get_serializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'serialize', ++ ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], 'void', ++ const=True) ++ ++ def get_to_bson_method(self): ++ # type: () -> MethodInfo ++ # Commands that require namespaces require it as a parameter to serialize() ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'toBSON', ++ ['const BSONObj& commandPassthroughFields'], 'BSONObj', const=True) ++ ++ def gen_serializer(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ indented_writer.write_line('builder->append("%s", 1);' % (self._command.name)) ++ ++ def gen_namespace_check(self, indented_writer, db_name, element): ++ # type: (writer.IndentedTextWriter, unicode, unicode) -> None ++ pass ++ ++ ++class _CommandFromType(_CommandBaseTypeInfo): ++ """Class for command code generation for custom type.""" ++ ++ def __init__(self, command): ++ # type: (ast.Command) -> None ++ """Create a _CommandFromType instance.""" ++ assert command.command_field ++ self._command = command ++ super(_CommandFromType, self).__init__(command) ++ ++ def get_constructor_method(self): ++ # type: () -> MethodInfo ++ cpp_type_info = cpp_types.get_cpp_type(self._command.command_field) ++ # Use the storage type for the constructor argument since the generated code will use ++ # std::move. ++ member_type = cpp_type_info.get_storage_type() ++ ++ class_name = common.title_case(self._struct.cpp_name) ++ ++ arg = "const %s %s" % (member_type, common.camel_case(self._command.command_field.cpp_name)) ++ return MethodInfo(class_name, class_name, [arg], explicit=True) ++ ++ def get_serializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'serialize', ++ ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], 'void', ++ const=True) ++ ++ def get_to_bson_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'toBSON', ++ ['const BSONObj& commandPassthroughFields'], 'BSONObj', const=True) ++ ++ def get_deserializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'parseProtected', ++ ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') ++ ++ def gen_getter_method(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ raise NotImplementedError ++ ++ def gen_member(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ raise NotImplementedError ++ ++ def gen_serializer(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ raise NotImplementedError ++ ++ def gen_namespace_check(self, indented_writer, db_name, element): ++ # type: (writer.IndentedTextWriter, unicode, unicode) -> None ++ # TODO: should the name of the first element be validated?? ++ raise NotImplementedError ++ ++ ++class _CommandWithNamespaceTypeInfo(_CommandBaseTypeInfo): ++ """Class for command code generation.""" ++ ++ def __init__(self, command): ++ # type: (ast.Command) -> None ++ """Create a _CommandWithNamespaceTypeInfo instance.""" ++ self._command = command ++ ++ super(_CommandWithNamespaceTypeInfo, self).__init__(command) ++ ++ def get_constructor_method(self): ++ # type: () -> MethodInfo ++ class_name = common.title_case(self._struct.cpp_name) ++ return MethodInfo(class_name, class_name, ['const NamespaceString nss'], explicit=True) ++ ++ def get_serializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'serialize', ++ ['const BSONObj& commandPassthroughFields', 'BSONObjBuilder* builder'], 'void', ++ const=True) ++ ++ def get_to_bson_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'toBSON', ++ ['const BSONObj& commandPassthroughFields'], 'BSONObj', const=True) ++ ++ def get_deserializer_method(self): ++ # type: () -> MethodInfo ++ return MethodInfo( ++ common.title_case(self._struct.cpp_name), 'parseProtected', ++ ['const IDLParserErrorContext& ctxt', 'const BSONObj& bsonObject'], 'void') ++ ++ def gen_getter_method(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ indented_writer.write_line('const NamespaceString& getNamespace() const { return _nss; }') ++ ++ def gen_member(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ indented_writer.write_line('NamespaceString _nss;') ++ ++ def gen_serializer(self, indented_writer): ++ # type: (writer.IndentedTextWriter) -> None ++ indented_writer.write_line('invariant(!_nss.isEmpty());') ++ indented_writer.write_line('builder->append("%s", _nss.coll());' % (self._command.name)) ++ indented_writer.write_empty_line() ++ ++ def gen_namespace_check(self, indented_writer, db_name, element): ++ # type: (writer.IndentedTextWriter, unicode, unicode) -> None ++ # TODO: should the name of the first element be validated?? ++ indented_writer.write_line('invariant(_nss.isEmpty());') ++ indented_writer.write_line('_nss = ctxt.parseNSCollectionRequired(%s, %s);' % (db_name, ++ element)) ++ ++ ++def get_struct_info(struct): ++ # type: (ast.Struct) -> StructTypeInfoBase ++ """Get type information about the struct or command to generate C++ code.""" ++ ++ if isinstance(struct, ast.Command): ++ if struct.namespace == common.COMMAND_NAMESPACE_IGNORED: ++ return _IgnoredCommandTypeInfo(struct) ++ elif struct.namespace == common.COMMAND_NAMESPACE_CONCATENATE_WITH_DB: ++ return _CommandWithNamespaceTypeInfo(struct) ++ return _CommandFromType(struct) ++ ++ return _StructTypeInfo(struct) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/syntax.py mongo-r4.0.23/buildscripts/idl/idl/syntax.py +--- mongo-r4.0.23.orig/buildscripts/idl/idl/syntax.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/syntax.py 2021-03-17 01:21:05.952000000 +0000 +@@ -95,7 +95,7 @@ def _item_and_type(dic): + # type: (Dict[Any, List[Any]]) -> Iterator[Tuple[Any, Any]] + """Return an Iterator of (key, value) pairs from a dictionary.""" + return itertools.chain.from_iterable( +- (_zip_scalar(value, key) for (key, value) in dic.viewitems())) ++ (_zip_scalar(value, key) for (key, value) in dic.items())) + + + class SymbolTable(object): +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/idl/syntax.py.orig mongo-r4.0.23/buildscripts/idl/idl/syntax.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/idl/syntax.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/idl/syntax.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,432 @@ ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++""" ++IDL Parser Syntax classes. ++ ++These class represent the structure of the raw IDL document. ++It maps 1-1 to the YAML file, and has not been checked if ++it follows the rules of the IDL, etc. ++""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++import itertools ++from typing import Any, Dict, Iterator, List, Optional, Tuple, Union ++ ++from . import common ++from . import errors ++ ++ ++class IDLParsedSpec(object): ++ """A parsed IDL document or a set of errors if parsing failed.""" ++ ++ def __init__(self, spec, error_collection): ++ # type: (IDLSpec, errors.ParserErrorCollection) -> None ++ """Must specify either an IDL document or errors, not both.""" ++ assert (spec is None and error_collection is not None) or (spec is not None ++ and error_collection is None) ++ self.spec = spec ++ self.errors = error_collection ++ ++ ++class IDLSpec(object): ++ """ ++ The in-memory representation of an IDL file. ++ ++ - Includes all imported files. ++ """ ++ ++ def __init__(self): ++ # type: () -> None ++ """Construct an IDL spec.""" ++ self.symbols = SymbolTable() # type: SymbolTable ++ self.globals = None # type: Optional[Global] ++ self.imports = None # type: Optional[Import] ++ ++ ++def parse_array_type(name): ++ # type: (unicode) -> unicode ++ """Parse a type name of the form 'array' and extract type.""" ++ if not name.startswith("array<") and not name.endswith(">"): ++ return None ++ ++ name = name[len("array<"):] ++ name = name[:-1] ++ ++ # V1 restriction, ban nested array types to reduce scope. ++ if name.startswith("array<") and name.endswith(">"): ++ return None ++ ++ return name ++ ++ ++def _zip_scalar(items, obj): ++ # type: (List[Any], Any) -> Iterator[Tuple[Any, Any]] ++ """Return an Iterator of (obj, list item) tuples.""" ++ return ((item, obj) for item in items) ++ ++ ++def _item_and_type(dic): ++ # type: (Dict[Any, List[Any]]) -> Iterator[Tuple[Any, Any]] ++ """Return an Iterator of (key, value) pairs from a dictionary.""" ++ return itertools.chain.from_iterable( ++ (_zip_scalar(value, key) for (key, value) in dic.viewitems())) ++ ++ ++class SymbolTable(object): ++ """ ++ IDL Symbol Table. ++ ++ - Contains all information to resolve commands, enums, structs, and types. ++ - Checks for duplicate names across the union of (commands, enums, types, structs) ++ """ ++ ++ def __init__(self): ++ # type: () -> None ++ """Construct an empty symbol table.""" ++ self.commands = [] # type: List[Command] ++ self.enums = [] # type: List[Enum] ++ self.structs = [] # type: List[Struct] ++ self.types = [] # type: List[Type] ++ ++ def _is_duplicate(self, ctxt, location, name, duplicate_class_name): ++ # type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> bool ++ """Return true if the given item already exist in the symbol table.""" ++ for (item, entity_type) in _item_and_type({ ++ "command": self.commands, ++ "enum": self.enums, ++ "struct": self.structs, ++ "type": self.types, ++ }): ++ if item.name == name: ++ ctxt.add_duplicate_symbol_error(location, name, duplicate_class_name, entity_type) ++ return True ++ ++ return False ++ ++ def add_enum(self, ctxt, idl_enum): ++ # type: (errors.ParserContext, Enum) -> None ++ """Add an IDL enum to the symbol table and check for duplicates.""" ++ if not self._is_duplicate(ctxt, idl_enum, idl_enum.name, "enum"): ++ self.enums.append(idl_enum) ++ ++ def add_struct(self, ctxt, struct): ++ # type: (errors.ParserContext, Struct) -> None ++ """Add an IDL struct to the symbol table and check for duplicates.""" ++ if not self._is_duplicate(ctxt, struct, struct.name, "struct"): ++ self.structs.append(struct) ++ ++ def add_type(self, ctxt, idltype): ++ # type: (errors.ParserContext, Type) -> None ++ """Add an IDL type to the symbol table and check for duplicates.""" ++ if not self._is_duplicate(ctxt, idltype, idltype.name, "type"): ++ self.types.append(idltype) ++ ++ def add_command(self, ctxt, command): ++ # type: (errors.ParserContext, Command) -> None ++ """Add an IDL command to the symbol table and check for duplicates.""" ++ if not self._is_duplicate(ctxt, command, command.name, "command"): ++ self.commands.append(command) ++ ++ def add_imported_symbol_table(self, ctxt, imported_symbols): ++ # type: (errors.ParserContext, SymbolTable) -> None ++ """ ++ Merge all the symbols in the imported_symbols symbol table into the symbol table. ++ ++ Marks imported structs as imported, and errors on duplicate symbols. ++ """ ++ for command in imported_symbols.commands: ++ if not self._is_duplicate(ctxt, command, command.name, "command"): ++ command.imported = True ++ self.commands.append(command) ++ ++ for struct in imported_symbols.structs: ++ if not self._is_duplicate(ctxt, struct, struct.name, "struct"): ++ struct.imported = True ++ self.structs.append(struct) ++ ++ for idl_enum in imported_symbols.enums: ++ if not self._is_duplicate(ctxt, idl_enum, idl_enum.name, "enum"): ++ idl_enum.imported = True ++ self.enums.append(idl_enum) ++ ++ for idltype in imported_symbols.types: ++ self.add_type(ctxt, idltype) ++ ++ def resolve_field_type(self, ctxt, location, field_name, type_name): ++ # type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> Optional[Union[Command, Enum, Struct, Type]] ++ """Find the type or struct a field refers to or log an error.""" ++ return self._resolve_field_type(ctxt, location, field_name, type_name) ++ ++ def _resolve_field_type(self, ctxt, location, field_name, type_name): ++ # type: (errors.ParserContext, common.SourceLocation, unicode, unicode) -> Optional[Union[Command, Enum, Struct, Type]] ++ """Find the type or struct a field refers to or log an error.""" ++ # pylint: disable=too-many-return-statements ++ ++ for command in self.commands: ++ if command.name == type_name: ++ return command ++ ++ for idl_enum in self.enums: ++ if idl_enum.name == type_name: ++ return idl_enum ++ ++ for struct in self.structs: ++ if struct.name == type_name: ++ return struct ++ ++ for idltype in self.types: ++ if idltype.name == type_name: ++ return idltype ++ ++ if type_name.startswith('array<'): ++ array_type_name = parse_array_type(type_name) ++ if not array_type_name: ++ ctxt.add_bad_array_type_name_error(location, field_name, type_name) ++ return None ++ ++ return self._resolve_field_type(ctxt, location, field_name, array_type_name) ++ ++ ctxt.add_unknown_type_error(location, field_name, type_name) ++ ++ return None ++ ++ ++class Global(common.SourceLocation): ++ """ ++ IDL global object container. ++ ++ Not all fields may be populated. If they do not exist in the source document, they are not ++ populated. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Global.""" ++ self.cpp_namespace = None # type: unicode ++ self.cpp_includes = [] # type: List[unicode] ++ super(Global, self).__init__(file_name, line, column) ++ ++ ++class Import(common.SourceLocation): ++ """IDL imports object.""" ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct an Imports section.""" ++ self.imports = [] # type: List[unicode] ++ ++ # These are not part of the IDL syntax but are produced by the parser. ++ # List of imports with structs. ++ self.resolved_imports = [] # type: List[unicode] ++ # All imports directly or indirectly included ++ self.dependencies = [] # type: List[unicode] ++ ++ super(Import, self).__init__(file_name, line, column) ++ ++ ++class Type(common.SourceLocation): ++ """ ++ Stores all type information about an IDL type. ++ ++ The fields name, description, cpp_type, and bson_serialization_type are required. ++ Other fields may be populated. If they do not exist in the source document, they are not ++ populated. ++ """ ++ ++ # pylint: disable=too-many-instance-attributes ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Type.""" ++ self.name = None # type: unicode ++ self.description = None # type: unicode ++ self.cpp_type = None # type: unicode ++ self.bson_serialization_type = None # type: List[unicode] ++ self.bindata_subtype = None # type: unicode ++ self.serializer = None # type: unicode ++ self.deserializer = None # type: unicode ++ self.default = None # type: unicode ++ ++ super(Type, self).__init__(file_name, line, column) ++ ++ ++class Field(common.SourceLocation): ++ """ ++ An instance of a field in a struct. ++ ++ The fields name, and type are required. ++ Other fields may be populated. If they do not exist in the source document, they are not ++ populated. ++ """ ++ ++ # pylint: disable=too-many-instance-attributes ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Field.""" ++ self.name = None # type: unicode ++ self.cpp_name = None # type: unicode ++ self.description = None # type: unicode ++ self.type = None # type: unicode ++ self.ignore = False # type: bool ++ self.optional = False # type: bool ++ self.default = None # type: unicode ++ self.supports_doc_sequence = False # type: bool ++ self.comparison_order = -1 # type: int ++ self.non_const_getter = False # type: bool ++ ++ # Internal fields - not generated by parser ++ self.serialize_op_msg_request_only = False # type: bool ++ self.constructed = False # type: bool ++ ++ super(Field, self).__init__(file_name, line, column) ++ ++ ++class ChainedStruct(common.SourceLocation): ++ """ ++ Stores all type information about an IDL chained struct. ++ ++ The fields name, and cpp_name are required. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Type.""" ++ self.name = None # type: unicode ++ self.cpp_name = None # type: unicode ++ ++ super(ChainedStruct, self).__init__(file_name, line, column) ++ ++ ++class ChainedType(common.SourceLocation): ++ """ ++ Stores all type information about an IDL chained type. ++ ++ The fields name, and cpp_name are required. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Type.""" ++ self.name = None # type: unicode ++ self.cpp_name = None # type: unicode ++ ++ super(ChainedType, self).__init__(file_name, line, column) ++ ++ ++class Struct(common.SourceLocation): ++ """ ++ IDL struct information. ++ ++ All fields are either required or have a non-None default. ++ """ ++ ++ # pylint: disable=too-many-instance-attributes ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Struct.""" ++ self.name = None # type: unicode ++ self.description = None # type: unicode ++ self.strict = True # type: bool ++ self.immutable = False # type: bool ++ self.inline_chained_structs = True # type: bool ++ self.generate_comparison_operators = False # type: bool ++ self.chained_types = None # type: List[ChainedType] ++ self.chained_structs = None # type: List[ChainedStruct] ++ self.fields = None # type: List[Field] ++ ++ # Command only property ++ self.cpp_name = None # type: unicode ++ ++ # Internal property that is not represented as syntax. An imported struct is read from an ++ # imported file, and no code is generated for it. ++ self.imported = False # type: bool ++ ++ # Internal property: cpp_namespace from globals section ++ self.cpp_namespace = None # type: unicode ++ ++ super(Struct, self).__init__(file_name, line, column) ++ ++ ++class Command(Struct): ++ """ ++ IDL command information, a subtype of Struct. ++ ++ Namespace is required. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct a Command.""" ++ self.namespace = None # type: unicode ++ self.type = None # type: unicode ++ ++ super(Command, self).__init__(file_name, line, column) ++ ++ ++class EnumValue(common.SourceLocation): ++ """ ++ IDL Enum Value information. ++ ++ All fields are either required or have a non-None default. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct an Enum.""" ++ self.name = None # type: unicode ++ self.value = None # type: unicode ++ ++ super(EnumValue, self).__init__(file_name, line, column) ++ ++ ++class Enum(common.SourceLocation): ++ """ ++ IDL Enum information. ++ ++ All fields are either required or have a non-None default. ++ """ ++ ++ def __init__(self, file_name, line, column): ++ # type: (unicode, int, int) -> None ++ """Construct an Enum.""" ++ self.name = None # type: unicode ++ self.description = None # type: unicode ++ self.type = None # type: unicode ++ self.values = None # type: List[EnumValue] ++ ++ # Internal property that is not represented as syntax. An imported enum is read from an ++ # imported file, and no code is generated for it. ++ self.imported = False # type: bool ++ ++ # Internal property: cpp_namespace from globals section ++ self.cpp_namespace = None # type: unicode ++ ++ super(Enum, self).__init__(file_name, line, column) +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/tests/test_binder.py mongo-r4.0.23/buildscripts/idl/tests/test_binder.py +--- mongo-r4.0.23.orig/buildscripts/idl/tests/test_binder.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/tests/test_binder.py 2021-03-17 01:21:05.952000000 +0000 +@@ -88,7 +88,7 @@ class TestBinder(testcase.IDLTestcase): + cpp_includes: + - 'bar' + - 'foo'""")) +- self.assertEquals(spec.globals.cpp_namespace, "something") ++ self.assertEqual(spec.globals.cpp_namespace, "something") + self.assertListEqual(spec.globals.cpp_includes, ['bar', 'foo']) + + def test_type_positive(self): +diff -upNr mongo-r4.0.23.orig/buildscripts/idl/tests/test_binder.py.orig mongo-r4.0.23/buildscripts/idl/tests/test_binder.py.orig +--- mongo-r4.0.23.orig/buildscripts/idl/tests/test_binder.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/idl/tests/test_binder.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,1613 @@ ++#!/usr/bin/env python2 ++# ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++# ++# pylint: disable=too-many-lines ++"""Test cases for IDL binder.""" ++ ++from __future__ import absolute_import, print_function, unicode_literals ++ ++import textwrap ++import unittest ++ ++# import package so that it works regardless of whether we run as a module or file ++if __package__ is None: ++ import sys ++ from os import path ++ sys.path.append(path.dirname(path.abspath(__file__))) ++ from context import idl ++ import testcase ++else: ++ from .context import idl ++ from . import testcase ++ ++# All YAML tests assume 4 space indent ++INDENT_SPACE_COUNT = 4 ++ ++ ++def fill_spaces(count): ++ # type: (int) -> unicode ++ """Fill a string full of spaces.""" ++ fill = '' ++ for _ in range(count * INDENT_SPACE_COUNT): ++ fill += ' ' ++ ++ return fill ++ ++ ++def indent_text(count, unindented_text): ++ # type: (int, unicode) -> unicode ++ """Indent each line of a multi-line string.""" ++ lines = unindented_text.splitlines() ++ fill = fill_spaces(count) ++ return '\n'.join(fill + line for line in lines) ++ ++ ++class TestBinder(testcase.IDLTestcase): ++ """Test cases for the IDL binder.""" ++ ++ # pylint: disable=too-many-public-methods ++ ++ def test_empty(self): ++ # type: () -> None ++ """Test an empty document works.""" ++ self.assert_bind("") ++ ++ def test_global_positive(self): ++ # type: () -> None ++ """Postive global tests.""" ++ spec = self.assert_bind( ++ textwrap.dedent(""" ++ global: ++ cpp_namespace: 'something' ++ cpp_includes: ++ - 'bar' ++ - 'foo'""")) ++ self.assertEquals(spec.globals.cpp_namespace, "something") ++ self.assertListEqual(spec.globals.cpp_includes, ['bar', 'foo']) ++ ++ def test_type_positive(self): ++ # type: () -> None ++ """Positive type tests.""" ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """)) ++ ++ # Test supported types ++ for bson_type in [ ++ "bool", "date", "null", "decimal", "double", "int", "long", "objectid", "regex", ++ "string", "timestamp", "undefined" ++ ]: ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: %s ++ default: foo ++ deserializer: BSONElement::fake ++ """ % (bson_type))) ++ ++ # Test supported numeric types ++ for cpp_type in [ ++ "std::int32_t", ++ "std::uint32_t", ++ "std::int32_t", ++ "std::uint64_t", ++ "std::vector", ++ "std::array", ++ ]: ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: %s ++ bson_serialization_type: int ++ deserializer: BSONElement::fake ++ """ % (cpp_type))) ++ ++ # Test object ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: object ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """)) ++ ++ # Test 'any' ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: any ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """)) ++ ++ # Test 'chain' ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """)) ++ ++ # Test supported bindata_subtype ++ for bindata_subtype in ["generic", "function", "uuid", "md5"]: ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: %s ++ deserializer: BSONElement::fake ++ """ % (bindata_subtype))) ++ ++ def test_type_negative(self): ++ # type: () -> None ++ """Negative type tests for properties that types and fields share.""" ++ ++ # Test bad bson type name ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: foo ++ """), idl.errors.ERROR_ID_BAD_BSON_TYPE) ++ ++ # Test bad cpp_type name ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: StringData ++ bson_serialization_type: string ++ deserializer: bar ++ """), idl.errors.ERROR_ID_NO_STRINGDATA) ++ ++ # Test unsupported serialization ++ for cpp_type in [ ++ "char", ++ "signed char", ++ "unsigned char", ++ "signed short int", ++ "short int", ++ "short", ++ "signed short", ++ "unsigned short", ++ "unsigned short int", ++ "signed int", ++ "signed", ++ "unsigned int", ++ "unsigned", ++ "signed long int", ++ "signed long", ++ "int", ++ "long int", ++ "long", ++ "unsigned long int", ++ "unsigned long", ++ "signed long long int", ++ "signed long long", ++ "long long int", ++ "long long", ++ "unsigned long int", ++ "unsigned long", ++ "wchar_t", ++ "char16_t", ++ "char32_t", ++ "int8_t", ++ "int16_t", ++ "int32_t", ++ "int64_t", ++ "uint8_t", ++ "uint16_t", ++ "uint32_t", ++ "uint64_t", ++ ]: ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: %s ++ bson_serialization_type: int ++ deserializer: BSONElement::int ++ """ % (cpp_type)), idl.errors.ERROR_ID_BAD_NUMERIC_CPP_TYPE) ++ ++ # Test the std prefix 8 and 16-byte integers fail ++ for std_cpp_type in ["std::int8_t", "std::int16_t", "std::uint8_t", "std::uint16_t"]: ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: %s ++ bson_serialization_type: int ++ deserializer: BSONElement::int ++ """ % (std_cpp_type)), idl.errors.ERROR_ID_BAD_NUMERIC_CPP_TYPE) ++ ++ # Test bindata_subtype missing ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ deserializer: BSONElement::fake ++ """), idl.errors.ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_VALUE) ++ ++ # Test fake bindata_subtype is wrong ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: foo ++ deserializer: BSONElement::fake ++ """), idl.errors.ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_VALUE) ++ ++ # Test deprecated bindata_subtype 'binary', and 'uuid_old' are wrong ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: binary ++ """), idl.errors.ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_VALUE) ++ ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: uuid_old ++ """), idl.errors.ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_VALUE) ++ ++ # Test bindata_subtype on wrong type ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ bindata_subtype: generic ++ deserializer: BSONElement::fake ++ """), idl.errors.ERROR_ID_BAD_BSON_BINDATA_SUBTYPE_TYPE) ++ ++ # Test bindata with default ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: uuid ++ default: 42 ++ """), idl.errors.ERROR_ID_BAD_BINDATA_DEFAULT) ++ ++ # Test bindata in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: ++ - bindata ++ - string ++ """), idl.errors.ERROR_ID_BAD_BSON_TYPE) ++ ++ # Test bindata in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: StringData ++ bson_serialization_type: ++ - bindata ++ - string ++ """), idl.errors.ERROR_ID_BAD_BSON_TYPE) ++ ++ # Test 'any' in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: ++ - any ++ - int ++ """), idl.errors.ERROR_ID_BAD_ANY_TYPE_USE) ++ ++ # Test object in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: ++ - object ++ - int ++ """), idl.errors.ERROR_ID_BAD_BSON_TYPE_LIST) ++ ++ # Test fake in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: ++ - int ++ - fake ++ """), idl.errors.ERROR_ID_BAD_BSON_TYPE) ++ ++ # Test 'chain' in list of types ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: ++ - chain ++ - int ++ """), idl.errors.ERROR_ID_BAD_ANY_TYPE_USE) ++ ++ # Test unsupported serialization ++ for bson_type in [ ++ "bool", "date", "null", "decimal", "double", "int", "long", "objectid", "regex", ++ "timestamp", "undefined" ++ ]: ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: std::string ++ bson_serialization_type: %s ++ serializer: foo ++ deserializer: BSONElement::fake ++ """ % (bson_type)), ++ idl.errors.ERROR_ID_CUSTOM_SCALAR_SERIALIZATION_NOT_SUPPORTED) ++ ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: std::string ++ bson_serialization_type: %s ++ deserializer: foo ++ """ % (bson_type)), ++ idl.errors.ERROR_ID_CUSTOM_SCALAR_SERIALIZATION_NOT_SUPPORTED) ++ ++ # Test 'any' serialization needs deserializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: any ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test 'chain' serialization needs deserializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: bar ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test 'string' serialization needs deserializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: bar ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test 'date' serialization needs deserializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: date ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test 'chain' serialization needs serializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ deserializer: bar ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test list of bson types needs deserializer ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ foofoo: ++ description: foo ++ cpp_type: std::int32_t ++ bson_serialization_type: ++ - int ++ - string ++ """), idl.errors.ERROR_ID_MISSING_AST_REQUIRED_FIELD) ++ ++ # Test array as name ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ array: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ deserializer: bar ++ """), idl.errors.ERROR_ID_ARRAY_NOT_VALID_TYPE) ++ ++ def test_struct_positive(self): ++ # type: () -> None ++ """Positive struct tests.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """) ++ ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: string ++ """)) ++ ++ def test_struct_negative(self): ++ # type: () -> None ++ """Negative struct tests.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """) ++ ++ # Test array as name ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ array: ++ description: foo ++ strict: true ++ fields: ++ foo: string ++ """), idl.errors.ERROR_ID_ARRAY_NOT_VALID_TYPE) ++ ++ def test_field_positive(self): ++ # type: () -> None ++ """Positive test cases for field.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ """) ++ ++ # Short type ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ bar: ++ description: foo ++ strict: false ++ fields: ++ foo: string ++ """)) ++ ++ # Long type ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ bar: ++ description: foo ++ strict: false ++ fields: ++ foo: ++ type: string ++ """)) ++ ++ # Long type with default ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ bar: ++ description: foo ++ strict: false ++ fields: ++ foo: ++ type: string ++ default: bar ++ """)) ++ ++ # Test array as field type ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: array ++ """)) ++ ++ # Test array as field type ++ self.assert_bind( ++ textwrap.dedent(""" ++ types: ++ arrayfake: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ arrayOfString: arrayfake ++ """)) ++ ++ def test_field_negative(self): ++ # type: () -> None ++ """Negative field tests.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ bindata: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: bindata ++ bindata_subtype: uuid ++ """) ++ ++ # Test field of a struct type with a default ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ fields: ++ field1: string ++ ++ bar: ++ description: foo ++ fields: ++ field2: ++ type: foo ++ default: foo ++ ++ """), idl.errors.ERROR_ID_FIELD_MUST_BE_EMPTY_FOR_STRUCT) ++ ++ # Test array as field name ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ array: string ++ """), idl.errors.ERROR_ID_ARRAY_NOT_VALID_TYPE) ++ ++ # Test recursive array as field type ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: array> ++ """), idl.errors.ERROR_ID_BAD_ARRAY_TYPE_NAME) ++ ++ # Test inherited default with array ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: array ++ """), idl.errors.ERROR_ID_ARRAY_NO_DEFAULT) ++ ++ # Test non-inherited default with array ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: ++ type: array ++ default: 123 ++ """), idl.errors.ERROR_ID_ARRAY_NO_DEFAULT) ++ ++ # Test bindata with default ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: ++ type: bindata ++ default: 42 ++ """), idl.errors.ERROR_ID_BAD_BINDATA_DEFAULT) ++ ++ # Test default and optional for the same field ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: true ++ fields: ++ foo: ++ type: string ++ default: 42 ++ optional: true ++ """), idl.errors.ERROR_ID_ILLEGAL_FIELD_DEFAULT_AND_OPTIONAL) ++ ++ # Test duplicate comparison order ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: false ++ generate_comparison_operators: true ++ fields: ++ foo: ++ type: string ++ comparison_order: 1 ++ bar: ++ type: string ++ comparison_order: 1 ++ """), idl.errors.ERROR_ID_IS_DUPLICATE_COMPARISON_ORDER) ++ ++ # Test field marked with non_const_getter in immutable struct ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ immutable: true ++ fields: ++ foo: ++ type: string ++ non_const_getter: true ++ """), idl.errors.ERROR_ID_NON_CONST_GETTER_IN_IMMUTABLE_STRUCT) ++ ++ def test_ignored_field_negative(self): ++ # type: () -> None ++ """Test that if a field is marked as ignored, no other properties are set.""" ++ for test_value in [ ++ "optional: true", ++ "default: foo", ++ ]: ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ strict: false ++ fields: ++ foo: ++ type: string ++ ignore: true ++ %s ++ """ % (test_value)), idl.errors.ERROR_ID_FIELD_MUST_BE_EMPTY_FOR_IGNORED) ++ ++ def test_chained_type_positive(self): ++ # type: () -> None ++ """Positive parser chaining test cases.""" ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ ++ foo1: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ """) ++ ++ # Chaining only ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ foo1: alias ++ """)) ++ ++ def test_chained_type_negative(self): ++ # type: () -> None ++ """Negative parser chaining test cases.""" ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ ++ foo1: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: foo ++ deserializer: foo ++ ++ """) ++ ++ # Chaining with strict struct ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: true ++ chained_types: ++ foo1: alias ++ """), idl.errors.ERROR_ID_CHAINED_NO_TYPE_STRICT) ++ ++ # Non-'any' type as chained type ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ string: alias ++ """), idl.errors.ERROR_ID_CHAINED_TYPE_WRONG_BSON_TYPE) ++ ++ # Chaining and fields only with same name ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ foo1: alias ++ fields: ++ foo1: string ++ """), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD) ++ ++ # Non-existent chained type ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ foobar1: alias ++ fields: ++ foo1: string ++ """), idl.errors.ERROR_ID_UNKNOWN_TYPE) ++ ++ # A regular field as a chained type ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: false ++ fields: ++ foo1: string ++ foo2: foobar1 ++ """), idl.errors.ERROR_ID_UNKNOWN_TYPE) ++ ++ # Array of chained types ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ bar1: ++ description: foo ++ strict: true ++ fields: ++ field1: array ++ """), idl.errors.ERROR_ID_NO_ARRAY_OF_CHAIN) ++ ++ def test_chained_struct_positive(self): ++ # type: () -> None ++ """Positive parser chaining test cases.""" ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ ++ foo1: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ structs: ++ chained: ++ description: foo ++ strict: false ++ chained_types: ++ foo1: alias ++ ++ chained2: ++ description: foo ++ strict: false ++ fields: ++ field1: string ++ """) ++ ++ # A struct with only chaining ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ chained_structs: ++ chained2: alias ++ """))) ++ ++ # Chaining struct's fields and explicit fields ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ chained_structs: ++ chained2: alias ++ fields: ++ str1: string ++ """))) ++ ++ # Chained types and structs ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ foo1: alias ++ chained_structs: ++ chained2: alias ++ fields: ++ str1: string ++ """))) ++ ++ # Non-strict chained struct ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_structs: ++ chained2: alias ++ fields: ++ foo1: string ++ """))) ++ ++ # Inline Chained struct with strict true ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ fields: ++ field1: string ++ ++ foobar: ++ description: foo ++ strict: false ++ inline_chained_structs: true ++ chained_structs: ++ bar1: alias ++ fields: ++ f1: string ++ ++ """))) ++ ++ # Inline Chained struct with strict true and inline_chained_structs defaulted ++ self.assert_bind(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ fields: ++ field1: string ++ ++ foobar: ++ description: foo ++ strict: false ++ chained_structs: ++ bar1: alias ++ fields: ++ f1: string ++ """))) ++ ++ def test_chained_struct_negative(self): ++ # type: () -> None ++ """Negative parser chaining test cases.""" ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ ++ foo1: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: chain ++ serializer: foo ++ deserializer: foo ++ default: foo ++ ++ structs: ++ chained: ++ description: foo ++ strict: false ++ fields: ++ field1: string ++ ++ chained2: ++ description: foo ++ strict: false ++ fields: ++ field1: string ++ """) ++ ++ # Non-existing chained struct ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ chained_structs: ++ foobar1: alias ++ """)), idl.errors.ERROR_ID_UNKNOWN_TYPE) ++ ++ # Type as chained struct ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ chained_structs: ++ foo1: alias ++ """)), idl.errors.ERROR_ID_CHAINED_STRUCT_NOT_FOUND) ++ ++ # Struct as chained type ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ chained: alias ++ """)), idl.errors.ERROR_ID_CHAINED_TYPE_NOT_FOUND) ++ ++ # Duplicated field names across chained struct's fields and fields ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_structs: ++ chained: alias ++ fields: ++ field1: string ++ """)), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD) ++ ++ # Duplicated field names across chained structs ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_structs: ++ chained: alias ++ chained2: alias ++ """)), idl.errors.ERROR_ID_CHAINED_DUPLICATE_FIELD) ++ ++ # Chained struct with strict true ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: true ++ fields: ++ field1: string ++ ++ foobar: ++ description: foo ++ strict: false ++ inline_chained_structs: false ++ chained_structs: ++ bar1: alias ++ fields: ++ f1: string ++ ++ """)), idl.errors.ERROR_ID_CHAINED_NO_NESTED_STRUCT_STRICT) ++ ++ # Chained struct with nested chained struct ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_structs: ++ chained: alias ++ ++ foobar: ++ description: foo ++ strict: false ++ chained_structs: ++ bar1: alias ++ fields: ++ f1: string ++ ++ """)), idl.errors.ERROR_ID_CHAINED_NO_NESTED_CHAINED) ++ ++ # Chained struct with nested chained type ++ self.assert_bind_fail(test_preamble + indent_text(1, ++ textwrap.dedent(""" ++ bar1: ++ description: foo ++ strict: false ++ chained_types: ++ foo1: alias ++ ++ foobar: ++ description: foo ++ strict: false ++ chained_structs: ++ bar1: alias ++ fields: ++ f1: bar1 ++ ++ """)), idl.errors.ERROR_ID_CHAINED_NO_NESTED_CHAINED) ++ ++ def test_enum_positive(self): ++ # type: () -> None ++ """Positive enum test cases.""" ++ ++ # Test int ++ self.assert_bind( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: int ++ values: ++ v1: 3 ++ v2: 1 ++ v3: 2 ++ """)) ++ ++ # Test string ++ self.assert_bind( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: string ++ values: ++ v1: 0 ++ v2: 1 ++ v3: 2 ++ """)) ++ ++ def test_enum_negative(self): ++ # type: () -> None ++ """Negative enum test cases.""" ++ ++ # Test wrong type ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: foo ++ values: ++ v1: 0 ++ """), idl.errors.ERROR_ID_ENUM_BAD_TYPE) ++ ++ # Test int - non continuous ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: int ++ values: ++ v1: 0 ++ v3: 2 ++ """), idl.errors.ERROR_ID_ENUM_NON_CONTINUOUS_RANGE) ++ ++ # Test int - dups ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: int ++ values: ++ v1: 1 ++ v3: 1 ++ """), idl.errors.ERROR_ID_ENUM_NON_UNIQUE_VALUES) ++ ++ # Test int - non-integer value ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: int ++ values: ++ v1: foo ++ v3: 1 ++ """), idl.errors.ERROR_ID_ENUM_BAD_INT_VAUE) ++ ++ # Test string - dups ++ self.assert_bind_fail( ++ textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: string ++ values: ++ v1: foo ++ v3: foo ++ """), idl.errors.ERROR_ID_ENUM_NON_UNIQUE_VALUES) ++ ++ def test_struct_enum_negative(self): ++ # type: () -> None ++ """Negative enum test cases.""" ++ ++ test_preamble = textwrap.dedent(""" ++ enums: ++ foo: ++ description: foo ++ type: int ++ values: ++ v1: 0 ++ v2: 1 ++ """) ++ ++ # Test array of enums ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo1: ++ description: foo ++ fields: ++ foo1: array ++ """), idl.errors.ERROR_ID_NO_ARRAY_ENUM) ++ ++ def test_command_positive(self): ++ # type: () -> None ++ """Positive command tests.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """) ++ ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ strict: true ++ fields: ++ foo1: string ++ """)) ++ ++ def test_command_negative(self): ++ # type: () -> None ++ """Negative command tests.""" ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ default: foo ++ """) ++ ++ # Commands cannot be fields in other commands ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: string ++ ++ bar: ++ description: foo ++ namespace: ignored ++ fields: ++ foo: foo ++ """), idl.errors.ERROR_ID_FIELD_NO_COMMAND) ++ ++ # Commands cannot be fields in structs ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: string ++ ++ structs: ++ bar: ++ description: foo ++ fields: ++ foo: foo ++ """), idl.errors.ERROR_ID_FIELD_NO_COMMAND) ++ ++ # Commands cannot have a field as the same name ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo: string ++ """), idl.errors.ERROR_ID_COMMAND_DUPLICATES_FIELD) ++ ++ def test_command_doc_sequence_positive(self): ++ # type: () -> None ++ """Positive supports_doc_sequence tests.""" ++ # pylint: disable=invalid-name ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ object: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: object ++ serializer: foo ++ deserializer: foo ++ ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ ++ structs: ++ foo_struct: ++ description: foo ++ strict: true ++ fields: ++ foo: object ++ """) ++ ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: ++ type: array ++ supports_doc_sequence: true ++ """)) ++ ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: ++ type: array ++ supports_doc_sequence: true ++ """)) ++ ++ def test_command_doc_sequence_negative(self): ++ # type: () -> None ++ """Negative supports_doc_sequence tests.""" ++ # pylint: disable=invalid-name ++ ++ # Setup some common types ++ test_preamble = textwrap.dedent(""" ++ types: ++ object: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: object ++ serializer: foo ++ deserializer: foo ++ ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ ++ any_type: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: any ++ serializer: foo ++ deserializer: foo ++ """) ++ ++ test_preamble2 = test_preamble + textwrap.dedent(""" ++ structs: ++ foo_struct: ++ description: foo ++ strict: true ++ fields: ++ foo: object ++ """) ++ ++ # A struct ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ structs: ++ foo: ++ description: foo ++ fields: ++ foo: ++ type: array ++ supports_doc_sequence: true ++ """), idl.errors.ERROR_ID_STRUCT_NO_DOC_SEQUENCE) ++ ++ # A non-array type ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo: ++ type: object ++ supports_doc_sequence: true ++ """), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_ARRAY) ++ ++ # An array of a scalar ++ self.assert_bind_fail(test_preamble2 + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: ++ type: array ++ supports_doc_sequence: true ++ """), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_OBJECT) ++ ++ # An array of 'any' ++ self.assert_bind_fail(test_preamble2 + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: ignored ++ fields: ++ foo1: ++ type: array ++ supports_doc_sequence: true ++ """), idl.errors.ERROR_ID_NO_DOC_SEQUENCE_FOR_NON_OBJECT) ++ ++ def test_command_type_positive(self): ++ # type: () -> None ++ """Positive command custom type test cases.""" ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ """) ++ ++ # string ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ strict: true ++ namespace: type ++ type: string ++ fields: ++ field1: string ++ """)) ++ ++ # array of string ++ self.assert_bind(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ strict: true ++ namespace: type ++ type: array ++ fields: ++ field1: string ++ """)) ++ ++ def test_command_type_negative(self): ++ # type: () -> None ++ """Negative command type test cases.""" ++ test_preamble = textwrap.dedent(""" ++ types: ++ string: ++ description: foo ++ cpp_type: foo ++ bson_serialization_type: string ++ serializer: foo ++ deserializer: foo ++ """) ++ ++ # supports_doc_sequence must be a bool ++ self.assert_bind_fail(test_preamble + textwrap.dedent(""" ++ commands: ++ foo: ++ description: foo ++ namespace: type ++ type: int ++ fields: ++ field1: string ++ """), idl.errors.ERROR_ID_UNKNOWN_TYPE) ++ ++ ++if __name__ == '__main__': ++ ++ unittest.main() +diff -upNr mongo-r4.0.23.orig/buildscripts/linter/base.py mongo-r4.0.23/buildscripts/linter/base.py +--- mongo-r4.0.23.orig/buildscripts/linter/base.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/linter/base.py 2021-03-17 01:21:05.952000000 +0000 +@@ -5,12 +5,11 @@ from __future__ import print_function + from abc import ABCMeta, abstractmethod + from typing import Dict, List, Optional + ++ABC = ABCMeta(str('ABC'), (object,), {'__slots__': ()}) + +-class LinterBase(object): ++class LinterBase(ABC): + """Base Class for all linters.""" + +- __metaclass__ = ABCMeta +- + def __init__(self, cmd_name, required_version, cmd_location=None): + # type: (str, str, Optional[str]) -> None + """ +diff -upNr mongo-r4.0.23.orig/buildscripts/linter/git.py mongo-r4.0.23/buildscripts/linter/git.py +--- mongo-r4.0.23.orig/buildscripts/linter/git.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/linter/git.py 2021-03-17 01:21:05.952000000 +0000 +@@ -175,7 +175,7 @@ def get_files_to_check_from_patch(patche + + lines = [] # type: List[str] + for patch in patches: +- with open(patch, "rb") as infile: ++ with open(patch, "r") as infile: + lines += infile.readlines() + + candidates = [check.match(line).group(1) for line in lines if check.match(line)] +diff -upNr mongo-r4.0.23.orig/buildscripts/linter/parallel.py mongo-r4.0.23/buildscripts/linter/parallel.py +--- mongo-r4.0.23.orig/buildscripts/linter/parallel.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/linter/parallel.py 2021-03-17 01:21:05.952000000 +0000 +@@ -2,7 +2,12 @@ + from __future__ import absolute_import + from __future__ import print_function + +-import Queue ++try: ++ import queue ++except ImportError: ++ #Python 2 ++ import Queue as queue ++ + import threading + import time + from multiprocessing import cpu_count +@@ -17,7 +22,7 @@ def parallel_process(items, func): + except NotImplementedError: + cpus = 1 + +- task_queue = Queue.Queue() # type: Queue.Queue ++ task_queue = queue.Queue() # type: queue.Queue + + # Use a list so that worker function will capture this variable + pp_event = threading.Event() +@@ -30,7 +35,7 @@ def parallel_process(items, func): + while not pp_event.is_set(): + try: + item = task_queue.get_nowait() +- except Queue.Empty: ++ except queue.Empty: + # if the queue is empty, exit the worker thread + pp_event.set() + return +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokeconfig/loggers/__init__.py mongo-r4.0.23/buildscripts/resmokeconfig/loggers/__init__.py +--- mongo-r4.0.23.orig/buildscripts/resmokeconfig/loggers/__init__.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokeconfig/loggers/__init__.py 2021-03-17 01:21:05.952000000 +0000 +@@ -16,7 +16,7 @@ def _get_named_loggers(): + named_loggers = {} + + try: +- (root, _dirs, files) = os.walk(dirname).next() ++ (root, _dirs, files) = next(os.walk(dirname)) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokeconfig/suites/__init__.py mongo-r4.0.23/buildscripts/resmokeconfig/suites/__init__.py +--- mongo-r4.0.23.orig/buildscripts/resmokeconfig/suites/__init__.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokeconfig/suites/__init__.py 2021-03-17 01:21:05.952000000 +0000 +@@ -16,7 +16,7 @@ def _get_named_suites(): + named_suites = {} + + try: +- (root, _dirs, files) = os.walk(dirname).next() ++ (root, _dirs, files) = next(os.walk(dirname)) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/config.py mongo-r4.0.23/buildscripts/resmokelib/config.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/config.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/config.py 2021-03-17 01:21:05.952000000 +0000 +@@ -62,7 +62,7 @@ DEFAULTS = { + "repeat": 1, + "report_failure_status": "fail", + "report_file": None, +- "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. ++ "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. + "service_executor": None, + "shell_conn_string": None, + "shell_port": None, +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/core/process.py mongo-r4.0.23/buildscripts/resmokelib/core/process.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/core/process.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/core/process.py 2021-03-17 01:21:05.952000000 +0000 +@@ -182,8 +182,8 @@ class Process(object): + finally: + win32api.CloseHandle(mongo_signal_handle) + +- print "Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ +- str(self._process.pid) ++ print("Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ ++ str(self._process.pid)) + + # Adapted from implementation of Popen.terminate() in subprocess.py of Python 2.7 + # because earlier versions do not catch exceptions. +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/logging/buildlogger.py mongo-r4.0.23/buildscripts/resmokelib/logging/buildlogger.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/logging/buildlogger.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/logging/buildlogger.py 2021-03-17 01:21:05.952000000 +0000 +@@ -262,7 +262,7 @@ class BuildloggerServer(object): + """Initialize BuildloggerServer.""" + tmp_globals = {} + self.config = {} +- execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config) ++ exec(compile(open(_BUILDLOGGER_CONFIG).read(), _BUILDLOGGER_CONFIG, 'exec'), tmp_globals, self.config) + + # Rename "slavename" to "username" if present. + if "slavename" in self.config and "username" not in self.config: +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/logging/buildlogger.py.orig mongo-r4.0.23/buildscripts/resmokelib/logging/buildlogger.py.orig +--- mongo-r4.0.23.orig/buildscripts/resmokelib/logging/buildlogger.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/logging/buildlogger.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,334 @@ ++"""Define handlers for communicating with a buildlogger server.""" ++ ++from __future__ import absolute_import ++ ++import functools ++import json ++import os ++import threading ++ ++import requests ++ ++from . import handlers ++from .. import config as _config ++ ++CREATE_BUILD_ENDPOINT = "/build" ++APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" ++CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" ++APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" ++ ++_BUILDLOGGER_CONFIG = os.getenv("BUILDLOGGER_CREDENTIALS", "mci.buildlogger") ++ ++_SEND_AFTER_LINES = 2000 ++_SEND_AFTER_SECS = 10 ++ ++# Initialized by resmokelib.logging.loggers.configure_loggers() ++BUILDLOGGER_FALLBACK = None ++ ++_INCOMPLETE_LOG_OUTPUT = threading.Event() ++ ++ ++def is_log_output_incomplete(): # noqa: D205,D400 ++ """Return true if we failed to write all of the log output to the buildlogger server, and return ++ false otherwise. ++ """ ++ return _INCOMPLETE_LOG_OUTPUT.is_set() ++ ++ ++def set_log_output_incomplete(): ++ """Indicate that we failed to write all of the log output to the buildlogger server.""" ++ _INCOMPLETE_LOG_OUTPUT.set() ++ ++ ++def _log_on_error(func): ++ """Provide decorator that causes exceptions to be logged by the "buildlogger" Logger instance. ++ ++ Return the wrapped function's return value, or None if an error was encountered. ++ """ ++ ++ @functools.wraps(func) ++ def wrapper(*args, **kwargs): ++ """Provide wrapper function.""" ++ try: ++ return func(*args, **kwargs) ++ except requests.HTTPError as err: ++ BUILDLOGGER_FALLBACK.error("Encountered an HTTP error: %s", err) ++ except requests.RequestException as err: ++ BUILDLOGGER_FALLBACK.error("Encountered a network error: %s", err) ++ except: # pylint: disable=bare-except ++ BUILDLOGGER_FALLBACK.exception("Encountered an error.") ++ return None ++ ++ return wrapper ++ ++ ++class _LogsSplitter(object): ++ """Class with static methods used to split list of log lines into smaller batches.""" ++ ++ @staticmethod ++ def split_logs(log_lines, max_size): # noqa: D406,D407,D411,D413 ++ """Split the log lines into batches of size less than or equal to max_size. ++ ++ Args: ++ log_lines: A list of log lines. ++ max_size: The maximum size in bytes a batch of log lines can have in JSON. ++ Returns: ++ A list of list of log lines. Each item is a list is a list of log lines ++ satisfying the size requirement. ++ """ ++ if not max_size: ++ return [log_lines] ++ ++ def line_size(line): ++ """Compute the encoded JSON size of a log line as part of an array. ++ ++ 2 is added to each string size to account for the array representation of the logs, ++ as each line is preceded by a '[' or a space and followed by a ',' or a ']'. ++ """ ++ return len(json.dumps(line, encoding="utf-8")) + 2 ++ ++ curr_logs = [] ++ curr_logs_size = 0 ++ split_logs = [] ++ for line in log_lines: ++ size = line_size(line) ++ if curr_logs_size + size > max_size: ++ split_logs.append(curr_logs) ++ curr_logs = [] ++ curr_logs_size = 0 ++ curr_logs.append(line) ++ curr_logs_size += size ++ split_logs.append(curr_logs) ++ return split_logs ++ ++ ++class _BaseBuildloggerHandler(handlers.BufferedHandler): ++ """Base class of the buildlogger handler for global logs and handler for test logs.""" ++ ++ def __init__(self, build_config, endpoint, capacity=_SEND_AFTER_LINES, ++ interval_secs=_SEND_AFTER_SECS): ++ """Initialize the buildlogger handler with the build id and credentials.""" ++ ++ handlers.BufferedHandler.__init__(self, capacity, interval_secs) ++ ++ username = build_config["username"] ++ password = build_config["password"] ++ self.http_handler = handlers.HTTPHandler(_config.BUILDLOGGER_URL, username, password) ++ ++ self.endpoint = endpoint ++ self.retry_buffer = [] ++ self.max_size = None ++ ++ def process_record(self, record): ++ """Return a tuple of the time the log record was created, and the message. ++ ++ This is necessary because the buildlogger expects the log messages to be ++ formatted in JSON as: ++ ++ [ [ , ], ++ [ , ], ++ ... ] ++ """ ++ msg = self.format(record) ++ return (record.created, msg) ++ ++ def post(self, *args, **kwargs): ++ """Provide convenience method for subclasses to use when making POST requests.""" ++ return self.http_handler.post(*args, **kwargs) ++ ++ def _append_logs(self, log_lines): # noqa: D406,D407,D413 ++ """Send a POST request to the handlers endpoint with the logs that have been captured. ++ ++ Returns: ++ The number of log lines that have been successfully sent. ++ """ ++ lines_sent = 0 ++ for chunk in _LogsSplitter.split_logs(log_lines, self.max_size): ++ chunk_lines_sent = self.__append_logs_chunk(chunk) ++ lines_sent += chunk_lines_sent ++ if chunk_lines_sent < len(chunk): ++ # Not all lines have been sent. We stop here. ++ break ++ return lines_sent ++ ++ def __append_logs_chunk(self, log_lines_chunk): # noqa: D406,D407,D413 ++ """Send log lines chunk, handle 413 Request Entity Too Large errors & retry, if necessary. ++ ++ Returns: ++ The number of log lines that have been successfully sent. ++ """ ++ try: ++ self.post(self.endpoint, data=log_lines_chunk) ++ return len(log_lines_chunk) ++ except requests.HTTPError as err: ++ # Handle the "Request Entity Too Large" error, set the max size and retry. ++ if err.response.status_code == requests.codes.request_entity_too_large: ++ response_data = err.response.json() ++ if isinstance(response_data, dict) and "max_size" in response_data: ++ new_max_size = response_data["max_size"] ++ if self.max_size and new_max_size >= self.max_size: ++ BUILDLOGGER_FALLBACK.exception( ++ "Received an HTTP 413 code, but already had max_size set") ++ return 0 ++ BUILDLOGGER_FALLBACK.warning( ++ "Received an HTTP 413 code, updating the request max_size to %s", ++ new_max_size) ++ self.max_size = new_max_size ++ return self._append_logs(log_lines_chunk) ++ BUILDLOGGER_FALLBACK.error("Encountered an HTTP error: %s", err) ++ except requests.RequestException as err: ++ BUILDLOGGER_FALLBACK.error("Encountered a network error: %s", err) ++ except: # pylint: disable=bare-except ++ BUILDLOGGER_FALLBACK.exception("Encountered an error.") ++ return 0 ++ ++ def _flush_buffer_with_lock(self, buf, close_called): ++ """Ensure all logging output has been flushed to the buildlogger server. ++ ++ If _append_logs() returns false, then the log messages are added ++ to a separate buffer and retried the next time flush() is ++ called. ++ """ ++ ++ self.retry_buffer.extend(buf) ++ ++ nb_sent = self._append_logs(self.retry_buffer) ++ if nb_sent: ++ self.retry_buffer = self.retry_buffer[nb_sent:] ++ if close_called and self.retry_buffer: ++ # The request to the logkeeper returned an error. We discard the log output rather than ++ # writing the messages to the fallback logkeeper to avoid putting additional pressure on ++ # the Evergreen database. ++ BUILDLOGGER_FALLBACK.warning( ++ "Failed to flush all log output (%d messages) to logkeeper.", len( ++ self.retry_buffer)) ++ ++ # We set a flag to indicate that we failed to flush all log output to logkeeper so ++ # resmoke.py can exit with a special return code. ++ set_log_output_incomplete() ++ ++ self.retry_buffer = [] ++ ++ ++class BuildloggerTestHandler(_BaseBuildloggerHandler): ++ """Buildlogger handler for the test logs.""" ++ ++ def __init__( # pylint: disable=too-many-arguments ++ self, build_config, build_id, test_id, capacity=_SEND_AFTER_LINES, ++ interval_secs=_SEND_AFTER_SECS): ++ """Initialize the buildlogger handler with the credentials, build id, and test id.""" ++ endpoint = APPEND_TEST_LOGS_ENDPOINT % { ++ "build_id": build_id, ++ "test_id": test_id, ++ } ++ _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) ++ ++ @_log_on_error ++ def _finish_test(self, failed=False): ++ """Send a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status.""" ++ self.post(self.endpoint, headers={ ++ "X-Sendlogs-Test-Done": "true", ++ "X-Sendlogs-Test-Failed": "true" if failed else "false", ++ }) ++ ++ def close(self): ++ """Close the buildlogger handler.""" ++ ++ _BaseBuildloggerHandler.close(self) ++ ++ # TODO: pass the test status (success/failure) to this method ++ self._finish_test() ++ ++ ++class BuildloggerGlobalHandler(_BaseBuildloggerHandler): ++ """Buildlogger handler for the global logs.""" ++ ++ def __init__(self, build_config, build_id, capacity=_SEND_AFTER_LINES, ++ interval_secs=_SEND_AFTER_SECS): ++ """Initialize the buildlogger handler with the credentials and build id.""" ++ endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id} ++ _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) ++ ++ ++class BuildloggerServer(object): ++ """A remote server to which build logs can be sent. ++ ++ It is used to retrieve handlers that can then be added to logger ++ instances to send the log to the servers. ++ """ ++ ++ @_log_on_error ++ def __init__(self): ++ """Initialize BuildloggerServer.""" ++ tmp_globals = {} ++ self.config = {} ++ execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config) ++ ++ # Rename "slavename" to "username" if present. ++ if "slavename" in self.config and "username" not in self.config: ++ self.config["username"] = self.config["slavename"] ++ del self.config["slavename"] ++ ++ # Rename "passwd" to "password" if present. ++ if "passwd" in self.config and "password" not in self.config: ++ self.config["password"] = self.config["passwd"] ++ del self.config["passwd"] ++ ++ @_log_on_error ++ def new_build_id(self, suffix): ++ """Return a new build id for sending global logs to.""" ++ username = self.config["username"] ++ password = self.config["password"] ++ builder = "%s_%s" % (self.config["builder"], suffix) ++ build_num = int(self.config["build_num"]) ++ ++ handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, username=username, ++ password=password, should_retry=True) ++ ++ response = handler.post(CREATE_BUILD_ENDPOINT, data={ ++ "builder": builder, ++ "buildnum": build_num, ++ "task_id": _config.EVERGREEN_TASK_ID, ++ }) ++ ++ return response["id"] ++ ++ @_log_on_error ++ def new_test_id(self, build_id, test_filename, test_command): ++ """Return a new test id for sending test logs to.""" ++ handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, ++ username=self.config["username"], ++ password=self.config["password"], should_retry=True) ++ ++ endpoint = CREATE_TEST_ENDPOINT % {"build_id": build_id} ++ response = handler.post( ++ endpoint, data={ ++ "test_filename": test_filename, ++ "command": test_command, ++ "phase": self.config.get("build_phase", "unknown"), ++ "task_id": _config.EVERGREEN_TASK_ID, ++ }) ++ ++ return response["id"] ++ ++ def get_global_handler(self, build_id, handler_info): ++ """Return the global handler.""" ++ return BuildloggerGlobalHandler(self.config, build_id, **handler_info) ++ ++ def get_test_handler(self, build_id, test_id, handler_info): ++ """Return the test handler.""" ++ return BuildloggerTestHandler(self.config, build_id, test_id, **handler_info) ++ ++ @staticmethod ++ def get_build_log_url(build_id): ++ """Return the build log URL.""" ++ base_url = _config.BUILDLOGGER_URL.rstrip("/") ++ endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id} ++ return "%s/%s" % (base_url, endpoint.strip("/")) ++ ++ @staticmethod ++ def get_test_log_url(build_id, test_id): ++ """Return the test log URL.""" ++ base_url = _config.BUILDLOGGER_URL.rstrip("/") ++ endpoint = APPEND_TEST_LOGS_ENDPOINT % {"build_id": build_id, "test_id": test_id} ++ return "%s/%s" % (base_url, endpoint.strip("/")) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/selector.py mongo-r4.0.23/buildscripts/resmokelib/selector.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/selector.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/selector.py 2021-03-17 01:21:05.956000000 +0000 +@@ -71,7 +71,7 @@ class TestFileExplorer(object): + A list of paths as a list(str). + """ + tests = [] +- with open(root_file_path, "rb") as filep: ++ with open(root_file_path, "r") as filep: + for test_path in filep: + test_path = test_path.strip() + tests.append(test_path) +@@ -310,7 +310,7 @@ def make_expression(conf): + elif isinstance(conf, dict): + if len(conf) != 1: + raise ValueError("Tag matching expressions should only contain one key") +- key = conf.keys()[0] ++ key = next(iter(conf.keys())) + value = conf[key] + if key == "$allOf": + return _AllOfExpression(_make_expression_list(value)) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/executor.py mongo-r4.0.23/buildscripts/resmokelib/testing/executor.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/executor.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/executor.py 2021-03-17 01:21:05.956000000 +0000 +@@ -62,7 +62,7 @@ class TestSuiteExecutor(object): # pyli + jobs_to_start = self.num_tests + + # Must be done after getting buildlogger configuration. +- self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)] ++ self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)] + + def run(self): + """Execute the test suite. +@@ -275,7 +275,7 @@ class TestSuiteExecutor(object): # pyli + queue.put(test_case) + + # Add sentinel value for each job to indicate when there are no more items to process. +- for _ in xrange(len(self._jobs)): ++ for _ in range(len(self._jobs)): + queue.put(None) + + return queue +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/interface.py mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/interface.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/interface.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/interface.py 2021-03-17 01:21:05.956000000 +0000 +@@ -3,6 +3,7 @@ + from __future__ import absolute_import + + import os.path ++import six + import time + + import pymongo +@@ -25,10 +26,10 @@ def make_fixture(class_name, *args, **kw + return _FIXTURES[class_name](*args, **kwargs) + + +-class Fixture(object): +- """Base class for all fixtures.""" +- +- __metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore ++class Fixture(six.with_metaclass(registry.make_registry_metaclass(_FIXTURES), object)): ++ """ ++ Base class for all fixtures. ++ """ + + # We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute + # is defined for all subclasses of Fixture. +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/replicaset.py mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/replicaset.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/replicaset.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/replicaset.py 2021-03-17 01:21:05.956000000 +0000 +@@ -77,11 +77,11 @@ class ReplicaSetFixture(interface.ReplFi + self.replset_name = self.mongod_options.get("replSet", "rs") + + if not self.nodes: +- for i in xrange(self.num_nodes): ++ for i in range(self.num_nodes): + node = self._new_mongod(i, self.replset_name) + self.nodes.append(node) + +- for i in xrange(self.num_nodes): ++ for i in range(self.num_nodes): + if self.linear_chain and i > 0: + self.nodes[i].mongod_options["set_parameters"][ + "failpoint.forceSyncSourceCandidate"] = { +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/shardedcluster.py mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/fixtures/shardedcluster.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/fixtures/shardedcluster.py 2021-03-17 01:21:05.956000000 +0000 +@@ -66,7 +66,7 @@ class ShardedClusterFixture(interface.Fi + self.configsvr.setup() + + if not self.shards: +- for i in xrange(self.num_shards): ++ for i in range(self.num_shards): + if self.num_rs_nodes_per_shard is None: + shard = self._new_standalone_shard(i) + elif isinstance(self.num_rs_nodes_per_shard, int): +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/hooks/interface.py mongo-r4.0.23/buildscripts/resmokelib/testing/hooks/interface.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/hooks/interface.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/hooks/interface.py 2021-03-17 01:21:05.956000000 +0000 +@@ -9,6 +9,8 @@ from ... import errors + from ...logging import loggers + from ...utils import registry + ++import six ++ + _HOOKS = {} # type: ignore + + +@@ -21,11 +23,8 @@ def make_hook(class_name, *args, **kwarg + return _HOOKS[class_name](*args, **kwargs) + + +-class Hook(object): ++class Hook(six.with_metaclass(registry.make_registry_metaclass(_HOOKS), object)): + """Common interface all Hooks will inherit from.""" +- +- __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore +- + REGISTERED_NAME = registry.LEAVE_UNREGISTERED + + def __init__(self, hook_logger, fixture, description): +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/suite.py mongo-r4.0.23/buildscripts/resmokelib/testing/suite.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/suite.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/suite.py 2021-03-17 01:21:05.956000000 +0000 +@@ -234,7 +234,7 @@ class Suite(object): # pylint: disable= + sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken)) + + combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) +- for iteration in xrange(num_iterations): ++ for iteration in range(num_iterations): + # Summarize each execution as a bulleted list of results. + bulleter_sb = [] + summary = self._summarize_report(reports[iteration], start_times[iteration], +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/summary.py mongo-r4.0.23/buildscripts/resmokelib/testing/summary.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/summary.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/summary.py 2021-03-17 01:21:05.956000000 +0000 +@@ -12,6 +12,6 @@ Summary = collections.namedtuple( + def combine(summary1, summary2): + """Return a summary representing the sum of 'summary1' and 'summary2'.""" + args = [] +- for i in xrange(len(Summary._fields)): ++ for i in range(len(Summary._fields)): + args.append(summary1[i] + summary2[i]) + return Summary._make(args) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/testcases/interface.py mongo-r4.0.23/buildscripts/resmokelib/testing/testcases/interface.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/testcases/interface.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/testcases/interface.py 2021-03-17 01:21:05.956000000 +0000 +@@ -7,6 +7,7 @@ from __future__ import absolute_import + + import os + import os.path ++import six + import unittest + + from ... import logging +@@ -22,11 +23,8 @@ def make_test_case(test_kind, *args, **k + return _TEST_CASES[test_kind](*args, **kwargs) + + +-class TestCase(unittest.TestCase): ++class TestCase(six.with_metaclass(registry.make_registry_metaclass(_TEST_CASES), unittest.TestCase)): + """A test case to execute.""" +- +- __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore +- + REGISTERED_NAME = registry.LEAVE_UNREGISTERED + + def __init__(self, logger, test_kind, test_name): +@@ -36,10 +34,10 @@ class TestCase(unittest.TestCase): + if not isinstance(logger, logging.Logger): + raise TypeError("logger must be a Logger instance") + +- if not isinstance(test_kind, basestring): ++ if not isinstance(test_kind, str): + raise TypeError("test_kind must be a string") + +- if not isinstance(test_name, basestring): ++ if not isinstance(test_name, str): + raise TypeError("test_name must be a string") + + # When the TestCase is created by the TestSuiteExecutor (through a call to make_test_case()) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/testing/testcases/jstest.py mongo-r4.0.23/buildscripts/resmokelib/testing/testcases/jstest.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/testing/testcases/jstest.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/testing/testcases/jstest.py 2021-03-17 01:21:05.956000000 +0000 +@@ -199,7 +199,7 @@ class JSTestCase(interface.ProcessTestCa + test_cases = [] + try: + # If there are multiple clients, make a new thread for each client. +- for thread_id in xrange(self.num_clients): ++ for thread_id in range(self.num_clients): + logger = self.logger.new_test_thread_logger(self.test_kind, str(thread_id)) + test_case = self._create_test_case_for_thread(logger, thread_id) + test_cases.append(test_case) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/archival.py mongo-r4.0.23/buildscripts/resmokelib/utils/archival.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/archival.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/archival.py 2021-03-17 01:21:05.956000000 +0000 +@@ -1,8 +1,13 @@ + """Archival utility.""" + +-from __future__ import absolute_import + +-import Queue ++ ++try: ++ import queue ++except ImportError: ++ #Python 2 ++ import Queue as queue ++ + import collections + import json + import math +@@ -45,7 +50,7 @@ def file_list_size(files): + def directory_size(directory): + """Return size (in bytes) of files in 'directory' tree.""" + dir_bytes = 0 +- for root_dir, _, files in os.walk(unicode(directory)): ++ for root_dir, _, files in os.walk(str(directory)): + for name in files: + full_name = os.path.join(root_dir, name) + try: +@@ -103,7 +108,7 @@ class Archival(object): # pylint: disab + self._lock = threading.Lock() + + # Start the worker thread to update the 'archival_json_file'. +- self._archive_file_queue = Queue.Queue() ++ self._archive_file_queue = queue.Queue() + self._archive_file_worker = threading.Thread(target=self._update_archive_file_wkr, + args=(self._archive_file_queue, + logger), name="archive_file_worker") +@@ -115,7 +120,7 @@ class Archival(object): # pylint: disab + self.s3_client = s3_client + + # Start the worker thread which uploads the archive. +- self._upload_queue = Queue.Queue() ++ self._upload_queue = queue.Queue() + self._upload_worker = threading.Thread(target=self._upload_to_s3_wkr, + args=(self._upload_queue, self._archive_file_queue, + logger, self.s3_client), name="upload_worker") +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/globstar.py mongo-r4.0.23/buildscripts/resmokelib/utils/globstar.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/globstar.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/globstar.py 2021-03-17 01:21:05.956000000 +0000 +@@ -134,7 +134,7 @@ def _list_dir(pathname): + """ + + try: +- (_root, dirs, files) = os.walk(pathname).next() ++ (_root, dirs, files) = next(os.walk(pathname)) + return (dirs, files) + except StopIteration: + return None # 'pathname' directory does not exist +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/__init__.py mongo-r4.0.23/buildscripts/resmokelib/utils/__init__.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/__init__.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/__init__.py 2021-03-17 01:21:05.956000000 +0000 +@@ -48,10 +48,10 @@ def rmtree(path, **kwargs): + See https://github.com/pypa/setuptools/issues/706. + """ + if is_windows(): +- if not isinstance(path, unicode): +- path = unicode(path, "utf-8") ++ if not isinstance(path, str): ++ path = str(path, "utf-8") + else: +- if isinstance(path, unicode): ++ if isinstance(path, str): + path = path.encode("utf-8") + shutil.rmtree(path, **kwargs) + +@@ -71,12 +71,12 @@ def remove_if_exists(path): + + def is_string_list(lst): + """Return true if 'lst' is a list of strings, and false otherwise.""" +- return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst) ++ return isinstance(lst, list) and all(isinstance(x, str) for x in lst) + + + def is_string_set(value): + """Return true if 'value' is a set of strings, and false otherwise.""" +- return isinstance(value, set) and all(isinstance(x, basestring) for x in value) ++ return isinstance(value, set) and all(isinstance(x, str) for x in value) + + + def is_js_file(filename): +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/__init__.py.orig mongo-r4.0.23/buildscripts/resmokelib/utils/__init__.py.orig +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/__init__.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/__init__.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,112 @@ ++"""Helper functions.""" ++ ++from __future__ import absolute_import ++from __future__ import print_function ++ ++import contextlib ++import os.path ++import shutil ++import sys ++ ++import yaml ++ ++from . import archival ++ ++ ++@contextlib.contextmanager ++def open_or_use_stdout(filename): ++ """Open the specified file for writing, or returns sys.stdout if filename is "-".""" ++ ++ if filename == "-": ++ yield sys.stdout ++ return ++ ++ line_buffered = 1 ++ try: ++ fp = open(filename, "w", line_buffered) ++ except IOError: ++ print("Could not open file {}".format(filename), file=sys.stderr) ++ sys.exit(1) ++ ++ try: ++ yield fp ++ finally: ++ fp.close() ++ ++ ++def default_if_none(value, default): ++ """Set default if value is 'None'.""" ++ return value if value is not None else default ++ ++ ++def rmtree(path, **kwargs): ++ """Wrap shutil.rmtreee. ++ ++ Use a UTF-8 unicode path if Windows. ++ See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8. ++ Use a bytes path to rmtree, otherwise. ++ See https://github.com/pypa/setuptools/issues/706. ++ """ ++ if is_windows(): ++ if not isinstance(path, unicode): ++ path = unicode(path, "utf-8") ++ else: ++ if isinstance(path, unicode): ++ path = path.encode("utf-8") ++ shutil.rmtree(path, **kwargs) ++ ++ ++def is_windows(): ++ """Return True if Windows.""" ++ return sys.platform.startswith("win32") or sys.platform.startswith("cygwin") ++ ++ ++def remove_if_exists(path): ++ """Remove path if it exists.""" ++ try: ++ os.remove(path) ++ except OSError: ++ pass ++ ++ ++def is_string_list(lst): ++ """Return true if 'lst' is a list of strings, and false otherwise.""" ++ return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst) ++ ++ ++def is_string_set(value): ++ """Return true if 'value' is a set of strings, and false otherwise.""" ++ return isinstance(value, set) and all(isinstance(x, basestring) for x in value) ++ ++ ++def is_js_file(filename): ++ """Return true if 'filename' ends in .js, and false otherwise.""" ++ return os.path.splitext(filename)[1] == ".js" ++ ++ ++def is_yaml_file(filename): ++ """Return true if 'filename' ends in .yml or .yaml, and false otherwise.""" ++ return os.path.splitext(filename)[1] in (".yaml", ".yml") ++ ++ ++def load_yaml_file(filename): ++ """Attempt to read 'filename' as YAML.""" ++ try: ++ with open(filename, "r") as fp: ++ return yaml.safe_load(fp) ++ except yaml.YAMLError as err: ++ raise ValueError("File '%s' contained invalid YAML: %s" % (filename, err)) ++ ++ ++def dump_yaml(value): ++ """Return 'value' formatted as YAML.""" ++ # Use block (indented) style for formatting YAML. ++ return yaml.safe_dump(value, default_flow_style=False).rstrip() ++ ++ ++def load_yaml(value): ++ """Attempt to parse 'value' as YAML.""" ++ try: ++ return yaml.safe_load(value) ++ except yaml.YAMLError as err: ++ raise ValueError("Attempted to parse invalid YAML value '%s': %s" % (value, err)) +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/jscomment.py mongo-r4.0.23/buildscripts/resmokelib/utils/jscomment.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/jscomment.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/jscomment.py 2021-03-17 01:21:05.956000000 +0000 +@@ -36,7 +36,7 @@ def get_tags(pathname): + # TODO: it might be worth supporting the block (indented) style of YAML lists in + # addition to the flow (bracketed) style + tags = yaml.safe_load(_strip_jscomments(match.group(1))) +- if not isinstance(tags, list) and all(isinstance(tag, basestring) for tag in tags): ++ if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): + raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) + return tags + except yaml.YAMLError as err: +diff -upNr mongo-r4.0.23.orig/buildscripts/resmokelib/utils/queue.py mongo-r4.0.23/buildscripts/resmokelib/utils/queue.py +--- mongo-r4.0.23.orig/buildscripts/resmokelib/utils/queue.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/resmokelib/utils/queue.py 2021-03-17 01:21:05.956000000 +0000 +@@ -8,7 +8,12 @@ See https://bugs.python.org/issue1167930 + + from __future__ import absolute_import + +-import Queue as _Queue ++try: ++ import queue as _Queue ++except ImportError: ++ #Python 2 ++ import Queue as _Queue ++ + import time + + # Exception that is raised when get_nowait() is called on an empty Queue. +diff -upNr mongo-r4.0.23.orig/buildscripts/utils.py mongo-r4.0.23/buildscripts/utils.py +--- mongo-r4.0.23.orig/buildscripts/utils.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/buildscripts/utils.py 2021-03-17 01:21:05.956000000 +0000 +@@ -139,8 +139,8 @@ def find_python(min_version=(2, 5)): + # In case the version of Python is somehow missing sys.version_info or sys.executable. + pass + +- version = re.compile(r"[Pp]ython ([\d\.]+)", re.MULTILINE) +- binaries = ("python27", "python2.7", "python26", "python2.6", "python25", "python2.5", "python") ++ version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE) ++ binaries = ('python3', 'python27', 'python2.7', 'python26', 'python2.6', 'python25', 'python2.5', 'python') + for binary in binaries: + try: + out, err = subprocess.Popen([binary, "-V"], stdout=subprocess.PIPE, +@@ -166,7 +166,7 @@ def replace_with_repr(unicode_error): + # repr() of the offending bytes into the decoded string + # at the position they occurred + offender = unicode_error.object[unicode_error.start:unicode_error.end] +- return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end) ++ return (str(repr(offender).strip("'").strip('"')), unicode_error.end) + + + codecs.register_error("repr", replace_with_repr) +diff -upNr mongo-r4.0.23.orig/SConstruct mongo-r4.0.23/SConstruct +--- mongo-r4.0.23.orig/SConstruct 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/SConstruct 2021-03-17 01:21:05.952000000 +0000 +@@ -435,7 +435,7 @@ win_version_min_choices = { + } + + add_option('win-version-min', +- choices=win_version_min_choices.keys(), ++ choices=list(win_version_min_choices.keys()), + default=None, + help='minimum Windows version to support', + type='choice', +@@ -563,7 +563,7 @@ except ValueError as e: + def variable_shlex_converter(val): + # If the argument is something other than a string, propogate + # it literally. +- if not isinstance(val, basestring): ++ if not isinstance(val, str): + return val + parse_mode = get_option('variable-parse-mode') + if parse_mode == 'auto': +@@ -906,7 +906,7 @@ SConsignFile(str(sconsDataDir.File('scon + def printLocalInfo(): + import sys, SCons + print( "scons version: " + SCons.__version__ ) +- print( "python version: " + " ".join( [ `i` for i in sys.version_info ] ) ) ++ print( "python version: " + " ".join( [ str(i) for i in sys.version_info ] ) ) + + printLocalInfo() + +@@ -2095,7 +2095,7 @@ def doConfigure(myenv): + # to make them real errors. + cloned.Append(CCFLAGS=['-Werror']) + conf = Configure(cloned, help=False, custom_tests = { +- 'CheckFlag' : lambda(ctx) : CheckFlagTest(ctx, tool, extension, flag) ++ 'CheckFlag' : lambda ctx : CheckFlagTest(ctx, tool, extension, flag) + }) + available = conf.CheckFlag() + conf.Finish() +@@ -2580,7 +2580,7 @@ def doConfigure(myenv): + # Select those unique black files that are associated with the + # currently enabled sanitizers, but filter out those that are + # zero length. +- blackfiles = {v for (k, v) in blackfiles_map.iteritems() if k in sanitizer_list} ++ blackfiles = {v for (k, v) in blackfiles_map.items() if k in sanitizer_list} + blackfiles = [f for f in blackfiles if os.stat(f.path).st_size != 0] + + # Filter out any blacklist options that the toolchain doesn't support. +@@ -3423,7 +3423,7 @@ def doConfigure(myenv): + + outputIndex = next((idx for idx in [0,1] if conf.CheckAltivecVbpermqOutput(idx)), None) + if outputIndex is not None: +- conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) + else: + myenv.ConfError("Running on ppc64le, but can't find a correct vec_vbpermq output index. Compiler or platform not supported") + +diff -upNr mongo-r4.0.23.orig/SConstruct.orig mongo-r4.0.23/SConstruct.orig +--- mongo-r4.0.23.orig/SConstruct.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/SConstruct.orig 2021-03-17 01:19:50.796000000 +0000 +@@ -0,0 +1,3781 @@ ++# -*- mode: python; -*- ++import atexit ++import copy ++import datetime ++import errno ++import json ++import os ++import re ++import shlex ++import shutil ++import stat ++import subprocess ++import sys ++import textwrap ++import uuid ++import multiprocessing ++ ++import SCons ++ ++# This must be first, even before EnsureSConsVersion, if ++# we are to avoid bulk loading all tools in the DefaultEnvironment. ++DefaultEnvironment(tools=[]) ++ ++# These come from site_scons/mongo. Import these things ++# after calling DefaultEnvironment, for the sake of paranoia. ++import mongo ++import mongo.platform as mongo_platform ++import mongo.toolchain as mongo_toolchain ++import mongo.generators as mongo_generators ++ ++EnsurePythonVersion(2, 7) ++EnsureSConsVersion(2, 5) ++ ++from buildscripts import utils ++from buildscripts import moduleconfig ++ ++import libdeps ++ ++atexit.register(mongo.print_build_failures) ++ ++def add_option(name, **kwargs): ++ ++ if 'dest' not in kwargs: ++ kwargs['dest'] = name ++ ++ if 'metavar' not in kwargs and kwargs.get('type', None) == 'choice': ++ kwargs['metavar'] = '[' + '|'.join(kwargs['choices']) + ']' ++ ++ AddOption('--' + name, **kwargs) ++ ++def get_option(name): ++ return GetOption(name) ++ ++def has_option(name): ++ optval = GetOption(name) ++ # Options with nargs=0 are true when their value is the empty tuple. Otherwise, ++ # if the value is falsish (empty string, None, etc.), coerce to False. ++ return True if optval == () else bool(optval) ++ ++def use_system_version_of_library(name): ++ return has_option('use-system-all') or has_option('use-system-' + name) ++ ++# Returns true if we have been configured to use a system version of any C++ library. If you ++# add a new C++ library dependency that may be shimmed out to the system, add it to the below ++# list. ++def using_system_version_of_cxx_libraries(): ++ cxx_library_names = ["tcmalloc", "boost"] ++ return True in [use_system_version_of_library(x) for x in cxx_library_names] ++ ++def make_variant_dir_generator(): ++ memoized_variant_dir = [False] ++ def generate_variant_dir(target, source, env, for_signature): ++ if not memoized_variant_dir[0]: ++ memoized_variant_dir[0] = env.subst('$BUILD_ROOT/$VARIANT_DIR') ++ return memoized_variant_dir[0] ++ return generate_variant_dir ++ ++ ++# Always randomize the build order to shake out missing edges, and to help the cache: ++# http://scons.org/doc/production/HTML/scons-user/ch24s06.html ++SetOption('random', 1) ++ ++# Options TODOs: ++# ++# - We should either alphabetize the entire list of options, or split them into logical groups ++# with clear boundaries, and then alphabetize the groups. There is no way in SCons though to ++# inform it of options groups. ++# ++# - Many of these options are currently only either present or absent. This is not good for ++# scripting the build invocation because it means you need to interpolate in the presence of ++# the whole option. It is better to make all options take an optional on/off or true/false ++# using the nargs='const' mechanism. ++# ++ ++add_option('prefix', ++ default='$BUILD_ROOT/install', ++ help='installation prefix', ++) ++ ++add_option('install-mode', ++ choices=['legacy', 'hygienic'], ++ default='legacy', ++ help='select type of installation', ++ nargs=1, ++ type='choice', ++) ++ ++add_option('nostrip', ++ help='do not strip installed binaries', ++ nargs=0, ++) ++ ++add_option('build-dir', ++ default='#build', ++ help='build output directory', ++) ++ ++add_option('release', ++ help='release build', ++ nargs=0, ++) ++ ++add_option('lto', ++ help='enable link time optimizations (experimental, except with MSVC)', ++ nargs=0, ++) ++ ++add_option('dynamic-windows', ++ help='dynamically link on Windows', ++ nargs=0, ++) ++ ++add_option('endian', ++ choices=['big', 'little', 'auto'], ++ default='auto', ++ help='endianness of target platform', ++ nargs=1, ++ type='choice', ++) ++ ++add_option('disable-minimum-compiler-version-enforcement', ++ help='allow use of unsupported older compilers (NEVER for production builds)', ++ nargs=0, ++) ++ ++add_option('ssl', ++ help='Enable SSL', ++ nargs=0 ++) ++ ++add_option('ssl-provider', ++ choices=['auto', 'openssl', 'native'], ++ default='auto', ++ help='Select the SSL engine to use', ++ nargs=1, ++ type='choice', ++) ++ ++add_option('mmapv1', ++ choices=['auto', 'on', 'off'], ++ const='on', ++ default='auto', ++ help='Enable MMapV1', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('wiredtiger', ++ choices=['on', 'off'], ++ const='on', ++ default='on', ++ help='Enable wiredtiger', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('mobile-se', ++ choices=['on', 'off'], ++ const='on', ++ default='off', ++ help='Enable Mobile Storage Engine', ++ nargs='?', ++ type='choice', ++) ++ ++js_engine_choices = ['mozjs', 'none'] ++add_option('js-engine', ++ choices=js_engine_choices, ++ default=js_engine_choices[0], ++ help='JavaScript scripting engine implementation', ++ type='choice', ++) ++ ++add_option('server-js', ++ choices=['on', 'off'], ++ default='on', ++ help='Build mongod without JavaScript support', ++ type='choice', ++) ++ ++add_option('libc++', ++ help='use libc++ (experimental, requires clang)', ++ nargs=0, ++) ++ ++add_option('use-glibcxx-debug', ++ help='Enable the glibc++ debug implementations of the C++ standard libary', ++ nargs=0, ++) ++ ++add_option('noshell', ++ help="don't build shell", ++ nargs=0, ++) ++ ++add_option('safeshell', ++ help="don't let shell scripts run programs (still, don't run untrusted scripts)", ++ nargs=0, ++) ++ ++add_option('dbg', ++ choices=['on', 'off'], ++ const='on', ++ default='off', ++ help='Enable runtime debugging checks', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('separate-debug', ++ choices=['on', 'off'], ++ const='on', ++ default='off', ++ help='Produce separate debug files (only effective in --install-mode=hygienic)', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('spider-monkey-dbg', ++ choices=['on', 'off'], ++ const='on', ++ default='off', ++ help='Enable SpiderMonkey debug mode', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('opt', ++ choices=['on', 'size', 'off'], ++ const='on', ++ help='Enable compile-time optimization', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('sanitize', ++ help='enable selected sanitizers', ++ metavar='san1,san2,...sanN', ++) ++ ++add_option('llvm-symbolizer', ++ default='llvm-symbolizer', ++ help='name of (or path to) the LLVM symbolizer', ++) ++ ++add_option('durableDefaultOn', ++ help='have durable default to on', ++ nargs=0, ++) ++ ++add_option('allocator', ++ choices=["auto", "system", "tcmalloc"], ++ default="auto", ++ help='allocator to use (use "auto" for best choice for current platform)', ++ type='choice', ++) ++ ++add_option('gdbserver', ++ help='build in gdb server support', ++ nargs=0, ++) ++ ++add_option('gcov', ++ help='compile with flags for gcov', ++ nargs=0, ++) ++ ++add_option('enable-free-mon', ++ choices=["auto", "on", "off"], ++ default="auto", ++ help='Disable support for Free Monitoring to avoid HTTP client library dependencies', ++ type='choice', ++) ++ ++add_option('use-sasl-client', ++ help='Support SASL authentication in the client library', ++ nargs=0, ++) ++ ++add_option('use-system-tcmalloc', ++ help='use system version of tcmalloc library', ++ nargs=0, ++) ++ ++add_option('use-system-pcre', ++ help='use system version of pcre library', ++ nargs=0, ++) ++ ++add_option('use-system-wiredtiger', ++ help='use system version of wiredtiger library', ++ nargs=0, ++) ++ ++add_option('system-boost-lib-search-suffixes', ++ help='Comma delimited sequence of boost library suffixes to search', ++) ++ ++add_option('use-system-boost', ++ help='use system version of boost libraries', ++ nargs=0, ++) ++ ++add_option('use-system-snappy', ++ help='use system version of snappy library', ++ nargs=0, ++) ++ ++add_option('use-system-valgrind', ++ help='use system version of valgrind library', ++ nargs=0, ++) ++ ++add_option('use-system-google-benchmark', ++ help='use system version of Google benchmark library', ++ nargs=0, ++) ++ ++add_option('use-system-zlib', ++ help='use system version of zlib library', ++ nargs=0, ++) ++ ++add_option('use-system-sqlite', ++ help='use system version of sqlite library', ++ nargs=0, ++) ++ ++add_option('use-system-stemmer', ++ help='use system version of stemmer', ++ nargs=0) ++ ++add_option('use-system-yaml', ++ help='use system version of yaml', ++ nargs=0, ++) ++ ++add_option('use-system-asio', ++ help="use system version of ASIO", ++ nargs=0, ++) ++ ++add_option('use-system-icu', ++ help="use system version of ICU", ++ nargs=0, ++) ++ ++add_option('use-system-intel_decimal128', ++ help='use system version of intel decimal128', ++ nargs=0, ++) ++ ++add_option('use-system-mongo-c', ++ choices=['on', 'off', 'auto'], ++ const='on', ++ default="auto", ++ help="use system version of the mongo-c-driver (auto will use it if it's found)", ++ nargs='?', ++ type='choice', ++) ++ ++add_option('use-system-all', ++ help='use all system libraries', ++ nargs=0, ++) ++ ++add_option('use-new-tools', ++ help='put new tools in the tarball', ++ nargs=0, ++) ++ ++add_option('build-mongoreplay', ++ help='when building with --use-new-tools, build mongoreplay ( requires pcap dev )', ++ nargs=1, ++) ++ ++add_option('use-cpu-profiler', ++ help='Link against the google-perftools profiler library', ++ nargs=0, ++) ++ ++add_option('build-fast-and-loose', ++ choices=['on', 'off', 'auto'], ++ const='on', ++ default='auto', ++ help='looser dependency checking', ++ nargs='?', ++ type='choice', ++) ++ ++add_option('disable-warnings-as-errors', ++ help="Don't add -Werror to compiler command line", ++ nargs=0, ++) ++ ++add_option('detect-odr-violations', ++ help="Have the linker try to detect ODR violations, if supported", ++ nargs=0, ++) ++ ++add_option('variables-help', ++ help='Print the help text for SCons variables', ++ nargs=0, ++) ++ ++add_option('osx-version-min', ++ help='minimum OS X version to support', ++) ++ ++win_version_min_choices = { ++ 'win7' : ('0601', '0000'), ++ 'ws08r2' : ('0601', '0000'), ++ 'win8' : ('0602', '0000'), ++ 'win81' : ('0603', '0000'), ++} ++ ++add_option('win-version-min', ++ choices=win_version_min_choices.keys(), ++ default=None, ++ help='minimum Windows version to support', ++ type='choice', ++) ++ ++add_option('cache', ++ choices=["all", "nolinked"], ++ const='all', ++ help='Use an object cache rather than a per-build variant directory (experimental)', ++ nargs='?', ++) ++ ++add_option('cache-dir', ++ default='$BUILD_ROOT/scons/cache', ++ help='Specify the directory to use for caching objects if --cache is in use', ++) ++ ++add_option("cxx-std", ++ choices=["14"], ++ default="14", ++ help="Select the C++ langauge standard to build with", ++) ++ ++def find_mongo_custom_variables(): ++ files = [] ++ for path in sys.path: ++ probe = os.path.join(path, 'mongo_custom_variables.py') ++ if os.path.isfile(probe): ++ files.append(probe) ++ return files ++ ++add_option('variables-files', ++ default=find_mongo_custom_variables(), ++ help="Specify variables files to load", ++) ++ ++link_model_choices = ['auto', 'object', 'static', 'dynamic', 'dynamic-strict', 'dynamic-sdk'] ++add_option('link-model', ++ choices=link_model_choices, ++ default='auto', ++ help='Select the linking model for the project', ++ type='choice' ++) ++ ++variable_parse_mode_choices=['auto', 'posix', 'other'] ++add_option('variable-parse-mode', ++ choices=variable_parse_mode_choices, ++ default=variable_parse_mode_choices[0], ++ help='Select which parsing mode is used to interpret command line variables', ++ type='choice', ++) ++ ++add_option('modules', ++ help="Comma-separated list of modules to build. Empty means none. Default is all.", ++) ++ ++add_option('runtime-hardening', ++ choices=["on", "off"], ++ default="on", ++ help="Enable runtime hardening features (e.g. stack smash protection)", ++ type='choice', ++) ++ ++add_option('use-s390x-crc32', ++ choices=["on", "off"], ++ default="on", ++ help="Enable CRC32 hardware accelaration on s390x", ++ type='choice', ++) ++ ++add_option('git-decider', ++ choices=["on", "off"], ++ const='on', ++ default="off", ++ help="Use git metadata for out-of-date detection for source files", ++ nargs='?', ++ type="choice", ++) ++ ++add_option('android-toolchain-path', ++ default=None, ++ help="Android NDK standalone toolchain path. Required when using --variables-files=etc/scons/android_ndk.vars", ++) ++ ++add_option('msvc-debugging-format', ++ choices=["codeview", "pdb"], ++ default="codeview", ++ help='Debugging format in debug builds using msvc. Codeview (/Z7) or Program database (/Zi). Default is codeview.', ++ type='choice', ++) ++ ++add_option('jlink', ++ help="Limit link concurrency to given value", ++ const=0.5, ++ default=None, ++ nargs='?', ++ type=float) ++ ++try: ++ with open("version.json", "r") as version_fp: ++ version_data = json.load(version_fp) ++ ++ if 'version' not in version_data: ++ print("version.json does not contain a version string") ++ Exit(1) ++ if 'githash' not in version_data: ++ version_data['githash'] = utils.get_git_version() ++ ++except IOError as e: ++ # If the file error wasn't because the file is missing, error out ++ if e.errno != errno.ENOENT: ++ print("Error opening version.json: {0}".format(e.strerror)) ++ Exit(1) ++ ++ version_data = { ++ 'version': utils.get_git_describe()[1:], ++ 'githash': utils.get_git_version(), ++ } ++ ++except ValueError as e: ++ print("Error decoding version.json: {0}".format(e)) ++ Exit(1) ++ ++# Setup the command-line variables ++def variable_shlex_converter(val): ++ # If the argument is something other than a string, propogate ++ # it literally. ++ if not isinstance(val, basestring): ++ return val ++ parse_mode = get_option('variable-parse-mode') ++ if parse_mode == 'auto': ++ parse_mode = 'other' if mongo_platform.is_running_os('windows') else 'posix' ++ return shlex.split(val, posix=(parse_mode == 'posix')) ++ ++def variable_arch_converter(val): ++ arches = { ++ 'x86_64': 'x86_64', ++ 'amd64': 'x86_64', ++ 'emt64': 'x86_64', ++ 'x86': 'i386', ++ } ++ val = val.lower() ++ ++ if val in arches: ++ return arches[val] ++ ++ # Uname returns a bunch of possible x86's on Linux. ++ # Check whether the value is an i[3456]86 processor. ++ if re.match(r'^i[3-6]86$', val): ++ return 'i386' ++ ++ # Return whatever val is passed in - hopefully it's legit ++ return val ++ ++# The Scons 'default' tool enables a lot of tools that we don't actually need to enable. ++# On platforms like Solaris, it actually does the wrong thing by enabling the sunstudio ++# toolchain first. As such it is simpler and more efficient to manually load the precise ++# set of tools we need for each platform. ++# If we aren't on a platform where we know the minimal set of tools, we fall back to loading ++# the 'default' tool. ++def decide_platform_tools(): ++ if mongo_platform.is_running_os('windows'): ++ # we only support MS toolchain on windows ++ return ['msvc', 'mslink', 'mslib', 'masm'] ++ elif mongo_platform.is_running_os('linux', 'solaris'): ++ return ['gcc', 'g++', 'gnulink', 'ar', 'gas'] ++ elif mongo_platform.is_running_os('darwin'): ++ return ['gcc', 'g++', 'applelink', 'ar', 'libtool', 'as', 'xcode'] ++ else: ++ return ["default"] ++ ++def variable_tools_converter(val): ++ tool_list = shlex.split(val) ++ return tool_list + [ ++ "distsrc", ++ "gziptool", ++ 'idl_tool', ++ "jsheader", ++ "mongo_benchmark", ++ "mongo_integrationtest", ++ "mongo_unittest", ++ "textfile", ++ ] ++ ++def variable_distsrc_converter(val): ++ if not val.endswith("/"): ++ return val + "/" ++ return val ++ ++variables_files = variable_shlex_converter(get_option('variables-files')) ++for file in variables_files: ++ print("Using variable customization file %s" % file) ++ ++env_vars = Variables( ++ files=variables_files, ++ args=ARGUMENTS ++) ++ ++sconsflags = os.environ.get('SCONSFLAGS', None) ++if sconsflags: ++ print("Using SCONSFLAGS environment variable arguments: %s" % sconsflags) ++ ++env_vars.Add('ABIDW', ++ help="Configures the path to the 'abidw' (a libabigail) utility") ++ ++env_vars.Add('AR', ++ help='Sets path for the archiver') ++ ++env_vars.Add('ARFLAGS', ++ help='Sets flags for the archiver', ++ converter=variable_shlex_converter) ++ ++env_vars.Add( ++ 'CACHE_SIZE', ++ help='Maximum size of the cache (in gigabytes)', ++ default=32, ++ converter=lambda x:int(x) ++) ++ ++env_vars.Add( ++ 'CACHE_PRUNE_TARGET', ++ help='Maximum percent in-use in cache after pruning', ++ default=66, ++ converter=lambda x:int(x) ++) ++ ++env_vars.Add('CC', ++ help='Select the C compiler to use') ++ ++env_vars.Add('CCFLAGS', ++ help='Sets flags for the C and C++ compiler', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('CFLAGS', ++ help='Sets flags for the C compiler', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('CPPDEFINES', ++ help='Sets pre-processor definitions for C and C++', ++ converter=variable_shlex_converter, ++ default=[]) ++ ++env_vars.Add('CPPPATH', ++ help='Adds paths to the preprocessor search path', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('CXX', ++ help='Select the C++ compiler to use') ++ ++env_vars.Add('CXXFLAGS', ++ help='Sets flags for the C++ compiler', ++ converter=variable_shlex_converter) ++ ++# Note: This probably is only really meaningful when configured via a variables file. It will ++# also override whatever the SCons platform defaults would be. ++env_vars.Add('ENV', ++ help='Sets the environment for subprocesses') ++ ++env_vars.Add('FRAMEWORKPATH', ++ help='Adds paths to the linker search path for darwin frameworks', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('FRAMEWORKS', ++ help='Adds extra darwin frameworks to link against', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('HOST_ARCH', ++ help='Sets the native architecture of the compiler', ++ converter=variable_arch_converter, ++ default=None) ++ ++env_vars.Add('ICECC', ++ help='Tell SCons where icecream icecc tool is') ++ ++env_vars.Add('ICERUN', ++ help='Tell SCons where icecream icerun tool is') ++ ++env_vars.Add('ICECC_CREATE_ENV', ++ help='Tell SCons where icecc-create-env tool is', ++ default='buildscripts/icecc_create_env') ++ ++env_vars.Add('ICECC_SCHEDULER', ++ help='Tell ICECC where the sceduler daemon is running') ++ ++env_vars.Add('LIBPATH', ++ help='Adds paths to the linker search path', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('LIBS', ++ help='Adds extra libraries to link against', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('LINKFLAGS', ++ help='Sets flags for the linker', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('MAXLINELENGTH', ++ help='Maximum line length before using temp files', ++ # This is very small, but appears to be the least upper bound ++ # across our platforms. ++ # ++ # See https://support.microsoft.com/en-us/help/830473/command-prompt-cmd.-exe-command-line-string-limitation ++ default=4095) ++ ++# Note: This is only really meaningful when configured via a variables file. See the ++# default_buildinfo_environment_data() function for examples of how to use this. ++env_vars.Add('MONGO_BUILDINFO_ENVIRONMENT_DATA', ++ help='Sets the info returned from the buildInfo command and --version command-line flag', ++ default=mongo_generators.default_buildinfo_environment_data()) ++ ++# Exposed to be able to cross compile Android/*nix from Windows without ending up with the .exe suffix. ++env_vars.Add('PROGSUFFIX', ++ help='Sets the suffix for built executable files') ++ ++env_vars.Add('MONGO_DIST_SRC_PREFIX', ++ help='Sets the prefix for files in the source distribution archive', ++ converter=variable_distsrc_converter, ++ default="mongodb-src-r${MONGO_VERSION}") ++ ++env_vars.Add('MONGO_DISTARCH', ++ help='Adds a string representing the target processor architecture to the dist archive', ++ default='$TARGET_ARCH') ++ ++env_vars.Add('MONGO_DISTMOD', ++ help='Adds a string that will be embedded in the dist archive naming', ++ default='') ++ ++env_vars.Add('MONGO_DISTNAME', ++ help='Sets the version string to be used in dist archive naming', ++ default='$MONGO_VERSION') ++ ++def validate_mongo_version(key, val, env): ++ regex = r'^(\d+)\.(\d+)\.(\d+)-?((?:(rc)(\d+))?.*)?' ++ if not re.match(regex, val): ++ print("Invalid MONGO_VERSION '{}', or could not derive from version.json or git metadata. Please add a conforming MONGO_VERSION=x.y.z[-extra] as an argument to SCons".format(val)) ++ Exit(1) ++ ++env_vars.Add('MONGO_VERSION', ++ help='Sets the version string for MongoDB', ++ default=version_data['version'], ++ validator=validate_mongo_version) ++ ++env_vars.Add('MONGO_GIT_HASH', ++ help='Sets the githash to store in the MongoDB version information', ++ default=version_data['githash']) ++ ++env_vars.Add('MSVC_USE_SCRIPT', ++ help='Sets the script used to setup Visual Studio.') ++ ++env_vars.Add('MSVC_VERSION', ++ help='Sets the version of Visual Studio to use (e.g. 12.0, 11.0, 10.0)') ++ ++env_vars.Add('OBJCOPY', ++ help='Sets the path to objcopy', ++ default=WhereIs('objcopy')) ++ ++env_vars.Add('RPATH', ++ help='Set the RPATH for dynamic libraries and executables', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('SHCCFLAGS', ++ help='Sets flags for the C and C++ compiler when building shared libraries', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('SHCFLAGS', ++ help='Sets flags for the C compiler when building shared libraries', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('SHCXXFLAGS', ++ help='Sets flags for the C++ compiler when building shared libraries', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('SHELL', ++ help='Pick the shell to use when spawning commands') ++ ++env_vars.Add('SHLINKFLAGS', ++ help='Sets flags for the linker when building shared libraries', ++ converter=variable_shlex_converter) ++ ++env_vars.Add('TARGET_ARCH', ++ help='Sets the architecture to build for', ++ converter=variable_arch_converter, ++ default=None) ++ ++env_vars.Add('TARGET_OS', ++ help='Sets the target OS to build for', ++ default=mongo_platform.get_running_os_name()) ++ ++env_vars.Add('TOOLS', ++ help='Sets the list of SCons tools to add to the environment', ++ converter=variable_tools_converter, ++ default=decide_platform_tools()) ++ ++env_vars.Add('VARIANT_DIR', ++ help='Sets the name (or generator function) for the variant directory', ++ default=mongo_generators.default_variant_dir_generator, ++) ++ ++env_vars.Add('VERBOSE', ++ help='Control build verbosity (auto, on/off true/false 1/0)', ++ default='auto', ++) ++ ++env_vars.Add('WINDOWS_OPENSSL_BIN', ++ help='Sets the path to the openssl binaries for packaging', ++ default='c:/openssl/bin') ++ ++# -- Validate user provided options -- ++ ++# A dummy environment that should *only* have the variables we have set. In practice it has ++# some other things because SCons isn't quite perfect about keeping variable initialization ++# scoped to Tools, but it should be good enough to do validation on any Variable values that ++# came from the command line or from loaded files. ++variables_only_env = Environment( ++ # Disable platform specific variable injection ++ platform=(lambda x: ()), ++ # But do *not* load any tools, since those might actually set variables. Note that this ++ # causes the value of our TOOLS variable to have no effect. ++ tools=[], ++ # Use the Variables specified above. ++ variables=env_vars, ++) ++ ++# don't run configure if user calls --help ++if GetOption('help'): ++ try: ++ Help('\nThe following variables may also be set like scons VARIABLE=value\n', append=True) ++ Help(env_vars.GenerateHelpText(variables_only_env), append=True) ++ except TypeError: ++ # The append=true kwarg is only supported in scons>=2.4. Without it, calls to Help() clobber ++ # the automatically generated options help, which we don't want. Users on older scons ++ # versions will need to use --variables-help to learn about which variables we support. ++ pass ++ ++ Return() ++ ++if ('CC' in variables_only_env) != ('CXX' in variables_only_env): ++ print('Cannot customize C compiler without customizing C++ compiler, and vice versa') ++ Exit(1) ++ ++# --- environment setup --- ++ ++# If the user isn't using the # to indicate top-of-tree or $ to expand a variable, forbid ++# relative paths. Relative paths don't really work as expected, because they end up relative to ++# the top level SConstruct, not the invokers CWD. We could in theory fix this with ++# GetLaunchDir, but that seems a step too far. ++buildDir = get_option('build-dir').rstrip('/') ++if buildDir[0] not in ['$', '#']: ++ if not os.path.isabs(buildDir): ++ print("Do not use relative paths with --build-dir") ++ Exit(1) ++ ++cacheDir = get_option('cache-dir').rstrip('/') ++if cacheDir[0] not in ['$', '#']: ++ if not os.path.isabs(cacheDir): ++ print("Do not use relative paths with --cache-dir") ++ Exit(1) ++ ++installDir = get_option('prefix').rstrip('/') ++if installDir[0] not in ['$', '#']: ++ if not os.path.isabs(installDir): ++ print("Do not use relative paths with --prefix") ++ Exit(1) ++ ++sconsDataDir = Dir(buildDir).Dir('scons') ++SConsignFile(str(sconsDataDir.File('sconsign'))) ++ ++def printLocalInfo(): ++ import sys, SCons ++ print( "scons version: " + SCons.__version__ ) ++ print( "python version: " + " ".join( [ `i` for i in sys.version_info ] ) ) ++ ++printLocalInfo() ++ ++boostLibs = [ "filesystem", "program_options", "system", "iostreams" ] ++ ++onlyServer = len( COMMAND_LINE_TARGETS ) == 0 or ( len( COMMAND_LINE_TARGETS ) == 1 and str( COMMAND_LINE_TARGETS[0] ) in [ "mongod" , "mongos" , "test" ] ) ++ ++releaseBuild = has_option("release") ++ ++dbg_opt_mapping = { ++ # --dbg, --opt : dbg opt ++ ( "on", None ) : ( True, False ), # special case interaction ++ ( "on", "on" ) : ( True, True ), ++ ( "on", "off" ) : ( True, False ), ++ ( "off", None ) : ( False, True ), ++ ( "off", "on" ) : ( False, True ), ++ ( "off", "off" ) : ( False, False ), ++ ( "on", "size" ) : ( True, True ), ++ ( "off", "size" ) : ( False, True ), ++} ++debugBuild, optBuild = dbg_opt_mapping[(get_option('dbg'), get_option('opt'))] ++optBuildForSize = True if optBuild and get_option('opt') == "size" else False ++ ++if releaseBuild and (debugBuild or not optBuild): ++ print("Error: A --release build may not have debugging, and must have optimization") ++ Exit(1) ++ ++noshell = has_option( "noshell" ) ++ ++jsEngine = get_option( "js-engine") ++ ++serverJs = get_option( "server-js" ) == "on" ++ ++usemozjs = (jsEngine.startswith('mozjs')) ++ ++if not serverJs and not usemozjs: ++ print("Warning: --server-js=off is not needed with --js-engine=none") ++ ++# We defer building the env until we have determined whether we want certain values. Some values ++# in the env actually have semantics for 'None' that differ from being absent, so it is better ++# to build it up via a dict, and then construct the Environment in one shot with kwargs. ++# ++# Yes, BUILD_ROOT vs BUILD_DIR is confusing. Ideally, BUILD_DIR would actually be called ++# VARIANT_DIR, and at some point we should probably do that renaming. Until we do though, we ++# also need an Environment variable for the argument to --build-dir, which is the parent of all ++# variant dirs. For now, we call that BUILD_ROOT. If and when we s/BUILD_DIR/VARIANT_DIR/g, ++# then also s/BUILD_ROOT/BUILD_DIR/g. ++envDict = dict(BUILD_ROOT=buildDir, ++ BUILD_DIR=make_variant_dir_generator(), ++ DIST_ARCHIVE_SUFFIX='.tgz', ++ DIST_BINARIES=[], ++ MODULE_BANNERS=[], ++ MODULE_INJECTORS=dict(), ++ ARCHIVE_ADDITION_DIR_MAP={}, ++ ARCHIVE_ADDITIONS=[], ++ PYTHON=utils.find_python(), ++ SERVER_ARCHIVE='${SERVER_DIST_BASENAME}${DIST_ARCHIVE_SUFFIX}', ++ UNITTEST_ALIAS='unittests', ++ # TODO: Move unittests.txt to $BUILD_DIR, but that requires ++ # changes to MCI. ++ UNITTEST_LIST='$BUILD_ROOT/unittests.txt', ++ INTEGRATION_TEST_ALIAS='integration_tests', ++ INTEGRATION_TEST_LIST='$BUILD_ROOT/integration_tests.txt', ++ BENCHMARK_ALIAS='benchmarks', ++ BENCHMARK_LIST='$BUILD_ROOT/benchmarks.txt', ++ CONFIGUREDIR='$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp', ++ CONFIGURELOG='$BUILD_ROOT/scons/config.log', ++ INSTALL_DIR=installDir, ++ CONFIG_HEADER_DEFINES={}, ++ LIBDEPS_TAG_EXPANSIONS=[], ++ ) ++ ++env = Environment(variables=env_vars, **envDict) ++del envDict ++ ++env.AddMethod(mongo_platform.env_os_is_wrapper, 'TargetOSIs') ++env.AddMethod(mongo_platform.env_get_os_name_wrapper, 'GetTargetOSName') ++ ++def fatal_error(env, msg, *args): ++ print(msg.format(*args)) ++ Exit(1) ++ ++def conf_error(env, msg, *args): ++ print(msg.format(*args)) ++ print("See {0} for details".format(env.File('$CONFIGURELOG').abspath)) ++ Exit(1) ++ ++env.AddMethod(fatal_error, 'FatalError') ++env.AddMethod(conf_error, 'ConfError') ++ ++# Normalize the VERBOSE Option, and make its value available as a ++# function. ++if env['VERBOSE'] == "auto": ++ env['VERBOSE'] = not sys.stdout.isatty() ++elif env['VERBOSE'] in ('1', "ON", "on", "True", "true", True): ++ env['VERBOSE'] = True ++elif env['VERBOSE'] in ('0', "OFF", "off", "False", "false", False): ++ env['VERBOSE'] = False ++else: ++ env.FatalError("Invalid value {0} for VERBOSE Variable", env['VERBOSE']) ++env.AddMethod(lambda env: env['VERBOSE'], 'Verbose') ++ ++if has_option('variables-help'): ++ print(env_vars.GenerateHelpText(env)) ++ Exit(0) ++ ++unknown_vars = env_vars.UnknownVariables() ++if unknown_vars: ++ env.FatalError("Unknown variables specified: {0}", ", ".join(unknown_vars.keys())) ++ ++def set_config_header_define(env, varname, varval = 1): ++ env['CONFIG_HEADER_DEFINES'][varname] = varval ++env.AddMethod(set_config_header_define, 'SetConfigHeaderDefine') ++ ++detectEnv = env.Clone() ++ ++# Identify the toolchain in use. We currently support the following: ++# These macros came from ++# http://nadeausoftware.com/articles/2012/10/c_c_tip_how_detect_compiler_name_and_version_using_compiler_predefined_macros ++toolchain_macros = { ++ 'GCC': 'defined(__GNUC__) && !defined(__clang__)', ++ 'clang': 'defined(__clang__)', ++ 'MSVC': 'defined(_MSC_VER)' ++} ++ ++def CheckForToolchain(context, toolchain, lang_name, compiler_var, source_suffix): ++ test_body = textwrap.dedent(""" ++ #if {0} ++ /* we are using toolchain {0} */ ++ #else ++ #error ++ #endif ++ """.format(toolchain_macros[toolchain])) ++ ++ print_tuple = (lang_name, context.env[compiler_var], toolchain) ++ context.Message('Checking if %s compiler "%s" is %s... ' % print_tuple) ++ ++ # Strip indentation from the test body to ensure that the newline at the end of the ++ # endif is the last character in the file (rather than a line of spaces with no ++ # newline), and that all of the preprocessor directives start at column zero. Both of ++ # these issues can trip up older toolchains. ++ result = context.TryCompile(test_body, source_suffix) ++ context.Result(result) ++ return result ++ ++endian = get_option( "endian" ) ++ ++if endian == "auto": ++ endian = sys.byteorder ++ ++if endian == "little": ++ env.SetConfigHeaderDefine("MONGO_CONFIG_BYTE_ORDER", "1234") ++elif endian == "big": ++ env.SetConfigHeaderDefine("MONGO_CONFIG_BYTE_ORDER", "4321") ++ ++# These preprocessor macros came from ++# http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros ++# ++# NOTE: Remember to add a trailing comma to form any required one ++# element tuples, or your configure checks will fail in strange ways. ++processor_macros = { ++ 'arm' : { 'endian': 'little', 'defines': ('__arm__',) }, ++ 'aarch64' : { 'endian': 'little', 'defines': ('__arm64__', '__aarch64__')}, ++ 'i386' : { 'endian': 'little', 'defines': ('__i386', '_M_IX86')}, ++ 'ppc64le' : { 'endian': 'little', 'defines': ('__powerpc64__',)}, ++ 's390x' : { 'endian': 'big', 'defines': ('__s390x__',)}, ++ 'sparc' : { 'endian': 'big', 'defines': ('__sparc',)}, ++ 'x86_64' : { 'endian': 'little', 'defines': ('__x86_64', '_M_AMD64')}, ++} ++ ++def CheckForProcessor(context, which_arch): ++ def run_compile_check(arch): ++ full_macros = " || ".join([ "defined(%s)" % (v) for v in processor_macros[arch]['defines']]) ++ ++ if not endian == processor_macros[arch]['endian']: ++ return False ++ ++ test_body = """ ++ #if {0} ++ /* Detected {1} */ ++ #else ++ #error not {1} ++ #endif ++ """.format(full_macros, arch) ++ ++ return context.TryCompile(textwrap.dedent(test_body), ".c") ++ ++ if which_arch: ++ ret = run_compile_check(which_arch) ++ context.Message('Checking if target processor is %s ' % which_arch) ++ context.Result(ret) ++ return ret; ++ ++ for k in processor_macros.keys(): ++ ret = run_compile_check(k) ++ if ret: ++ context.Result('Detected a %s processor' % k) ++ return k ++ ++ context.Result('Could not detect processor model/architecture') ++ return False ++ ++# Taken from http://nadeausoftware.com/articles/2012/01/c_c_tip_how_use_compiler_predefined_macros_detect_operating_system ++os_macros = { ++ "windows": "defined(_WIN32)", ++ "solaris": "defined(__sun)", ++ "freebsd": "defined(__FreeBSD__)", ++ "openbsd": "defined(__OpenBSD__)", ++ "iOS": "defined(__APPLE__) && TARGET_OS_IOS && !TARGET_OS_SIMULATOR", ++ "iOS-sim": "defined(__APPLE__) && TARGET_OS_IOS && TARGET_OS_SIMULATOR", ++ "tvOS": "defined(__APPLE__) && TARGET_OS_TV && !TARGET_OS_SIMULATOR", ++ "tvOS-sim": "defined(__APPLE__) && TARGET_OS_TV && TARGET_OS_SIMULATOR", ++ "watchOS": "defined(__APPLE__) && TARGET_OS_WATCH && !TARGET_OS_SIMULATOR", ++ "watchOS-sim": "defined(__APPLE__) && TARGET_OS_WATCH && TARGET_OS_SIMULATOR", ++ ++ # NOTE: Once we have XCode 8 required, we can rely on the value of TARGET_OS_OSX. In case ++ # we are on an older XCode, use TARGET_OS_MAC and TARGET_OS_IPHONE. We don't need to correct ++ # the above declarations since we will never target them with anything other than XCode 8. ++ "macOS": "defined(__APPLE__) && (TARGET_OS_OSX || (TARGET_OS_MAC && !TARGET_OS_IPHONE))", ++ "linux": "defined(__linux__)", ++ "android": "defined(__ANDROID__)", ++} ++ ++def CheckForOS(context, which_os): ++ test_body = """ ++ #if defined(__APPLE__) ++ #include ++ #endif ++ #if {0} ++ /* detected {1} */ ++ #else ++ #error ++ #endif ++ """.format(os_macros[which_os], which_os) ++ context.Message('Checking if target OS {0} is supported by the toolchain... '.format(which_os)) ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++def CheckForCXXLink(context): ++ test_body = """ ++ #include ++ #include ++ ++ int main() { ++ std::cout << "Hello, World" << std::endl; ++ return EXIT_SUCCESS; ++ } ++ """ ++ context.Message('Checking that the C++ compiler can link a C++ program... ') ++ ret = context.TryLink(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++detectConf = Configure(detectEnv, help=False, custom_tests = { ++ 'CheckForToolchain' : CheckForToolchain, ++ 'CheckForProcessor': CheckForProcessor, ++ 'CheckForOS': CheckForOS, ++ 'CheckForCXXLink': CheckForCXXLink, ++}) ++ ++if not detectConf.CheckCC(): ++ env.ConfError( ++ "C compiler {0} doesn't work", ++ detectEnv['CC']) ++ ++if not detectConf.CheckCXX(): ++ env.ConfError( ++ "C++ compiler {0} doesn't work", ++ detectEnv['CXX']) ++ ++if not detectConf.CheckForCXXLink(): ++ env.ConfError( ++ "C++ compiler {0} can't link C++ programs", ++ detectEnv['CXX']) ++ ++toolchain_search_sequence = [ "GCC", "clang" ] ++if mongo_platform.is_running_os('windows'): ++ toolchain_search_sequence = [ 'MSVC', 'clang', 'GCC' ] ++for candidate_toolchain in toolchain_search_sequence: ++ if detectConf.CheckForToolchain(candidate_toolchain, "C++", "CXX", ".cpp"): ++ detected_toolchain = candidate_toolchain ++ break ++ ++if not detected_toolchain: ++ env.ConfError("Couldn't identify the C++ compiler") ++ ++if not detectConf.CheckForToolchain(detected_toolchain, "C", "CC", ".c"): ++ env.ConfError("C compiler does not match identified C++ compiler") ++ ++# Now that we've detected the toolchain, we add methods to the env ++# to get the canonical name of the toolchain and to test whether ++# scons is using a particular toolchain. ++def get_toolchain_name(self): ++ return detected_toolchain.lower() ++def is_toolchain(self, *args): ++ actual_toolchain = self.ToolchainName() ++ for v in args: ++ if v.lower() == actual_toolchain: ++ return True ++ return False ++ ++env.AddMethod(get_toolchain_name, 'ToolchainName') ++env.AddMethod(is_toolchain, 'ToolchainIs') ++ ++if env['TARGET_ARCH']: ++ if not detectConf.CheckForProcessor(env['TARGET_ARCH']): ++ env.ConfError("Could not detect processor specified in TARGET_ARCH variable") ++else: ++ detected_processor = detectConf.CheckForProcessor(None) ++ if not detected_processor: ++ env.ConfError("Failed to detect a supported target architecture") ++ env['TARGET_ARCH'] = detected_processor ++ ++if env['TARGET_OS'] not in os_macros: ++ print("No special config for [{0}] which probably means it won't work".format(env['TARGET_OS'])) ++elif not detectConf.CheckForOS(env['TARGET_OS']): ++ env.ConfError("TARGET_OS ({0}) is not supported by compiler", env['TARGET_OS']) ++ ++detectConf.Finish() ++ ++env['CC_VERSION'] = mongo_toolchain.get_toolchain_ver(env, 'CC') ++env['CXX_VERSION'] = mongo_toolchain.get_toolchain_ver(env, 'CXX') ++ ++if not env['HOST_ARCH']: ++ env['HOST_ARCH'] = env['TARGET_ARCH'] ++ ++# In some places we have POSIX vs Windows cpp files, and so there's an additional ++# env variable to interpolate their names in child sconscripts ++ ++env['TARGET_OS_FAMILY'] = 'posix' if env.TargetOSIs('posix') else env.GetTargetOSName() ++ ++# Currently we only use tcmalloc on windows and linux x86_64. Other ++# linux targets (power, s390x, arm) do not currently support tcmalloc. ++# ++# Normalize the allocator option and store it in the Environment. It ++# would be nicer to use SetOption here, but you can't reset user ++# options for some strange reason in SCons. Instead, we store this ++# option as a new variable in the environment. ++if get_option('allocator') == "auto": ++ # using an allocator besides system on android would require either fixing or disabling ++ # gperftools on android ++ if env.TargetOSIs('windows') or \ ++ env.TargetOSIs('linux') and not env.TargetOSIs('android'): ++ env['MONGO_ALLOCATOR'] = "tcmalloc" ++ else: ++ env['MONGO_ALLOCATOR'] = "system" ++else: ++ env['MONGO_ALLOCATOR'] = get_option('allocator') ++ ++if has_option("cache"): ++ if has_option("gcov"): ++ env.FatalError("Mixing --cache and --gcov doesn't work correctly yet. See SERVER-11084") ++ env.CacheDir(str(env.Dir(cacheDir))) ++ ++# Normalize the link model. If it is auto, then for now both developer and release builds ++# use the "static" mode. Somday later, we probably want to make the developer build default ++# dynamic, but that will require the hygienic builds project. ++link_model = get_option('link-model') ++if link_model == "auto": ++ link_model = "static" ++ ++# Windows can't currently support anything other than 'object' or 'static', until ++# we have both hygienic builds and have annotated functions for export. ++if env.TargetOSIs('windows') and link_model not in ['object', 'static', 'dynamic-sdk']: ++ env.FatalError("Windows builds must use the 'object', 'dynamic-sdk', or 'static' link models") ++ ++ ++# The mongodbtoolchain currently doesn't produce working binaries if ++# you combine a dynamic build with a non-system allocator, but the ++# failure mode is non-obvious. For now, prevent people from wandering ++# inadvertantly into this trap. Remove this constraint when ++# https://jira.mongodb.org/browse/SERVER-27675 is resolved. ++if (link_model == 'dynamic') and ('mongodbtoolchain' in env['CXX']) and (env['MONGO_ALLOCATOR'] != 'system'): ++ env.FatalError('Cannot combine the MongoDB toolchain, a dynamic build, and a non-system allocator. Choose two.') ++ ++# The 'object' mode for libdeps is enabled by setting _LIBDEPS to $_LIBDEPS_OBJS. The other two ++# modes operate in library mode, enabled by setting _LIBDEPS to $_LIBDEPS_LIBS. ++env['_LIBDEPS'] = '$_LIBDEPS_OBJS' if link_model == "object" else '$_LIBDEPS_LIBS' ++ ++env['BUILDERS']['ProgramObject'] = env['BUILDERS']['StaticObject'] ++env['BUILDERS']['LibraryObject'] = env['BUILDERS']['StaticObject'] ++ ++env['SHARPREFIX'] = '$LIBPREFIX' ++env['SHARSUFFIX'] = '${SHLIBSUFFIX}${LIBSUFFIX}' ++env['BUILDERS']['SharedArchive'] = SCons.Builder.Builder( ++ action=env['BUILDERS']['StaticLibrary'].action, ++ emitter='$SHAREMITTER', ++ prefix='$SHARPREFIX', ++ suffix='$SHARSUFFIX', ++ src_suffix=env['BUILDERS']['SharedLibrary'].src_suffix, ++) ++ ++if link_model.startswith("dynamic"): ++ ++ def library(env, target, source, *args, **kwargs): ++ sharedLibrary = env.SharedLibrary(target, source, *args, **kwargs) ++ sharedArchive = env.SharedArchive(target, source=sharedLibrary[0].sources, *args, **kwargs) ++ sharedLibrary.extend(sharedArchive) ++ return sharedLibrary ++ ++ env['BUILDERS']['Library'] = library ++ env['BUILDERS']['LibraryObject'] = env['BUILDERS']['SharedObject'] ++ ++ # TODO: Ideally, the conditions below should be based on a ++ # detection of what linker we are using, not the local OS, but I ++ # doubt very much that we will see the mach-o linker on anything ++ # other than Darwin, or a BFD/sun-esque linker elsewhere. ++ ++ # On Darwin, we need to tell the linker that undefined symbols are ++ # resolved via dynamic lookup; otherwise we get build failures. On ++ # other unixes, we need to suppress as-needed behavior so that ++ # initializers are ensured present, even if there is no visible ++ # edge to the library in the symbol graph. ++ # ++ # NOTE: The darwin linker flag is only needed because the library ++ # graph is not a DAG. Once the graph is a DAG, we will require all ++ # edges to be expressed, and we should drop the flag. When that ++ # happens, we should also add -z,defs flag on ELF platforms to ++ # ensure that missing symbols due to unnamed dependency edges ++ # result in link errors. ++ # ++ # NOTE: The `illegal_cyclic_or_unresolved_dependencies_whitelisted` ++ # tag can be applied to a library to indicate that it does not (or ++ # cannot) completely express all of its required link dependencies. ++ # This can occur for four reasons: ++ # ++ # - No unique provider for the symbol: Some symbols do not have a ++ # unique dependency that provides a definition, in which case it ++ # is impossible for the library to express a dependency edge to ++ # resolve the symbol ++ # ++ # - The library is part of a cycle: If library A depends on B, ++ # which depends on C, which depends on A, then it is impossible ++ # to express all three edges in SCons, since otherwise there is ++ # no way to sequence building the libraries. The cyclic ++ # libraries actually work at runtime, because some parent object ++ # links all of them. ++ # ++ # - The symbol is provided by an executable into which the library ++ # will be linked. The mongo::inShutdown symbol is a good ++ # example. ++ # ++ # - The symbol is provided by a third-party library, outside of our ++ # control. ++ # ++ # All of these are defects in the linking model. In an effort to ++ # eliminate these issues, we have begun tagging those libraries ++ # that are affected, and requiring that all non-tagged libraries ++ # correctly express all dependencies. As we repair each defective ++ # library, we can remove the tag. When all the tags are removed ++ # the graph will be acyclic. Libraries which are incomplete for the ++ # final reason, "libraries outside of our control", may remain for ++ # reasons beyond our control. Such libraries ideally should ++ # have no dependencies (and thus be leaves in our linking DAG). ++ # If that condition is met, then the graph will be acyclic. ++ ++ if env.TargetOSIs('darwin'): ++ if link_model == "dynamic-strict": ++ # Darwin is strict by default ++ pass ++ else: ++ def libdeps_tags_expand_incomplete(source, target, env, for_signature): ++ # On darwin, since it is strict by default, we need to add a flag ++ # when libraries are tagged incomplete. ++ if ('illegal_cyclic_or_unresolved_dependencies_whitelisted' ++ in target[0].get_env().get("LIBDEPS_TAGS", [])): ++ return ["-Wl,-undefined,dynamic_lookup"] ++ return [] ++ env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete) ++ elif env.TargetOSIs('windows'): ++ if link_model == "dynamic-strict": ++ # Windows is strict by default ++ pass ++ else: ++ def libdeps_tags_expand_incomplete(source, target, env, for_signature): ++ # On windows, since it is strict by default, we need to add a flag ++ # when libraries are tagged incomplete. ++ if ('illegal_cyclic_or_unresolved_dependencies_whitelisted' ++ in target[0].get_env().get("LIBDEPS_TAGS", [])): ++ return ["/FORCE:UNRESOLVED"] ++ return [] ++ env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete) ++ else: ++ env.AppendUnique(LINKFLAGS=["-Wl,--no-as-needed"]) ++ ++ # Using zdefs doesn't work at all with the sanitizers ++ if not has_option('sanitize'): ++ ++ if link_model == "dynamic-strict": ++ env.AppendUnique(SHLINKFLAGS=["-Wl,-z,defs"]) ++ else: ++ # On BFD/gold linker environments, which are not strict by ++ # default, we need to add a flag when libraries are not ++ # tagged incomplete. ++ def libdeps_tags_expand_incomplete(source, target, env, for_signature): ++ if ('illegal_cyclic_or_unresolved_dependencies_whitelisted' ++ not in target[0].get_env().get("LIBDEPS_TAGS", [])): ++ return ["-Wl,-z,defs"] ++ return [] ++ env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete) ++ ++if optBuild: ++ env.SetConfigHeaderDefine("MONGO_CONFIG_OPTIMIZED_BUILD") ++ ++# Enable the fast decider if exlicltly requested or if in 'auto' mode and not in conflict with other ++# options. ++if get_option('build-fast-and-loose') == 'on' or \ ++ (get_option('build-fast-and-loose') == 'auto' and \ ++ not has_option('release') and \ ++ not has_option('cache')): ++ # See http://www.scons.org/wiki/GoFastButton for details ++ env.Decider('MD5-timestamp') ++ env.SetOption('max_drift', 1) ++ ++# If the user has requested the git decider, enable it if it is available. We want to do this after ++# we set the basic decider above, so that we 'chain' to that one. ++if get_option('git-decider') == 'on': ++ git_decider = Tool('git_decider') ++ if git_decider.exists(env): ++ git_decider(env) ++ ++# On non-windows platforms, we may need to differentiate between flags being used to target an ++# executable (like -fPIE), vs those being used to target a (shared) library (like -fPIC). To do so, ++# we inject a new family of SCons variables PROG*FLAGS, by reaching into the various COMs. ++if not env.TargetOSIs('windows'): ++ env["CCCOM"] = env["CCCOM"].replace("$CFLAGS", "$CFLAGS $PROGCFLAGS") ++ env["CXXCOM"] = env["CXXCOM"].replace("$CXXFLAGS", "$CXXFLAGS $PROGCXXFLAGS") ++ env["CCCOM"] = env["CCCOM"].replace("$CCFLAGS", "$CCFLAGS $PROGCCFLAGS") ++ env["CXXCOM"] = env["CXXCOM"].replace("$CCFLAGS", "$CCFLAGS $PROGCCFLAGS") ++ env["LINKCOM"] = env["LINKCOM"].replace("$LINKFLAGS", "$LINKFLAGS $PROGLINKFLAGS") ++ ++if not env.Verbose(): ++ env.Append( CCCOMSTR = "Compiling $TARGET" ) ++ env.Append( CXXCOMSTR = env["CCCOMSTR"] ) ++ env.Append( SHCCCOMSTR = "Compiling $TARGET" ) ++ env.Append( SHCXXCOMSTR = env["SHCCCOMSTR"] ) ++ env.Append( LINKCOMSTR = "Linking $TARGET" ) ++ env.Append( SHLINKCOMSTR = env["LINKCOMSTR"] ) ++ env.Append( ARCOMSTR = "Generating library $TARGET" ) ++ ++# Link tools other than mslink don't setup TEMPFILE in LINKCOM, ++# disabling SCons automatically falling back to a temp file when ++# running link commands that are over MAXLINELENGTH. With our object ++# file linking mode, we frequently hit even the large linux command ++# line length, so we want it everywhere. If we aren't using mslink, ++# add TEMPFILE in. For verbose builds when using a tempfile, we need ++# some trickery so that we print the command we are running, and not ++# just the invocation of the compiler being fed the command file. ++if not 'mslink' in env['TOOLS']: ++ if env.Verbose(): ++ env["LINKCOM"] = "${{TEMPFILE('{0}', '')}}".format(env['LINKCOM']) ++ env["SHLINKCOM"] = "${{TEMPFILE('{0}', '')}}".format(env['SHLINKCOM']) ++ if not 'libtool' in env['TOOLS']: ++ env["ARCOM"] = "${{TEMPFILE('{0}', '')}}".format(env['ARCOM']) ++ else: ++ env["LINKCOM"] = "${{TEMPFILE('{0}', 'LINKCOMSTR')}}".format(env['LINKCOM']) ++ env["SHLINKCOM"] = "${{TEMPFILE('{0}', 'SHLINKCOMSTR')}}".format(env['SHLINKCOM']) ++ if not 'libtool' in env['TOOLS']: ++ env["ARCOM"] = "${{TEMPFILE('{0}', 'ARCOMSTR')}}".format(env['ARCOM']) ++ ++if env['_LIBDEPS'] == '$_LIBDEPS_OBJS': ++ # The libraries we build in LIBDEPS_OBJS mode are just placeholders for tracking dependencies. ++ # This avoids wasting time and disk IO on them. ++ def write_uuid_to_file(env, target, source): ++ with open(env.File(target[0]).abspath, 'w') as fake_lib: ++ fake_lib.write(str(uuid.uuid4())) ++ fake_lib.write('\n') ++ ++ def noop_action(env, target, source): ++ pass ++ ++ env['ARCOM'] = write_uuid_to_file ++ env['ARCOMSTR'] = 'Generating placeholder library $TARGET' ++ env['RANLIBCOM'] = noop_action ++ env['RANLIBCOMSTR'] = 'Skipping ranlib for $TARGET' ++ ++libdeps.setup_environment(env, emitting_shared=(link_model.startswith("dynamic"))) ++ ++# Both the abidw tool and the thin archive tool must be loaded after ++# libdeps, so that the scanners they inject can see the library ++# dependencies added by libdeps. ++if link_model.startswith("dynamic"): ++ # Add in the abi linking tool if the user requested and it is ++ # supported on this platform. ++ if env.get('ABIDW'): ++ abilink = Tool('abilink') ++ if abilink.exists(env): ++ abilink(env) ++ ++if env['_LIBDEPS'] == '$_LIBDEPS_LIBS': ++ # The following platforms probably aren't using the binutils ++ # toolchain, or may be using it for the archiver but not the ++ # linker, and binutils currently is the olny thing that supports ++ # thin archives. Don't even try on those platforms. ++ if not env.TargetOSIs('solaris', 'darwin', 'windows', 'openbsd'): ++ env.Tool('thin_archive') ++ ++if env.TargetOSIs('linux', 'freebsd', 'openbsd'): ++ env['LINK_LIBGROUP_START'] = '-Wl,--start-group' ++ env['LINK_LIBGROUP_END'] = '-Wl,--end-group' ++ # NOTE: The leading and trailing spaces here are important. Do not remove them. ++ env['LINK_WHOLE_ARCHIVE_LIB_START'] = '-Wl,--whole-archive ' ++ env['LINK_WHOLE_ARCHIVE_LIB_END'] = ' -Wl,--no-whole-archive' ++elif env.TargetOSIs('darwin'): ++ env['LINK_LIBGROUP_START'] = '' ++ env['LINK_LIBGROUP_END'] = '' ++ # NOTE: The trailing space here is important. Do not remove it. ++ env['LINK_WHOLE_ARCHIVE_LIB_START'] = '-Wl,-force_load ' ++ env['LINK_WHOLE_ARCHIVE_LIB_END'] = '' ++elif env.TargetOSIs('solaris'): ++ env['LINK_LIBGROUP_START'] = '-Wl,-z,rescan-start' ++ env['LINK_LIBGROUP_END'] = '-Wl,-z,rescan-end' ++ # NOTE: The leading and trailing spaces here are important. Do not remove them. ++ env['LINK_WHOLE_ARCHIVE_LIB_START'] = '-Wl,-z,allextract ' ++ env['LINK_WHOLE_ARCHIVE_LIB_END'] = ' -Wl,-z,defaultextract' ++elif env.TargetOSIs('windows'): ++ env['LINK_WHOLE_ARCHIVE_LIB_START'] = '/WHOLEARCHIVE:' ++ env['LINK_WHOLE_ARCHIVE_LIB_END'] = '' ++ ++# ---- other build setup ----- ++if debugBuild: ++ env.SetConfigHeaderDefine("MONGO_CONFIG_DEBUG_BUILD") ++else: ++ env.AppendUnique( CPPDEFINES=[ 'NDEBUG' ] ) ++ ++if env.TargetOSIs('linux'): ++ env.Append( LIBS=["m"] ) ++ if not env.TargetOSIs('android'): ++ env.Append( LIBS=["resolv"] ) ++ ++elif env.TargetOSIs('solaris'): ++ env.Append( LIBS=["socket","resolv","lgrp"] ) ++ ++elif env.TargetOSIs('freebsd'): ++ env.Append( LIBS=[ "kvm" ] ) ++ env.Append( CCFLAGS=[ "-fno-omit-frame-pointer" ] ) ++ ++elif env.TargetOSIs('darwin'): ++ env.Append( LIBS=["resolv"] ) ++ ++elif env.TargetOSIs('openbsd'): ++ env.Append( LIBS=[ "kvm" ] ) ++ ++elif env.TargetOSIs('windows'): ++ dynamicCRT = has_option("dynamic-windows") ++ ++ env['DIST_ARCHIVE_SUFFIX'] = '.zip' ++ ++ # If tools configuration fails to set up 'cl' in the path, fall back to importing the whole ++ # shell environment and hope for the best. This will work, for instance, if you have loaded ++ # an SDK shell. ++ for pathdir in env['ENV']['PATH'].split(os.pathsep): ++ if os.path.exists(os.path.join(pathdir, 'cl.exe')): ++ break ++ else: ++ print("NOTE: Tool configuration did not find 'cl' compiler, falling back to os environment") ++ env['ENV'] = dict(os.environ) ++ ++ env.Append(CPPDEFINES=[ ++ # This tells the Windows compiler not to link against the .lib files ++ # and to use boost as a bunch of header-only libraries ++ "BOOST_ALL_NO_LIB", ++ ]) ++ ++ env.Append( CPPDEFINES=[ "_UNICODE" ] ) ++ env.Append( CPPDEFINES=[ "UNICODE" ] ) ++ ++ # /EHsc exception handling style for visual studio ++ # /W3 warning level ++ env.Append(CCFLAGS=["/EHsc","/W3"]) ++ ++ # some warnings we don't like: ++ # c4355 ++ # 'this' : used in base member initializer list ++ # The this pointer is valid only within nonstatic member functions. It cannot be used in the initializer list for a base class. ++ # c4800 ++ # 'type' : forcing value to bool 'true' or 'false' (performance warning) ++ # This warning is generated when a value that is not bool is assigned or coerced into type bool. ++ # c4267 ++ # 'var' : conversion from 'size_t' to 'type', possible loss of data ++ # When compiling with /Wp64, or when compiling on a 64-bit operating system, type is 32 bits but size_t is 64 bits when compiling for 64-bit targets. To fix this warning, use size_t instead of a type. ++ # c4244 ++ # 'conversion' conversion from 'type1' to 'type2', possible loss of data ++ # An integer type is converted to a smaller integer type. ++ # c4290 ++ # C++ exception specification ignored except to indicate a function is not __declspec(nothrow ++ # A function is declared using exception specification, which Visual C++ accepts but does not ++ # implement ++ # c4068 ++ # unknown pragma -- added so that we can specify unknown pragmas for other compilers ++ # c4351 ++ # on extremely old versions of MSVC (pre 2k5), default constructing an array member in a ++ # constructor's initialization list would not zero the array members "in some cases". ++ # since we don't target MSVC versions that old, this warning is safe to ignore. ++ # c4373 ++ # Older versions of MSVC would fail to make a function in a derived class override a virtual ++ # function in the parent, when defined inline and at least one of the parameters is made const. ++ # The behavior is incorrect under the standard. MSVC is fixed now, and the warning exists ++ # merely to alert users who may have relied upon the older, non-compliant behavior. Our code ++ # should not have any problems with the older behavior, so we can just disable this warning. ++ env.Append( CCFLAGS=["/wd4355", "/wd4800", "/wd4267", "/wd4244", ++ "/wd4290", "/wd4068", "/wd4351", "/wd4373"] ) ++ ++ # some warnings we should treat as errors: ++ # c4013 ++ # 'function' undefined; assuming extern returning int ++ # This warning occurs when files compiled for the C language use functions not defined ++ # in a header file. ++ # c4099 ++ # identifier' : type name first seen using 'objecttype1' now seen using 'objecttype2' ++ # This warning occurs when classes and structs are declared with a mix of struct and class ++ # which can cause linker failures ++ # c4930 ++ # 'identifier': prototyped function not called (was a variable definition intended?) ++ # This warning indicates a most-vexing parse error, where a user declared a function that ++ # was probably intended as a variable definition. A common example is accidentally ++ # declaring a function called lock that takes a mutex when one meant to create a guard ++ # object called lock on the stack. ++ env.Append( CCFLAGS=["/we4013", "/we4099", "/we4930"] ) ++ ++ # Warnings as errors ++ if not has_option("disable-warnings-as-errors"): ++ env.Append( CCFLAGS=["/WX"] ) ++ ++ env.Append( CPPDEFINES=["_CONSOLE","_CRT_SECURE_NO_WARNINGS", "_SCL_SECURE_NO_WARNINGS"] ) ++ ++ # this would be for pre-compiled headers, could play with it later ++ #env.Append( CCFLAGS=['/Yu"pch.h"'] ) ++ ++ # Don't send error reports in case of internal compiler error ++ env.Append( CCFLAGS= ["/errorReport:none"] ) ++ ++ # Select debugging format. /Zi gives faster links but seem to use more memory ++ if get_option('msvc-debugging-format') == "codeview": ++ env['CCPDBFLAGS'] = "/Z7" ++ elif get_option('msvc-debugging-format') == "pdb": ++ env['CCPDBFLAGS'] = '/Zi /Fd${TARGET}.pdb' ++ ++ # /DEBUG will tell the linker to create a .pdb file ++ # which WinDbg and Visual Studio will use to resolve ++ # symbols if you want to debug a release-mode image. ++ # Note that this means we can't do parallel links in the build. ++ # ++ # Please also note that this has nothing to do with _DEBUG or optimization. ++ env.Append( LINKFLAGS=["/DEBUG"] ) ++ ++ # /MD: use the multithreaded, DLL version of the run-time library (MSVCRT.lib/MSVCR###.DLL) ++ # /MT: use the multithreaded, static version of the run-time library (LIBCMT.lib) ++ # /MDd: Defines _DEBUG, _MT, _DLL, and uses MSVCRTD.lib/MSVCRD###.DLL ++ # /MTd: Defines _DEBUG, _MT, and causes your application to use the ++ # debug multithread version of the run-time library (LIBCMTD.lib) ++ ++ winRuntimeLibMap = { ++ #dyn #dbg ++ ( False, False ) : "/MT", ++ ( False, True ) : "/MTd", ++ ( True, False ) : "/MD", ++ ( True, True ) : "/MDd", ++ } ++ ++ env.Append(CCFLAGS=[winRuntimeLibMap[(dynamicCRT, debugBuild)]]) ++ ++ if optBuild: ++ # /O1: optimize for size ++ # /O2: optimize for speed (as opposed to size) ++ # /Oy-: disable frame pointer optimization (overrides /O2, only affects 32-bit) ++ # /INCREMENTAL: NO - disable incremental link - avoid the level of indirection for function ++ # calls ++ ++ optStr = "/O2" if not optBuildForSize else "/O1" ++ env.Append( CCFLAGS=[optStr, "/Oy-"] ) ++ env.Append( LINKFLAGS=["/INCREMENTAL:NO"]) ++ else: ++ env.Append( CCFLAGS=["/Od"] ) ++ ++ if debugBuild and not optBuild: ++ # /RTC1: - Enable Stack Frame Run-Time Error Checking; Reports when a variable is used ++ # without having been initialized (implies /Od: no optimizations) ++ env.Append( CCFLAGS=["/RTC1"] ) ++ ++ # Support large object files since some unit-test sources contain a lot of code ++ env.Append( CCFLAGS=["/bigobj"] ) ++ ++ # Set Source and Executable character sets to UTF-8, this will produce a warning C4828 if the ++ # file contains invalid UTF-8. ++ env.Append( CCFLAGS=["/utf-8" ]) ++ ++ # Enforce type conversion rules for rvalue reference types as a result of a cast operation. ++ env.Append( CCFLAGS=["/Zc:rvalueCast"] ) ++ ++ # Disable string literal type conversion, instead const_cast must be explicitly specified. ++ env.Append( CCFLAGS=["/Zc:strictStrings"] ) ++ ++ # Treat volatile according to the ISO standard and do not guarantee acquire/release semantics. ++ env.Append( CCFLAGS=["/volatile:iso"] ) ++ ++ # This gives 32-bit programs 4 GB of user address space in WOW64, ignored in 64-bit builds. ++ env.Append( LINKFLAGS=["/LARGEADDRESSAWARE"] ) ++ ++ env.Append( ++ LIBS=[ ++ 'DbgHelp.lib', ++ 'Iphlpapi.lib', ++ 'Psapi.lib', ++ 'advapi32.lib', ++ 'bcrypt.lib', ++ 'crypt32.lib', ++ 'dnsapi.lib', ++ 'kernel32.lib', ++ 'shell32.lib', ++ 'pdh.lib', ++ 'version.lib', ++ 'winmm.lib', ++ 'ws2_32.lib', ++ 'secur32.lib', ++ ], ++ ) ++ ++# When building on visual studio, this sets the name of the debug symbols file ++if env.ToolchainIs('msvc'): ++ env['PDB'] = '${TARGET.base}.pdb' ++ ++if env.TargetOSIs('posix'): ++ ++ # On linux, C code compiled with gcc/clang -std=c11 causes ++ # __STRICT_ANSI__ to be set, and that drops out all of the feature ++ # test definitions, resulting in confusing errors when we run C ++ # language configure checks and expect to be able to find newer ++ # POSIX things. Explicitly enabling _XOPEN_SOURCE fixes that, and ++ # should be mostly harmless as on Linux, these macros are ++ # cumulative. The C++ compiler already sets _XOPEN_SOURCE, and, ++ # notably, setting it again does not disable any other feature ++ # test macros, so this is safe to do. Other platforms like macOS ++ # and BSD have crazy rules, so don't try this there. ++ # ++ # Furthermore, as both C++ compilers appears to unconditioanlly ++ # define _GNU_SOURCE (because libstdc++ requires it), it seems ++ # prudent to explicitly add that too, so that C language checks ++ # see a consistent set of definitions. ++ if env.TargetOSIs('linux'): ++ env.AppendUnique( ++ CPPDEFINES=[ ++ ('_XOPEN_SOURCE', 700), ++ '_GNU_SOURCE', ++ ], ++ ) ++ ++ # Everything on OS X is position independent by default. Solaris doesn't support PIE. ++ if not env.TargetOSIs('darwin', 'solaris'): ++ if get_option('runtime-hardening') == "on": ++ # If runtime hardening is requested, then build anything ++ # destined for an executable with the necessary flags for PIE. ++ env.AppendUnique( ++ PROGCCFLAGS=['-fPIE'], ++ PROGLINKFLAGS=['-pie'], ++ ) ++ ++ # -Winvalid-pch Warn if a precompiled header (see Precompiled Headers) is found in the search path but can't be used. ++ env.Append( CCFLAGS=["-fno-omit-frame-pointer", ++ "-fno-strict-aliasing", ++ "-ggdb", ++ "-pthread", ++ "-Wall", ++ "-Wsign-compare", ++ "-Wno-unknown-pragmas", ++ "-Winvalid-pch"] ) ++ # env.Append( " -Wconversion" ) TODO: this doesn't really work yet ++ if env.TargetOSIs('linux', 'darwin', 'solaris'): ++ if not has_option("disable-warnings-as-errors"): ++ env.Append( CCFLAGS=["-Werror"] ) ++ ++ env.Append( CXXFLAGS=["-Woverloaded-virtual"] ) ++ if env.ToolchainIs('clang'): ++ env.Append( CXXFLAGS=['-Werror=unused-result'] ) ++ ++ # On OS X, clang doesn't want the pthread flag at link time, or it ++ # issues warnings which make it impossible for us to declare link ++ # warnings as errors. See http://stackoverflow.com/a/19382663. ++ if not (env.TargetOSIs('darwin') and env.ToolchainIs('clang')): ++ env.Append( LINKFLAGS=["-pthread"] ) ++ ++ # SERVER-9761: Ensure early detection of missing symbols in dependent libraries at program ++ # startup. ++ if env.TargetOSIs('darwin'): ++ if env.TargetOSIs('macOS'): ++ env.Append( LINKFLAGS=["-Wl,-bind_at_load"] ) ++ else: ++ env.Append( LINKFLAGS=["-Wl,-z,now"] ) ++ env.Append( LINKFLAGS=["-rdynamic"] ) ++ ++ env.Append( LIBS=[] ) ++ ++ #make scons colorgcc friendly ++ for key in ('HOME', 'TERM'): ++ try: ++ env['ENV'][key] = os.environ[key] ++ except KeyError: ++ pass ++ ++ # Python uses APPDATA to determine the location of user installed ++ # site-packages. If we do not pass this variable down to Python ++ # subprocesses then anything installed with `pip install --user` ++ # will be inaccessible leading to import errors. ++ if env.TargetOSIs('windows'): ++ appdata = os.getenv('APPDATA', None) ++ if appdata is not None: ++ env['ENV']['APPDATA'] = appdata ++ ++ if env.TargetOSIs('linux') and has_option( "gcov" ): ++ env.Append( CCFLAGS=["-fprofile-arcs", "-ftest-coverage"] ) ++ env.Append( LINKFLAGS=["-fprofile-arcs", "-ftest-coverage"] ) ++ ++ if optBuild and not optBuildForSize: ++ env.Append( CCFLAGS=["-O2"] ) ++ elif optBuild and optBuildForSize: ++ env.Append( CCFLAGS=["-Os"] ) ++ else: ++ env.Append( CCFLAGS=["-O0"] ) ++ ++ # Promote linker warnings into errors. We can't yet do this on OS X because its linker considers ++ # noall_load obsolete and warns about it. ++ if not has_option("disable-warnings-as-errors"): ++ env.Append( ++ LINKFLAGS=[ ++ '-Wl,-fatal_warnings' if env.TargetOSIs('darwin') else "-Wl,--fatal-warnings", ++ ] ++ ) ++ ++mmapv1 = False ++if get_option('mmapv1') == 'auto': ++ # The mmapv1 storage engine is only supported on x86 ++ # targets. Unless explicitly requested, disable it on all other ++ # platforms. ++ mmapv1 = (env['TARGET_ARCH'] in ['i386', 'x86_64']) ++elif get_option('mmapv1') == 'on': ++ mmapv1 = True ++ ++wiredtiger = False ++if get_option('wiredtiger') == 'on': ++ # Wiredtiger only supports 64-bit architecture, and will fail to compile on 32-bit ++ # so disable WiredTiger automatically on 32-bit since wiredtiger is on by default ++ if env['TARGET_ARCH'] == 'i386': ++ env.FatalError("WiredTiger is not supported on 32-bit platforms\n" ++ "Re-run scons with --wiredtiger=off to build on 32-bit platforms") ++ else: ++ wiredtiger = True ++ env.SetConfigHeaderDefine("MONGO_CONFIG_WIREDTIGER_ENABLED") ++ ++mobile_se = False ++if get_option('mobile-se') == 'on': ++ mobile_se = True ++ ++if env['TARGET_ARCH'] == 'i386': ++ # If we are using GCC or clang to target 32 bit, set the ISA minimum to 'nocona', ++ # and the tuning to 'generic'. The choice of 'nocona' is selected because it ++ # -- includes MMX extenions which we need for tcmalloc on 32-bit ++ # -- can target 32 bit ++ # -- is at the time of this writing a widely-deployed 10 year old microarchitecture ++ # -- is available as a target architecture from GCC 4.0+ ++ # However, we only want to select an ISA, not the nocona specific scheduling, so we ++ # select the generic tuning. For installations where hardware and system compiler rev are ++ # contemporaries, the generic scheduling should be appropriate for a wide range of ++ # deployed hardware. ++ ++ if env.ToolchainIs('GCC', 'clang'): ++ env.Append( CCFLAGS=['-march=nocona', '-mtune=generic'] ) ++ ++# Needed for auth tests since key files are stored in git with mode 644. ++if not env.TargetOSIs('windows'): ++ for keysuffix in [ "1" , "2" ]: ++ keyfile = "jstests/libs/key%s" % keysuffix ++ os.chmod( keyfile , stat.S_IWUSR|stat.S_IRUSR ) ++ ++# boostSuffixList is used when using system boost to select a search sequence ++# for boost libraries. ++boostSuffixList = ["-mt", ""] ++if get_option("system-boost-lib-search-suffixes") is not None: ++ if not use_system_version_of_library("boost"): ++ env.FatalError("The --system-boost-lib-search-suffixes option is only valid " ++ "with --use-system-boost") ++ boostSuffixList = get_option("system-boost-lib-search-suffixes") ++ if boostSuffixList == "": ++ boostSuffixList = [] ++ else: ++ boostSuffixList = boostSuffixList.split(',') ++ ++# discover modules, and load the (python) module for each module's build.py ++mongo_modules = moduleconfig.discover_modules('src/mongo/db/modules', get_option('modules')) ++env['MONGO_MODULES'] = [m.name for m in mongo_modules] ++ ++# --- check system --- ++ssl_provider = None ++free_monitoring = get_option("enable-free-mon") ++ ++def doConfigure(myenv): ++ global wiredtiger ++ global ssl_provider ++ global free_monitoring ++ ++ # Check that the compilers work. ++ # ++ # TODO: Currently, we have some flags already injected. Eventually, this should test the ++ # bare compilers, and we should re-check at the very end that TryCompile and TryLink still ++ # work with the flags we have selected. ++ if myenv.ToolchainIs('msvc'): ++ compiler_minimum_string = "Microsoft Visual Studio 2015 Update 3" ++ compiler_test_body = textwrap.dedent( ++ """ ++ #if !defined(_MSC_VER) ++ #error ++ #endif ++ ++ #if _MSC_VER < 1900 || (_MSC_VER == 1900 && _MSC_FULL_VER < 190024218) ++ #error %s or newer is required to build MongoDB ++ #endif ++ ++ int main(int argc, char* argv[]) { ++ return 0; ++ } ++ """ % compiler_minimum_string) ++ elif myenv.ToolchainIs('gcc'): ++ compiler_minimum_string = "GCC 5.3.0" ++ compiler_test_body = textwrap.dedent( ++ """ ++ #if !defined(__GNUC__) || defined(__clang__) ++ #error ++ #endif ++ ++ #if (__GNUC__ < 5) || (__GNUC__ == 5 && __GNUC_MINOR__ < 3) || (__GNUC__ == 5 && __GNUC_MINOR__ == 3 && __GNUC_PATCHLEVEL__ < 0) ++ #error %s or newer is required to build MongoDB ++ #endif ++ ++ int main(int argc, char* argv[]) { ++ return 0; ++ } ++ """ % compiler_minimum_string) ++ elif myenv.ToolchainIs('clang'): ++ compiler_minimum_string = "clang 3.8 (or Apple XCode 8.3.2)" ++ compiler_test_body = textwrap.dedent( ++ """ ++ #if !defined(__clang__) ++ #error ++ #endif ++ ++ #if defined(__apple_build_version__) ++ #if __apple_build_version__ < 8020042 ++ #error %s or newer is required to build MongoDB ++ #endif ++ #elif (__clang_major__ < 3) || (__clang_major__ == 3 && __clang_minor__ < 8) ++ #error %s or newer is required to build MongoDB ++ #endif ++ ++ int main(int argc, char* argv[]) { ++ return 0; ++ } ++ """ % (compiler_minimum_string, compiler_minimum_string)) ++ else: ++ myenv.ConfError("Error: can't check compiler minimum; don't know this compiler...") ++ ++ def CheckForMinimumCompiler(context, language): ++ extension_for = { ++ "C" : ".c", ++ "C++" : ".cpp", ++ } ++ context.Message("Checking if %s compiler is %s or newer..." % ++ (language, compiler_minimum_string)) ++ result = context.TryCompile(compiler_test_body, extension_for[language]) ++ context.Result(result) ++ return result; ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckForMinimumCompiler' : CheckForMinimumCompiler, ++ }) ++ ++ c_compiler_validated = conf.CheckForMinimumCompiler('C') ++ cxx_compiler_validated = conf.CheckForMinimumCompiler('C++') ++ ++ suppress_invalid = has_option("disable-minimum-compiler-version-enforcement") ++ if releaseBuild and suppress_invalid: ++ env.FatalError("--disable-minimum-compiler-version-enforcement is forbidden with --release") ++ ++ if not (c_compiler_validated and cxx_compiler_validated): ++ if not suppress_invalid: ++ env.ConfError("ERROR: Refusing to build with compiler that does not meet requirements") ++ print("WARNING: Ignoring failed compiler version check per explicit user request.") ++ print("WARNING: The build may fail, binaries may crash, or may run but corrupt data...") ++ ++ # Figure out what our minimum windows version is. If the user has specified, then use ++ # that. ++ if env.TargetOSIs('windows'): ++ if has_option('win-version-min'): ++ win_version_min = get_option('win-version-min') ++ else: ++ # If no minimum version has beeen specified, use our default ++ win_version_min = 'ws08r2' ++ ++ env['WIN_VERSION_MIN'] = win_version_min ++ win_version_min = win_version_min_choices[win_version_min] ++ env.Append( CPPDEFINES=[("_WIN32_WINNT", "0x" + win_version_min[0])] ) ++ env.Append( CPPDEFINES=[("NTDDI_VERSION", "0x" + win_version_min[0] + win_version_min[1])] ) ++ ++ conf.Finish() ++ ++ # We require macOS 10.11 or newer ++ if env.TargetOSIs('darwin'): ++ ++ # TODO: Better error messages, mention the various -mX-version-min-flags in the error, and ++ # single source of truth for versions, plumbed through #ifdef ladder and error messages. ++ def CheckDarwinMinima(context): ++ test_body = """ ++ #include ++ #include ++ #include ++ ++ #if TARGET_OS_OSX && (__MAC_OS_X_VERSION_MIN_REQUIRED < __MAC_10_11) ++ #error 1 ++ #endif ++ """ ++ ++ context.Message("Checking for sufficient {0} target version minimum... ".format(context.env['TARGET_OS'])) ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests={ ++ "CheckDarwinMinima" : CheckDarwinMinima, ++ }) ++ ++ if not conf.CheckDarwinMinima(): ++ conf.env.ConfError("Required target minimum of macOS 10.11 not found") ++ ++ conf.Finish() ++ ++ def AddFlagIfSupported(env, tool, extension, flag, link, **mutation): ++ def CheckFlagTest(context, tool, extension, flag): ++ if link: ++ if tool == 'C': ++ test_body = """ ++ #include ++ #include ++ int main() { ++ printf("Hello, World!"); ++ return EXIT_SUCCESS; ++ }""" ++ elif tool == 'C++': ++ test_body = """ ++ #include ++ #include ++ int main() { ++ std::cout << "Hello, World!" << std::endl; ++ return EXIT_SUCCESS; ++ }""" ++ context.Message('Checking if linker supports %s... ' % (flag)) ++ ret = context.TryLink(textwrap.dedent(test_body), extension) ++ else: ++ test_body = "" ++ context.Message('Checking if %s compiler supports %s... ' % (tool, flag)) ++ ret = context.TryCompile(textwrap.dedent(test_body), extension) ++ context.Result(ret) ++ return ret ++ ++ if env.ToolchainIs('msvc'): ++ env.FatalError("AddFlagIfSupported is not currently supported with MSVC") ++ ++ test_mutation = mutation ++ if env.ToolchainIs('gcc'): ++ test_mutation = copy.deepcopy(mutation) ++ # GCC helpfully doesn't issue a diagnostic on unknown flags of the form -Wno-xxx ++ # unless other diagnostics are triggered. That makes it tough to check for support ++ # for -Wno-xxx. To work around, if we see that we are testing for a flag of the ++ # form -Wno-xxx (but not -Wno-error=xxx), we also add -Wxxx to the flags. GCC does ++ # warn on unknown -Wxxx style flags, so this lets us probe for availablity of ++ # -Wno-xxx. ++ for kw in test_mutation.keys(): ++ test_flags = test_mutation[kw] ++ for test_flag in test_flags: ++ if test_flag.startswith("-Wno-") and not test_flag.startswith("-Wno-error="): ++ test_flags.append(re.sub("^-Wno-", "-W", test_flag)) ++ ++ cloned = env.Clone() ++ cloned.Append(**test_mutation) ++ ++ # For GCC, we don't need anything since bad flags are already errors, but ++ # adding -Werror won't hurt. For clang, bad flags are only warnings, so we need -Werror ++ # to make them real errors. ++ cloned.Append(CCFLAGS=['-Werror']) ++ conf = Configure(cloned, help=False, custom_tests = { ++ 'CheckFlag' : lambda(ctx) : CheckFlagTest(ctx, tool, extension, flag) ++ }) ++ available = conf.CheckFlag() ++ conf.Finish() ++ if available: ++ env.Append(**mutation) ++ return available ++ ++ def AddToCFLAGSIfSupported(env, flag): ++ return AddFlagIfSupported(env, 'C', '.c', flag, False, CFLAGS=[flag]) ++ ++ def AddToCCFLAGSIfSupported(env, flag): ++ return AddFlagIfSupported(env, 'C', '.c', flag, False, CCFLAGS=[flag]) ++ ++ def AddToCXXFLAGSIfSupported(env, flag): ++ return AddFlagIfSupported(env, 'C++', '.cpp', flag, False, CXXFLAGS=[flag]) ++ ++ def AddToLINKFLAGSIfSupported(env, flag): ++ return AddFlagIfSupported(env, 'C', '.c', flag, True, LINKFLAGS=[flag]) ++ ++ def AddToSHLINKFLAGSIfSupported(env, flag): ++ return AddFlagIfSupported(env, 'C', '.c', flag, True, SHLINKFLAGS=[flag]) ++ ++ ++ if myenv.ToolchainIs('clang', 'gcc'): ++ # This warning was added in g++-4.8. ++ AddToCCFLAGSIfSupported(myenv, '-Wno-unused-local-typedefs') ++ ++ # Clang likes to warn about unused functions, which seems a tad aggressive and breaks ++ # -Werror, which we want to be able to use. ++ AddToCCFLAGSIfSupported(myenv, '-Wno-unused-function') ++ ++ # TODO: Note that the following two flags are added to CCFLAGS even though they are ++ # really C++ specific. We need to do this because SCons passes CXXFLAGS *before* ++ # CCFLAGS, but CCFLAGS contains -Wall, which re-enables the warnings we are trying to ++ # suppress. In the future, we should move all warning flags to CCWARNFLAGS and ++ # CXXWARNFLAGS and add these to CCOM and CXXCOM as appropriate. ++ # ++ # Clang likes to warn about unused private fields, but some of our third_party ++ # libraries have such things. ++ AddToCCFLAGSIfSupported(myenv, '-Wno-unused-private-field') ++ ++ # Prevents warning about using deprecated features (such as auto_ptr in c++11) ++ # Using -Wno-error=deprecated-declarations does not seem to work on some compilers, ++ # including at least g++-4.6. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-deprecated-declarations") ++ ++ # As of clang-3.4, this warning appears in v8, and gets escalated to an error. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-out-of-range-compare") ++ ++ # As of clang in Android NDK 17, these warnings appears in boost and/or ICU, and get escalated to errors ++ AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-constant-compare") ++ AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-zero-compare") ++ AddToCCFLAGSIfSupported(myenv, "-Wno-tautological-unsigned-enum-zero-compare") ++ ++ # New in clang-3.4, trips up things mostly in third_party, but in a few places in the ++ # primary mongo sources as well. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-unused-const-variable") ++ ++ # Prevents warning about unused but set variables found in boost version 1.49 ++ # in boost/date_time/format_date_parser.hpp which does not work for compilers ++ # GCC >= 4.6. Error explained in https://svn.boost.org/trac/boost/ticket/6136 . ++ AddToCCFLAGSIfSupported(myenv, "-Wno-unused-but-set-variable") ++ ++ # This has been suppressed in gcc 4.8, due to false positives, but not in clang. So ++ # we explicitly disable it here. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-missing-braces") ++ ++ # Suppress warnings about not consistently using override everywhere in a class. It seems ++ # very pedantic, and we have a fair number of instances. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-inconsistent-missing-override") ++ ++ # Don't issue warnings about potentially evaluated expressions ++ AddToCCFLAGSIfSupported(myenv, "-Wno-potentially-evaluated-expression") ++ ++ # Warn about moves of prvalues, which can inhibit copy elision. ++ AddToCXXFLAGSIfSupported(myenv, "-Wpessimizing-move") ++ ++ # Warn about redundant moves, such as moving a local variable in a return that is different ++ # than the return type. ++ AddToCXXFLAGSIfSupported(myenv, "-Wredundant-move") ++ ++ # Disable warning about variables that may not be initialized ++ # Failures are triggered in the case of boost::optional in GCC 4.8.x ++ # TODO: re-evaluate when we move to GCC 5.3 ++ # see: http://stackoverflow.com/questions/21755206/how-to-get-around-gcc-void-b-4-may-be-used-uninitialized-in-this-funct ++ AddToCXXFLAGSIfSupported(myenv, "-Wno-maybe-uninitialized") ++ ++ # Disable warning about templates that can't be implicitly instantiated. It is an attempt to ++ # make a link error into an easier-to-debug compiler failure, but it triggers false ++ # positives if explicit instantiation is used in a TU that can see the full definition. This ++ # is a problem at least for the S2 headers. ++ AddToCXXFLAGSIfSupported(myenv, "-Wno-undefined-var-template") ++ ++ # This warning was added in clang-4.0, but it warns about code that is required on some ++ # platforms. Since the warning just states that 'explicit instantiation of [a template] that ++ # occurs after an explicit specialization has no effect', it is harmless on platforms where ++ # it isn't required ++ AddToCXXFLAGSIfSupported(myenv, "-Wno-instantiation-after-specialization") ++ ++ # This warning was added in clang-5 and flags many of our lambdas. Since it isn't actively ++ # harmful to capture unused variables we are suppressing for now with a plan to fix later. ++ AddToCCFLAGSIfSupported(myenv, "-Wno-unused-lambda-capture") ++ ++ # This warning was added in clang-5 and incorrectly flags our implementation of ++ # exceptionToStatus(). See https://bugs.llvm.org/show_bug.cgi?id=34804 ++ AddToCCFLAGSIfSupported(myenv, "-Wno-exceptions") ++ ++ ++ # Check if we can set "-Wnon-virtual-dtor" when "-Werror" is set. The only time we can't set it is on ++ # clang 3.4, where a class with virtual function(s) and a non-virtual destructor throws a warning when ++ # it shouldn't. ++ def CheckNonVirtualDtor(context): ++ ++ test_body = """ ++ class Base { ++ public: ++ virtual void foo() const = 0; ++ protected: ++ ~Base() {}; ++ }; ++ ++ class Derived : public Base { ++ public: ++ virtual void foo() const {} ++ }; ++ """ ++ ++ context.Message('Checking -Wnon-virtual-dtor for false positives... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ myenvClone = myenv.Clone() ++ myenvClone.Append( CCFLAGS=['-Werror'] ) ++ myenvClone.Append( CXXFLAGS=["-Wnon-virtual-dtor"] ) ++ conf = Configure(myenvClone, help=False, custom_tests = { ++ 'CheckNonVirtualDtor' : CheckNonVirtualDtor, ++ }) ++ if conf.CheckNonVirtualDtor(): ++ myenv.Append( CXXFLAGS=["-Wnon-virtual-dtor"] ) ++ conf.Finish() ++ ++ # As of XCode 9, this flag must be present (it is not enabled ++ # by -Wall), in order to enforce that -mXXX-version-min=YYY ++ # will enforce that you don't use APIs from ZZZ. ++ if env.TargetOSIs('darwin'): ++ AddToCCFLAGSIfSupported(env, '-Wunguarded-availability') ++ ++ if get_option('runtime-hardening') == "on": ++ # Enable 'strong' stack protection preferentially, but fall back to 'all' if it is not ++ # available. Note that we need to add these to the LINKFLAGS as well, since otherwise we ++ # might not link libssp when we need to (see SERVER-12456). ++ if myenv.ToolchainIs('gcc', 'clang'): ++ if AddToCCFLAGSIfSupported(myenv, '-fstack-protector-strong'): ++ myenv.Append( ++ LINKFLAGS=[ ++ '-fstack-protector-strong', ++ ] ++ ) ++ elif AddToCCFLAGSIfSupported(myenv, '-fstack-protector-all'): ++ myenv.Append( ++ LINKFLAGS=[ ++ '-fstack-protector-all', ++ ] ++ ) ++ ++ if myenv.ToolchainIs('clang'): ++ # TODO: There are several interesting things to try here, but they each have ++ # consequences we need to investigate. ++ # ++ # - fsanitize=bounds: This does static bounds checking. We can ++ # probably turn this on along with fsanitize-trap so that we ++ # don't depend on the ASAN runtime. ++ # ++ # - fsanitize=safestack: This looks very interesting, and is ++ # probably what we want. However there are a few problems: ++ # ++ # - It relies on having the RT library available, and it is ++ # unclear whether we can ship binaries that depend on ++ # that. ++ # ++ # - It is incompatible with a shared object build. ++ # ++ # - It may not work with SpiderMonkey due to needing to ++ # inform the GC about the stacks so that mark-sweep ++ # ++ # - fsanitize=cfi: Again, very interesting, however it ++ # requires LTO builds. ++ pass ++ ++ if has_option('osx-version-min'): ++ message=""" ++ The --osx-version-min option is no longer supported. ++ ++ To specify a target minimum for Darwin platforms, please explicitly add the appropriate options ++ to CCFLAGS and LINKFLAGS on the command line: ++ ++ macOS: scons CCFLAGS="-mmacosx-version-min=10.11" LINKFLAGS="-mmacosx-version-min=10.11" .. ++ iOS : scons CCFLAGS="-miphoneos-version-min=10.3" LINKFLAGS="-miphoneos-version-min=10.3" ... ++ tvOS : scons CCFLAGS="-mtvos-version-min=10.3" LINKFLAGS="-tvos-version-min=10.3" ... ++ ++ Note that MongoDB requires macOS 10.10, iOS 10.2, or tvOS 10.2 or later. ++ """ ++ myenv.ConfError(textwrap.dedent(message)) ++ ++ usingLibStdCxx = False ++ if has_option('libc++'): ++ if not myenv.ToolchainIs('clang'): ++ myenv.FatalError('libc++ is currently only supported for clang') ++ if AddToCXXFLAGSIfSupported(myenv, '-stdlib=libc++'): ++ myenv.Append(LINKFLAGS=['-stdlib=libc++']) ++ else: ++ myenv.ConfError('libc++ requested, but compiler does not support -stdlib=libc++' ) ++ else: ++ def CheckLibStdCxx(context): ++ test_body = """ ++ #include ++ #if !defined(__GLIBCXX__) ++ #error ++ #endif ++ """ ++ ++ context.Message('Checking if we are using libstdc++... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckLibStdCxx' : CheckLibStdCxx, ++ }) ++ usingLibStdCxx = conf.CheckLibStdCxx() ++ conf.Finish() ++ ++ if not myenv.ToolchainIs('msvc'): ++ if get_option('cxx-std') == "14": ++ if not AddToCXXFLAGSIfSupported(myenv, '-std=c++14'): ++ myenv.ConfError('Compiler does not honor -std=c++14') ++ if not AddToCFLAGSIfSupported(myenv, '-std=c11'): ++ myenv.ConfError("C++14 mode selected for C++ files, but can't enable C11 for C files") ++ ++ if using_system_version_of_cxx_libraries(): ++ print( 'WARNING: System versions of C++ libraries must be compiled with C++14 support' ) ++ ++ # We appear to have C++14, or at least a flag to enable it. Check that the declared C++ ++ # language level is not less than C++14, and that we can at least compile an 'auto' ++ # expression. We don't check the __cplusplus macro when using MSVC because as of our ++ # current required MS compiler version (MSVS 2015 Update 2), they don't set it. If ++ # MSFT ever decides (in MSVS 2017?) to define __cplusplus >= 201402L, remove the exception ++ # here for _MSC_VER ++ def CheckCxx14(context): ++ test_body = """ ++ #ifndef _MSC_VER ++ #if __cplusplus < 201402L ++ #error ++ #endif ++ #endif ++ auto DeducedReturnTypesAreACXX14Feature() { ++ return 0; ++ } ++ """ ++ ++ context.Message('Checking for C++14... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckCxx14' : CheckCxx14, ++ }) ++ ++ if not conf.CheckCxx14(): ++ myenv.ConfError('C++14 support is required to build MongoDB') ++ ++ conf.Finish() ++ ++ def CheckMemset_s(context): ++ test_body = """ ++ #define __STDC_WANT_LIB_EXT1__ 1 ++ #include ++ int main(int argc, char* argv[]) { ++ void* data = nullptr; ++ return memset_s(data, 0, 0, 0); ++ } ++ """ ++ ++ context.Message('Checking for memset_s... ') ++ ret = context.TryLink(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(env, custom_tests = { ++ 'CheckMemset_s' : CheckMemset_s, ++ }) ++ if conf.CheckMemset_s(): ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_MEMSET_S") ++ ++ if conf.CheckFunc('strnlen'): ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_STRNLEN") ++ ++ conf.Finish() ++ ++ # If we are using libstdc++, check to see if we are using a ++ # libstdc++ that is older than our GCC minimum of 5.3.0. This is ++ # primarly to help people using clang on OS X but forgetting to ++ # use --libc++ (or set the target OS X version high enough to get ++ # it as the default). We would, ideally, check the __GLIBCXX__ ++ # version, but for various reasons this is not workable. Instead, ++ # we switch on the fact that the header ++ # wasn't introduced until libstdc++ 5.3.0. Yes, this is a terrible ++ # hack. ++ if usingLibStdCxx: ++ def CheckModernLibStdCxx(context): ++ test_body = """ ++ #if !__has_include() ++ #error "libstdc++ from GCC 5.3.0 or newer is required" ++ #endif ++ """ ++ ++ context.Message('Checking for libstdc++ 5.3.0 or better... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckModernLibStdCxx' : CheckModernLibStdCxx, ++ }) ++ ++ suppress_invalid = has_option("disable-minimum-compiler-version-enforcement") ++ if not conf.CheckModernLibStdCxx() and not suppress_invalid: ++ myenv.ConfError("When using libstdc++, MongoDB requires libstdc++ from GCC 5.3.0 or newer") ++ ++ conf.Finish() ++ ++ if has_option("use-glibcxx-debug"): ++ # If we are using a modern libstdc++ and this is a debug build and we control all C++ ++ # dependencies, then turn on the debugging features in libstdc++. ++ # TODO: Need a new check here. ++ if not debugBuild: ++ myenv.FatalError("--use-glibcxx-debug requires --dbg=on") ++ if not usingLibStdCxx: ++ myenv.FatalError("--use-glibcxx-debug is only compatible with the GNU implementation " ++ "of the C++ standard libary") ++ if using_system_version_of_cxx_libraries(): ++ myenv.FatalError("--use-glibcxx-debug not compatible with system versions of " ++ "C++ libraries.") ++ myenv.Append(CPPDEFINES=["_GLIBCXX_DEBUG"]); ++ ++ # Check if we have a modern Windows SDK ++ if env.TargetOSIs('windows'): ++ def CheckWindowsSDKVersion(context): ++ ++ test_body = """ ++ #include ++ #if !defined(NTDDI_WINBLUE) ++ #error Need Windows SDK Version 8.1 or higher ++ #endif ++ """ ++ ++ context.Message('Checking Windows SDK is 8.1 or newer... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckWindowsSDKVersion' : CheckWindowsSDKVersion, ++ }) ++ ++ if not conf.CheckWindowsSDKVersion(): ++ myenv.ConfError('Windows SDK Version 8.1 or higher is required to build MongoDB') ++ ++ conf.Finish() ++ ++ # Check if we are on a POSIX system by testing if _POSIX_VERSION is defined. ++ def CheckPosixSystem(context): ++ ++ test_body = """ ++ // POSIX requires the existence of unistd.h, so if we can't include unistd.h, we ++ // are definitely not a POSIX system. ++ #include ++ #if !defined(_POSIX_VERSION) ++ #error not a POSIX system ++ #endif ++ """ ++ ++ context.Message('Checking if we are on a POSIX system... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckPosixSystem' : CheckPosixSystem, ++ }) ++ posix_system = conf.CheckPosixSystem() ++ ++ conf.Finish() ++ ++ # Check if we are on a system that support the POSIX clock_gettime function ++ # and the "monotonic" clock. ++ posix_monotonic_clock = False ++ if posix_system: ++ def CheckPosixMonotonicClock(context): ++ ++ test_body = """ ++ #include ++ #if !(defined(_POSIX_TIMERS) && _POSIX_TIMERS > 0) ++ #error POSIX clock_gettime not supported ++ #elif !(defined(_POSIX_MONOTONIC_CLOCK) && _POSIX_MONOTONIC_CLOCK >= 0) ++ #error POSIX monotonic clock not supported ++ #endif ++ """ ++ ++ context.Message('Checking if the POSIX monotonic clock is supported... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckPosixMonotonicClock' : CheckPosixMonotonicClock, ++ }) ++ posix_monotonic_clock = conf.CheckPosixMonotonicClock() ++ ++ # On 32-bit systems, we need to define this in order to get access to ++ # the 64-bit versions of fseek, etc. ++ # except on 32 bit android where it breaks boost ++ if not conf.CheckTypeSize('off_t', includes="#include ", expect=8): ++ if not env.TargetOSIs('android'): ++ myenv.Append(CPPDEFINES=["_FILE_OFFSET_BITS=64"]) ++ ++ conf.Finish() ++ ++ if has_option('sanitize'): ++ ++ if not myenv.ToolchainIs('clang', 'gcc'): ++ env.FatalError('sanitize is only supported with clang or gcc') ++ ++ if myenv.ToolchainIs('gcc'): ++ # GCC's implementation of ASAN depends on libdl. ++ env.Append(LIBS=['dl']) ++ ++ if env['MONGO_ALLOCATOR'] == 'tcmalloc': ++ # There are multiply defined symbols between the sanitizer and ++ # our vendorized tcmalloc. ++ env.FatalError("Cannot use --sanitize with tcmalloc") ++ ++ sanitizer_list = get_option('sanitize').split(',') ++ ++ using_lsan = 'leak' in sanitizer_list ++ using_asan = 'address' in sanitizer_list or using_lsan ++ using_tsan = 'thread' in sanitizer_list ++ using_ubsan = 'undefined' in sanitizer_list ++ ++ # If the user asked for leak sanitizer, turn on the detect_leaks ++ # ASAN_OPTION. If they asked for address sanitizer as well, drop ++ # 'leak', because -fsanitize=leak means no address. ++ # ++ # --sanitize=leak: -fsanitize=leak, detect_leaks=1 ++ # --sanitize=address,leak: -fsanitize=address, detect_leaks=1 ++ # --sanitize=address: -fsanitize=address ++ # ++ if using_lsan: ++ if using_asan: ++ myenv['ENV']['ASAN_OPTIONS'] = "detect_leaks=1" ++ myenv['ENV']['LSAN_OPTIONS'] = "suppressions=%s" % myenv.File("#etc/lsan.suppressions").abspath ++ if 'address' in sanitizer_list: ++ sanitizer_list.remove('leak') ++ ++ sanitizer_option = '-fsanitize=' + ','.join(sanitizer_list) ++ ++ if AddToCCFLAGSIfSupported(myenv, sanitizer_option): ++ myenv.Append(LINKFLAGS=[sanitizer_option]) ++ myenv.Append(CCFLAGS=['-fno-omit-frame-pointer']) ++ else: ++ myenv.ConfError('Failed to enable sanitizers with flag: {0}', sanitizer_option ) ++ ++ blackfiles_map = { ++ "address" : myenv.File("#etc/asan.blacklist"), ++ "leak" : myenv.File("#etc/asan.blacklist"), ++ "thread" : myenv.File("#etc/tsan.blacklist"), ++ "undefined" : myenv.File("#etc/ubsan.blacklist"), ++ } ++ ++ # Select those unique black files that are associated with the ++ # currently enabled sanitizers, but filter out those that are ++ # zero length. ++ blackfiles = {v for (k, v) in blackfiles_map.iteritems() if k in sanitizer_list} ++ blackfiles = [f for f in blackfiles if os.stat(f.path).st_size != 0] ++ ++ # Filter out any blacklist options that the toolchain doesn't support. ++ supportedBlackfiles = [] ++ blackfilesTestEnv = myenv.Clone() ++ for blackfile in blackfiles: ++ if AddToCCFLAGSIfSupported(blackfilesTestEnv, "-fsanitize-blacklist=%s" % blackfile): ++ supportedBlackfiles.append(blackfile) ++ blackfilesTestEnv = None ++ blackfiles = sorted(supportedBlackfiles) ++ ++ # If we ended up with any blackfiles after the above filters, ++ # then expand them into compiler flag arguments, and use a ++ # generator to return at command line expansion time so that ++ # we can change the signature if the file contents change. ++ if blackfiles: ++ blacklist_options=["-fsanitize-blacklist=%s" % blackfile for blackfile in blackfiles] ++ def SanitizerBlacklistGenerator(source, target, env, for_signature): ++ if for_signature: ++ return [f.get_csig() for f in blackfiles] ++ return blacklist_options ++ myenv.AppendUnique( ++ SANITIZER_BLACKLIST_GENERATOR=SanitizerBlacklistGenerator, ++ CCFLAGS="${SANITIZER_BLACKLIST_GENERATOR}", ++ LINKFLAGS="${SANITIZER_BLACKLIST_GENERATOR}", ++ ) ++ ++ llvm_symbolizer = get_option('llvm-symbolizer') ++ if os.path.isabs(llvm_symbolizer): ++ if not myenv.File(llvm_symbolizer).exists(): ++ print("WARNING: Specified symbolizer '%s' not found" % llvm_symbolizer) ++ llvm_symbolizer = None ++ else: ++ llvm_symbolizer = myenv.WhereIs(llvm_symbolizer) ++ ++ tsan_options = "" ++ if llvm_symbolizer: ++ myenv['ENV']['ASAN_SYMBOLIZER_PATH'] = llvm_symbolizer ++ myenv['ENV']['LSAN_SYMBOLIZER_PATH'] = llvm_symbolizer ++ tsan_options = "external_symbolizer_path=\"%s\" " % llvm_symbolizer ++ elif using_lsan: ++ myenv.FatalError("Using the leak sanitizer requires a valid symbolizer") ++ ++ if using_tsan: ++ tsan_options += "suppressions=\"%s\" " % myenv.File("#etc/tsan.suppressions").abspath ++ myenv['ENV']['TSAN_OPTIONS'] = tsan_options ++ ++ if using_ubsan: ++ # By default, undefined behavior sanitizer doesn't stop on ++ # the first error. Make it so. Newer versions of clang ++ # have renamed the flag. ++ if not AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover"): ++ AddToCCFLAGSIfSupported(myenv, "-fno-sanitize-recover=undefined") ++ ++ if myenv.ToolchainIs('msvc') and optBuild: ++ # http://blogs.msdn.com/b/vcblog/archive/2013/09/11/introducing-gw-compiler-switch.aspx ++ # ++ myenv.Append( CCFLAGS=["/Gw", "/Gy"] ) ++ myenv.Append( LINKFLAGS=["/OPT:REF"]) ++ ++ # http://blogs.msdn.com/b/vcblog/archive/2014/03/25/linker-enhancements-in-visual-studio-2013-update-2-ctp2.aspx ++ # ++ myenv.Append( CCFLAGS=["/Zc:inline"]) ++ ++ if myenv.ToolchainIs('gcc', 'clang'): ++ # This tells clang/gcc to use the gold linker if it is available - we prefer the gold linker ++ # because it is much faster. Don't use it if the user has already configured another linker ++ # selection manually. ++ if not any(flag.startswith('-fuse-ld=') for flag in env['LINKFLAGS']): ++ AddToLINKFLAGSIfSupported(myenv, '-fuse-ld=gold') ++ ++ # Explicitly enable GNU build id's if the linker supports it. ++ AddToLINKFLAGSIfSupported(myenv, '-Wl,--build-id') ++ ++ # Explicitly use the new gnu hash section if the linker offers ++ # it, except on android since older runtimes seem to not ++ # support it. For that platform, use 'both'. ++ if env.TargetOSIs('android'): ++ AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=both') ++ else: ++ AddToLINKFLAGSIfSupported(myenv, '-Wl,--hash-style=gnu') ++ ++ # Try to have the linker tell us about ODR violations. Don't ++ # use it when using clang with libstdc++, as libstdc++ was ++ # probably built with GCC. That combination appears to cause ++ # false positives for the ODR detector. See SERVER-28133 for ++ # additional details. ++ if (get_option('detect-odr-violations') and ++ not (myenv.ToolchainIs('clang') and usingLibStdCxx)): ++ AddToLINKFLAGSIfSupported(myenv, '-Wl,--detect-odr-violations') ++ ++ # Disallow an executable stack. Also, issue a warning if any files are found that would ++ # cause the stack to become executable if the noexecstack flag was not in play, so that we ++ # can find them and fix them. We do this here after we check for ld.gold because the ++ # --warn-execstack is currently only offered with gold. ++ # ++ # TODO: Add -Wl,--fatal-warnings once WT-2629 is fixed. We probably can do that ++ # unconditionally above, and not need to do it as an AddToLINKFLAGSIfSupported step, since ++ # both gold and binutils ld both support it. ++ AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,noexecstack") ++ AddToLINKFLAGSIfSupported(myenv, "-Wl,--warn-execstack") ++ ++ # If possible with the current linker, mark relocations as read-only. ++ AddToLINKFLAGSIfSupported(myenv, "-Wl,-z,relro") ++ ++ # Avoid deduping symbols on OS X debug builds, as it takes a long time. ++ if not optBuild and myenv.ToolchainIs('clang') and env.TargetOSIs('darwin'): ++ AddToLINKFLAGSIfSupported(myenv, "-Wl,-no_deduplicate") ++ ++ # Apply any link time optimization settings as selected by the 'lto' option. ++ if has_option('lto'): ++ if myenv.ToolchainIs('msvc'): ++ # Note that this is actually more aggressive than LTO, it is whole program ++ # optimization due to /GL. However, this is historically what we have done for ++ # windows, so we are keeping it. ++ # ++ # /GL implies /LTCG, so no need to say it in CCFLAGS, but we do need /LTCG on the ++ # link flags. ++ myenv.Append(CCFLAGS=['/GL']) ++ myenv.Append(LINKFLAGS=['/LTCG']) ++ myenv.Append(ARFLAGS=['/LTCG']) ++ elif myenv.ToolchainIs('gcc', 'clang'): ++ # For GCC and clang, the flag is -flto, and we need to pass it both on the compile ++ # and link lines. ++ if not AddToCCFLAGSIfSupported(myenv, '-flto') or \ ++ not AddToLINKFLAGSIfSupported(myenv, '-flto'): ++ myenv.ConfError("Link time optimization requested, " ++ "but selected compiler does not honor -flto" ) ++ ++ if myenv.TargetOSIs('darwin'): ++ AddToLINKFLAGSIfSupported(myenv, '-Wl,-object_path_lto,${TARGET}.lto') ++ ++ else: ++ myenv.ConfError("Don't know how to enable --lto on current toolchain") ++ ++ if get_option('runtime-hardening') == "on" and optBuild: ++ # Older glibc doesn't work well with _FORTIFY_SOURCE=2. Selecting 2.11 as the minimum was an ++ # emperical decision, as that is the oldest non-broken glibc we seem to require. It is possible ++ # that older glibc's work, but we aren't trying. ++ # ++ # https://gforge.inria.fr/tracker/?func=detail&group_id=131&atid=607&aid=14070 ++ # https://github.com/jedisct1/libsodium/issues/202 ++ def CheckForGlibcKnownToSupportFortify(context): ++ test_body=""" ++ #include ++ #if !__GLIBC_PREREQ(2, 11) ++ #error ++ #endif ++ """ ++ context.Message('Checking for glibc with non-broken _FORTIFY_SOURCE...') ++ ret = context.TryCompile(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckForFortify': CheckForGlibcKnownToSupportFortify, ++ }) ++ ++ # Fortify only possibly makes sense on POSIX systems, and we know that clang is not a valid ++ # combination: ++ # ++ # http://lists.llvm.org/pipermail/cfe-dev/2015-November/045852.html ++ # ++ if env.TargetOSIs('posix') and not env.ToolchainIs('clang') and conf.CheckForFortify(): ++ conf.env.Append( ++ CPPDEFINES=[ ++ ('_FORTIFY_SOURCE', 2), ++ ], ++ ) ++ ++ myenv = conf.Finish() ++ ++ # We set this to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43052 ++ if not myenv.ToolchainIs('msvc'): ++ AddToCCFLAGSIfSupported(myenv, "-fno-builtin-memcmp") ++ ++ def CheckThreadLocal(context): ++ test_body = """ ++ thread_local int tsp_int = 1; ++ int main(int argc, char** argv) {{ ++ return !(tsp_int == argc); ++ }} ++ """ ++ context.Message('Checking for storage class thread_local ') ++ ret = context.TryLink(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckThreadLocal': CheckThreadLocal ++ }) ++ if not conf.CheckThreadLocal(): ++ env.ConfError("Compiler must support the thread_local storage class") ++ conf.Finish() ++ ++ def CheckCXX14EnableIfT(context): ++ test_body = """ ++ #include ++ #include ++ ++ template ++ struct scons { ++ bool hasSupport() { return false; } ++ }; ++ ++ template <> ++ struct scons> { ++ bool hasSupport() { return true; } ++ }; ++ ++ int main(int argc, char **argv) { ++ scons<> SCons; ++ return SCons.hasSupport() ? EXIT_SUCCESS : EXIT_FAILURE; ++ } ++ """ ++ context.Message('Checking for C++14 std::enable_if_t support...') ++ ret = context.TryCompile(textwrap.dedent(test_body), '.cpp') ++ context.Result(ret) ++ return ret ++ ++ # Check for std::enable_if_t support without using the __cplusplus macro ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckCXX14EnableIfT' : CheckCXX14EnableIfT, ++ }) ++ ++ if conf.CheckCXX14EnableIfT(): ++ conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_STD_ENABLE_IF_T') ++ ++ myenv = conf.Finish() ++ ++ def CheckCXX14MakeUnique(context): ++ test_body = """ ++ #include ++ int main(int argc, char **argv) { ++ auto foo = std::make_unique(5); ++ return 0; ++ } ++ """ ++ context.Message('Checking for C++14 std::make_unique support... ') ++ ret = context.TryCompile(textwrap.dedent(test_body), '.cpp') ++ context.Result(ret) ++ return ret ++ ++ # Check for std::make_unique support without using the __cplusplus macro ++ conf = Configure(myenv, help=False, custom_tests = { ++ 'CheckCXX14MakeUnique': CheckCXX14MakeUnique, ++ }) ++ ++ if conf.CheckCXX14MakeUnique(): ++ conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_STD_MAKE_UNIQUE') ++ ++ # pthread_setname_np was added in GLIBC 2.12, and Solaris 11.3 ++ if posix_system: ++ myenv = conf.Finish() ++ ++ def CheckPThreadSetNameNP(context): ++ compile_test_body = textwrap.dedent(""" ++ #ifndef _GNU_SOURCE ++ #define _GNU_SOURCE ++ #endif ++ #include ++ ++ int main() { ++ pthread_setname_np(pthread_self(), "test"); ++ return 0; ++ } ++ """) ++ ++ context.Message("Checking if pthread_setname_np is supported... ") ++ result = context.TryCompile(compile_test_body, ".cpp") ++ context.Result(result) ++ return result ++ ++ conf = Configure(myenv, custom_tests = { ++ 'CheckPThreadSetNameNP': CheckPThreadSetNameNP, ++ }) ++ ++ if conf.CheckPThreadSetNameNP(): ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_PTHREAD_SETNAME_NP") ++ ++ myenv = conf.Finish() ++ ++ def CheckBoostMinVersion(context): ++ compile_test_body = textwrap.dedent(""" ++ #include ++ ++ #if BOOST_VERSION < 104900 ++ #error ++ #endif ++ """) ++ ++ context.Message("Checking if system boost version is 1.49 or newer...") ++ result = context.TryCompile(compile_test_body, ".cpp") ++ context.Result(result) ++ return result ++ ++ conf = Configure(myenv, custom_tests = { ++ 'CheckBoostMinVersion': CheckBoostMinVersion, ++ }) ++ ++ libdeps.setup_conftests(conf) ++ ++ ### --ssl and --ssl-provider checks ++ def checkOpenSSL(conf): ++ sslLibName = "ssl" ++ cryptoLibName = "crypto" ++ sslLinkDependencies = ["crypto", "dl"] ++ if conf.env.TargetOSIs('freebsd'): ++ sslLinkDependencies = ["crypto"] ++ ++ if conf.env.TargetOSIs('windows'): ++ sslLibName = "ssleay32" ++ cryptoLibName = "libeay32" ++ sslLinkDependencies = ["libeay32"] ++ ++ # Used to import system certificate keychains ++ if conf.env.TargetOSIs('darwin'): ++ conf.env.AppendUnique(FRAMEWORKS=[ ++ 'CoreFoundation', ++ 'Security', ++ ]) ++ ++ def maybeIssueDarwinSSLAdvice(env): ++ if env.TargetOSIs('macOS'): ++ advice = textwrap.dedent( ++ """\ ++ NOTE: Recent versions of macOS no longer ship headers for the system OpenSSL libraries. ++ NOTE: Either build without the --ssl flag, or describe how to find OpenSSL. ++ NOTE: Set the include path for the OpenSSL headers with the CPPPATH SCons variable. ++ NOTE: Set the library path for OpenSSL libraries with the LIBPATH SCons variable. ++ NOTE: If you are using HomeBrew, and have installed OpenSSL, this might look like: ++ \tscons CPPPATH=/usr/local/opt/openssl/include LIBPATH=/usr/local/opt/openssl/lib ... ++ NOTE: Consult the output of 'brew info openssl' for details on the correct paths.""" ++ ) ++ print(advice) ++ brew = env.WhereIs('brew') ++ if brew: ++ try: ++ # TODO: If we could programmatically extract the paths from the info output ++ # we could give a better message here, but brew info's machine readable output ++ # doesn't seem to include the whole 'caveats' section. ++ message = subprocess.check_output([brew, "info", "openssl"]) ++ advice = textwrap.dedent( ++ """\ ++ NOTE: HomeBrew installed to {0} appears to have OpenSSL installed. ++ NOTE: Consult the output from '{0} info openssl' to determine CPPPATH and LIBPATH.""" ++ ).format(brew, message) ++ ++ print(advice) ++ except: ++ pass ++ ++ if not conf.CheckLibWithHeader( ++ cryptoLibName, ++ ["openssl/crypto.h"], ++ "C", ++ "SSLeay_version(0);", ++ autoadd=True): ++ maybeIssueDarwinSSLAdvice(conf.env) ++ conf.env.ConfError("Couldn't find OpenSSL crypto.h header and library") ++ ++ def CheckLibSSL(context): ++ res = SCons.Conftest.CheckLib(context, ++ libs=[sslLibName], ++ extra_libs=sslLinkDependencies, ++ header='#include "openssl/ssl.h"', ++ language="C", ++ call="SSL_version(NULL);", ++ autoadd=True) ++ context.did_show_result = 1 ++ return not res ++ ++ conf.AddTest("CheckLibSSL", CheckLibSSL) ++ ++ if not conf.CheckLibSSL(): ++ maybeIssueDarwinSSLAdvice(conf.env) ++ conf.env.ConfError("Couldn't find OpenSSL ssl.h header and library") ++ ++ def CheckLinkSSL(context): ++ test_body = """ ++ #include ++ #include ++ #include ++ ++ int main() { ++ SSL_library_init(); ++ SSL_load_error_strings(); ++ ERR_load_crypto_strings(); ++ ++ OpenSSL_add_all_algorithms(); ++ ERR_free_strings(); ++ ++ return EXIT_SUCCESS; ++ } ++ """ ++ context.Message("Checking that linking to OpenSSL works...") ++ ret = context.TryLink(textwrap.dedent(test_body), ".c") ++ context.Result(ret) ++ return ret ++ ++ conf.AddTest("CheckLinkSSL", CheckLinkSSL) ++ ++ if not conf.CheckLinkSSL(): ++ maybeIssueDarwinSSLAdvice(conf.env) ++ conf.env.ConfError("SSL is enabled, but is unavailable") ++ ++ if conf.CheckDeclaration( ++ "FIPS_mode_set", ++ includes=""" ++ #include ++ #include ++ """): ++ conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_FIPS_MODE_SET') ++ ++ if conf.CheckDeclaration( ++ "d2i_ASN1_SEQUENCE_ANY", ++ includes=""" ++ #include ++ """): ++ conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_ASN1_ANY_DEFINITIONS') ++ ++ def CheckOpenSSL_EC_DH(context): ++ compile_test_body = textwrap.dedent(""" ++ #include ++ ++ int main() { ++ SSL_CTX_set_ecdh_auto(0, 0); ++ SSL_set_ecdh_auto(0, 0); ++ return 0; ++ } ++ """) ++ ++ context.Message("Checking if SSL_[CTX_]_set_ecdh_auto is supported... ") ++ result = context.TryCompile(compile_test_body, ".cpp") ++ context.Result(result) ++ return result ++ ++ conf.AddTest("CheckOpenSSL_EC_DH", CheckOpenSSL_EC_DH) ++ if conf.CheckOpenSSL_EC_DH(): ++ conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_SSL_SET_ECDH_AUTO') ++ ++ ssl_provider = get_option("ssl-provider") ++ if ssl_provider == 'auto': ++ if conf.env.TargetOSIs('windows', 'darwin', 'macOS'): ++ ssl_provider = 'native' ++ else: ++ ssl_provider = 'openssl' ++ ++ if ssl_provider == 'native': ++ if conf.env.TargetOSIs('windows'): ++ ssl_provider = 'windows' ++ env.SetConfigHeaderDefine("MONGO_CONFIG_SSL_PROVIDER", "MONGO_CONFIG_SSL_PROVIDER_WINDOWS") ++ conf.env.Append( MONGO_CRYPTO=["windows"] ) ++ ++ elif conf.env.TargetOSIs('darwin', 'macOS'): ++ ssl_provider = 'apple' ++ env.SetConfigHeaderDefine("MONGO_CONFIG_SSL_PROVIDER", "MONGO_CONFIG_SSL_PROVIDER_APPLE") ++ conf.env.Append( MONGO_CRYPTO=["apple"] ) ++ conf.env.AppendUnique(FRAMEWORKS=[ ++ 'CoreFoundation', ++ 'Security', ++ ]) ++ ++ if ssl_provider == 'openssl': ++ if has_option("ssl"): ++ checkOpenSSL(conf) ++ # Working OpenSSL available, use it. ++ env.SetConfigHeaderDefine("MONGO_CONFIG_SSL_PROVIDER", "MONGO_CONFIG_SSL_PROVIDER_OPENSSL") ++ ++ conf.env.Append( MONGO_CRYPTO=["openssl"] ) ++ else: ++ # If we don't need an SSL build, we can get by with TomCrypt. ++ conf.env.Append( MONGO_CRYPTO=["tom"] ) ++ ++ if has_option( "ssl" ): ++ # Either crypto engine is native, ++ # or it's OpenSSL and has been checked to be working. ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_SSL") ++ print("Using SSL Provider: {0}".format(ssl_provider)) ++ else: ++ ssl_provider = "none" ++ ++ # The Windows build needs the openssl binaries if it targets openssl or includes the tools ++ # since the tools link against openssl ++ if conf.env.TargetOSIs('windows') and (ssl_provider == "openssl" or has_option("use-new-tools")): ++ # Add the SSL binaries to the zip file distribution ++ def addOpenSslLibraryToDistArchive(file_name): ++ openssl_bin_path = os.path.normpath(env['WINDOWS_OPENSSL_BIN'].lower()) ++ full_file_name = os.path.join(openssl_bin_path, file_name) ++ if os.path.exists(full_file_name): ++ env.Append(ARCHIVE_ADDITIONS=[full_file_name]) ++ env.Append(ARCHIVE_ADDITION_DIR_MAP={ ++ openssl_bin_path: "bin" ++ }) ++ return True ++ else: ++ return False ++ ++ files = ['ssleay32.dll', 'libeay32.dll'] ++ for extra_file in files: ++ if not addOpenSslLibraryToDistArchive(extra_file): ++ print("WARNING: Cannot find SSL library '%s'" % extra_file) ++ ++ ++ ++ if free_monitoring == "auto": ++ if "enterprise" not in env['MONGO_MODULES']: ++ free_monitoring = "on" ++ else: ++ free_monitoring = "off" ++ ++ if not env.TargetOSIs("windows") \ ++ and free_monitoring == "on" \ ++ and not conf.CheckLibWithHeader( ++ "curl", ++ ["curl/curl.h"], "C", ++ "curl_global_init(0);", ++ autoadd=False): ++ env.ConfError("Could not find and curl lib") ++ ++ if use_system_version_of_library("pcre"): ++ conf.FindSysLibDep("pcre", ["pcre"]) ++ conf.FindSysLibDep("pcrecpp", ["pcrecpp"]) ++ else: ++ env.Prepend(CPPDEFINES=['PCRE_STATIC']) ++ ++ if use_system_version_of_library("snappy"): ++ conf.FindSysLibDep("snappy", ["snappy"]) ++ ++ if use_system_version_of_library("zlib"): ++ conf.FindSysLibDep("zlib", ["zdll" if conf.env.TargetOSIs('windows') else "z"]) ++ ++ if use_system_version_of_library("stemmer"): ++ conf.FindSysLibDep("stemmer", ["stemmer"]) ++ ++ if use_system_version_of_library("yaml"): ++ conf.FindSysLibDep("yaml", ["yaml-cpp"]) ++ ++ if use_system_version_of_library("intel_decimal128"): ++ conf.FindSysLibDep("intel_decimal128", ["bid"]) ++ ++ if use_system_version_of_library("icu"): ++ conf.FindSysLibDep("icudata", ["icudata"]) ++ # We can't use FindSysLibDep() for icui18n and icuuc below, since SConf.CheckLib() (which ++ # FindSysLibDep() relies on) doesn't expose an 'extra_libs' parameter to indicate that the ++ # library being tested has additional dependencies (icuuc depends on icudata, and icui18n ++ # depends on both). As a workaround, we skip the configure check for these two libraries and ++ # manually assign the library name. We hope that if the user has icudata installed on their ++ # system, then they also have icu18n and icuuc installed. ++ conf.env['LIBDEPS_ICUI18N_SYSLIBDEP'] = 'icui18n' ++ conf.env['LIBDEPS_ICUUC_SYSLIBDEP'] = 'icuuc' ++ ++ if wiredtiger and use_system_version_of_library("wiredtiger"): ++ if not conf.CheckCXXHeader( "wiredtiger.h" ): ++ myenv.ConfError("Cannot find wiredtiger headers") ++ conf.FindSysLibDep("wiredtiger", ["wiredtiger"]) ++ ++ if use_system_version_of_library("sqlite"): ++ if not conf.CheckCXXHeader( "sqlite3.h" ): ++ myenv.ConfError("Cannot find sqlite headers") ++ conf.FindSysLibDep("sqlite", ["sqlite3"]) ++ ++ conf.env.Append( ++ CPPDEFINES=[ ++ "BOOST_SYSTEM_NO_DEPRECATED", ++ "BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS", ++ ] ++ ) ++ ++ if use_system_version_of_library("boost"): ++ if not conf.CheckCXXHeader( "boost/filesystem/operations.hpp" ): ++ myenv.ConfError("can't find boost headers") ++ if not conf.CheckBoostMinVersion(): ++ myenv.ConfError("system's version of boost is too old. version 1.49 or better required") ++ ++ # Note that on Windows with using-system-boost builds, the following ++ # FindSysLibDep calls do nothing useful (but nothing problematic either) ++ # ++ # NOTE: Pass --system-boost-lib-search-suffixes= to suppress these checks, which you ++ # might want to do if using autolib linking on Windows, for example. ++ if boostSuffixList: ++ for b in boostLibs: ++ boostlib = "boost_" + b ++ conf.FindSysLibDep( ++ boostlib, ++ [boostlib + suffix for suffix in boostSuffixList], ++ language='C++') ++ if posix_system: ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_HEADER_UNISTD_H") ++ conf.CheckLib('rt') ++ conf.CheckLib('dl') ++ ++ if posix_monotonic_clock: ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK") ++ ++ if (conf.CheckCXXHeader( "execinfo.h" ) and ++ conf.CheckDeclaration('backtrace', includes='#include ') and ++ conf.CheckDeclaration('backtrace_symbols', includes='#include ') and ++ conf.CheckDeclaration('backtrace_symbols_fd', includes='#include ')): ++ ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_EXECINFO_BACKTRACE") ++ ++ conf.env["_HAVEPCAP"] = conf.CheckLib( ["pcap", "wpcap"], autoadd=False ) ++ ++ if env.TargetOSIs('solaris'): ++ conf.CheckLib( "nsl" ) ++ ++ conf.env['MONGO_BUILD_SASL_CLIENT'] = bool(has_option("use-sasl-client")) ++ if conf.env['MONGO_BUILD_SASL_CLIENT'] and not conf.CheckLibWithHeader( ++ "sasl2", ++ ["stddef.h","sasl/sasl.h"], ++ "C", ++ "sasl_version_info(0, 0, 0, 0, 0, 0);", ++ autoadd=False ): ++ myenv.ConfError("Couldn't find SASL header/libraries") ++ ++ # requires ports devel/libexecinfo to be installed ++ if env.TargetOSIs('freebsd', 'openbsd'): ++ if not conf.CheckLib("execinfo"): ++ myenv.ConfError("Cannot find libexecinfo, please install devel/libexecinfo.") ++ ++ # 'tcmalloc' needs to be the last library linked. Please, add new libraries before this ++ # point. ++ if myenv['MONGO_ALLOCATOR'] == 'tcmalloc': ++ if use_system_version_of_library('tcmalloc'): ++ conf.FindSysLibDep("tcmalloc", ["tcmalloc"]) ++ elif myenv['MONGO_ALLOCATOR'] == 'system': ++ pass ++ else: ++ myenv.FatalError("Invalid --allocator parameter: $MONGO_ALLOCATOR") ++ ++ def CheckStdAtomic(context, base_type, extra_message): ++ test_body = """ ++ #include ++ ++ int main() {{ ++ std::atomic<{0}> x; ++ ++ x.store(0); ++ {0} y = 1; ++ x.fetch_add(y); ++ x.fetch_sub(y); ++ x.exchange(y); ++ x.compare_exchange_strong(y, x); ++ x.is_lock_free(); ++ return x.load(); ++ }} ++ """.format(base_type) ++ ++ context.Message( ++ "Checking if std::atomic<{0}> works{1}... ".format( ++ base_type, extra_message ++ ) ++ ) ++ ++ ret = context.TryLink(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ conf.AddTest("CheckStdAtomic", CheckStdAtomic) ++ ++ def check_all_atomics(extra_message=''): ++ for t in ('int64_t', 'uint64_t', 'int32_t', 'uint32_t'): ++ if not conf.CheckStdAtomic(t, extra_message): ++ return False ++ return True ++ ++ if not check_all_atomics(): ++ if not conf.CheckLib('atomic', symbol=None, header=None, language='C', autoadd=1): ++ myenv.ConfError("Some atomic ops are not intrinsically supported, but " ++ "no libatomic found") ++ if not check_all_atomics(' with libatomic'): ++ myenv.ConfError("The toolchain does not support std::atomic, cannot continue") ++ ++ def CheckExtendedAlignment(context, size): ++ test_body = """ ++ #include ++ #include ++ #include ++ ++ static_assert(alignof(std::max_align_t) < {0}, "whatever"); ++ ++ alignas({0}) std::mutex aligned_mutex; ++ alignas({0}) std::atomic aligned_atomic; ++ ++ struct alignas({0}) aligned_struct_mutex {{ ++ std::mutex m; ++ }}; ++ ++ struct alignas({0}) aligned_struct_atomic {{ ++ std::atomic m; ++ }}; ++ ++ struct holds_aligned_mutexes {{ ++ alignas({0}) std::mutex m1; ++ alignas({0}) std::mutex m2; ++ }} hm; ++ ++ struct holds_aligned_atomics {{ ++ alignas({0}) std::atomic a1; ++ alignas({0}) std::atomic a2; ++ }} ha; ++ """.format(size) ++ ++ context.Message('Checking for extended alignment {0} for concurrency types... '.format(size)) ++ ret = context.TryCompile(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret) ++ return ret ++ ++ conf.AddTest('CheckExtendedAlignment', CheckExtendedAlignment) ++ ++ # If we don't have a specialized search sequence for this ++ # architecture, assume 64 byte cache lines, which is pretty ++ # standard. If for some reason the compiler can't offer that, try ++ # 32. ++ default_alignment_search_sequence = [ 64, 32 ] ++ ++ # The following are the target architectures for which we have ++ # some knowledge that they have larger cache line sizes. In ++ # particular, POWER8 uses 128 byte lines and zSeries uses 256. We ++ # start at the goal state, and work down until we find something ++ # the compiler can actualy do for us. ++ extended_alignment_search_sequence = { ++ 'ppc64le' : [ 128, 64, 32 ], ++ 's390x' : [ 256, 128, 64, 32 ], ++ } ++ ++ for size in extended_alignment_search_sequence.get(env['TARGET_ARCH'], default_alignment_search_sequence): ++ if conf.CheckExtendedAlignment(size): ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_MAX_EXTENDED_ALIGNMENT", size) ++ break ++ ++ def CheckMongoCMinVersion(context): ++ compile_test_body = textwrap.dedent(""" ++ #include ++ ++ #if !MONGOC_CHECK_VERSION(1,13,0) ++ #error ++ #endif ++ """) ++ ++ context.Message("Checking if mongoc version is 1.13.0 or newer...") ++ result = context.TryCompile(compile_test_body, ".cpp") ++ context.Result(result) ++ return result ++ ++ conf.AddTest('CheckMongoCMinVersion', CheckMongoCMinVersion) ++ ++ if env.TargetOSIs('darwin'): ++ def CheckMongoCFramework(context): ++ context.Message("Checking for mongoc_get_major_version() in darwin framework mongoc...") ++ test_body = """ ++ #include ++ ++ int main() { ++ mongoc_get_major_version(); ++ ++ return EXIT_SUCCESS; ++ } ++ """ ++ ++ lastFRAMEWORKS = context.env['FRAMEWORKS'] ++ context.env.Append(FRAMEWORKS=['mongoc']) ++ result = context.TryLink(textwrap.dedent(test_body), ".c") ++ context.Result(result) ++ context.env['FRAMEWORKS'] = lastFRAMEWORKS ++ return result ++ ++ conf.AddTest('CheckMongoCFramework', CheckMongoCFramework) ++ ++ mongoc_mode = get_option('use-system-mongo-c') ++ conf.env['MONGO_HAVE_LIBMONGOC'] = False ++ if mongoc_mode != 'off': ++ if conf.CheckLibWithHeader( ++ ["mongoc-1.0"], ++ ["mongoc/mongoc.h"], ++ "C", ++ "mongoc_get_major_version();", ++ autoadd=False ): ++ conf.env['MONGO_HAVE_LIBMONGOC'] = "library" ++ if not conf.env['MONGO_HAVE_LIBMONGOC'] and env.TargetOSIs('darwin') and conf.CheckMongoCFramework(): ++ conf.env['MONGO_HAVE_LIBMONGOC'] = "framework" ++ if not conf.env['MONGO_HAVE_LIBMONGOC'] and mongoc_mode == 'on': ++ myenv.ConfError("Failed to find the required C driver headers") ++ if conf.env['MONGO_HAVE_LIBMONGOC'] and not conf.CheckMongoCMinVersion(): ++ myenv.ConfError("Version of mongoc is too old. Version 1.13+ required") ++ ++ # ask each module to configure itself and the build environment. ++ moduleconfig.configure_modules(mongo_modules, conf) ++ ++ if env['TARGET_ARCH'] == "ppc64le": ++ # This checks for an altivec optimization we use in full text search. ++ # Different versions of gcc appear to put output bytes in different ++ # parts of the output vector produced by vec_vbpermq. This configure ++ # check looks to see which format the compiler produces. ++ # ++ # NOTE: This breaks cross compiles, as it relies on checking runtime functionality for the ++ # environment we're in. A flag to choose the index, or the possibility that we don't have ++ # multiple versions to support (after a compiler upgrade) could solve the problem if we ++ # eventually need them. ++ def CheckAltivecVbpermqOutput(context, index): ++ test_body = """ ++ #include ++ #include ++ #include ++ #include ++ ++ int main() {{ ++ using Native = __vector signed char; ++ const size_t size = sizeof(Native); ++ const Native bits = {{ 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0 }}; ++ ++ uint8_t inputBuf[size]; ++ std::memset(inputBuf, 0xFF, sizeof(inputBuf)); ++ ++ for (size_t offset = 0; offset <= size; offset++) {{ ++ Native vec = vec_vsx_ld(0, reinterpret_cast(inputBuf)); ++ ++ uint64_t mask = vec_extract(vec_vbpermq(vec, bits), {0}); ++ ++ size_t initialZeros = (mask == 0 ? size : __builtin_ctzll(mask)); ++ if (initialZeros != offset) {{ ++ return 1; ++ }} ++ ++ if (offset < size) {{ ++ inputBuf[offset] = 0; // Add an initial 0 for the next loop. ++ }} ++ }} ++ ++ return 0; ++ }} ++ """.format(index) ++ ++ context.Message('Checking for vec_vbperm output in index {0}... '.format(index)) ++ ret = context.TryRun(textwrap.dedent(test_body), ".cpp") ++ context.Result(ret[0]) ++ return ret[0] ++ ++ conf.AddTest('CheckAltivecVbpermqOutput', CheckAltivecVbpermqOutput) ++ ++ outputIndex = next((idx for idx in [0,1] if conf.CheckAltivecVbpermqOutput(idx)), None) ++ if outputIndex is not None: ++ conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) ++ else: ++ myenv.ConfError("Running on ppc64le, but can't find a correct vec_vbpermq output index. Compiler or platform not supported") ++ ++ return conf.Finish() ++ ++env = doConfigure( env ) ++ ++# TODO: Later, this should live somewhere more graceful. ++if get_option('install-mode') == 'hygienic': ++ ++ if get_option('separate-debug') == "on": ++ env.Tool('separate_debug') ++ ++ env.Tool('auto_install_binaries') ++ if env['PLATFORM'] == 'posix': ++ env.AppendUnique( ++ RPATH=[ ++ env.Literal('\\$$ORIGIN/../lib') ++ ], ++ LINKFLAGS=[ ++ # Most systems *require* -z,origin to make origin work, but android ++ # blows up at runtime if it finds DF_ORIGIN_1 in DT_FLAGS_1. ++ # https://android.googlesource.com/platform/bionic/+/cbc80ba9d839675a0c4891e2ab33f39ba51b04b2/linker/linker.h#68 ++ # https://android.googlesource.com/platform/bionic/+/cbc80ba9d839675a0c4891e2ab33f39ba51b04b2/libc/include/elf.h#215 ++ '-Wl,-z,origin' if not env.TargetOSIs('android') else [], ++ '-Wl,--enable-new-dtags', ++ ], ++ SHLINKFLAGS=[ ++ # -h works for both the sun linker and the gnu linker. ++ "-Wl,-h,${TARGET.file}", ++ ] ++ ) ++ elif env['PLATFORM'] == 'darwin': ++ env.AppendUnique( ++ LINKFLAGS=[ ++ '-Wl,-rpath,@loader_path/../lib', ++ '-Wl,-rpath,@loader_path/../Frameworks' ++ ], ++ SHLINKFLAGS=[ ++ "-Wl,-install_name,@rpath/${TARGET.file}", ++ ], ++ ) ++elif get_option('separate-debug') == "on": ++ env.FatalError('Cannot use --separate-debug without --install-mode=hygienic') ++ ++# Now that we are done with configure checks, enable icecream, if available. ++env.Tool('icecream') ++ ++# If the flags in the environment are configured for -gsplit-dwarf, ++# inject the necessary emitter. ++split_dwarf = Tool('split_dwarf') ++if split_dwarf.exists(env): ++ split_dwarf(env) ++ ++# Load the compilation_db tool. We want to do this after configure so we don't end up with ++# compilation database entries for the configure tests, which is weird. ++env.Tool("compilation_db") ++ ++# If we can, load the dagger tool for build dependency graph introspection. ++# Dagger is only supported on Linux and OSX (not Windows or Solaris). ++should_dagger = ( mongo_platform.is_running_os('osx') or mongo_platform.is_running_os('linux') ) and "dagger" in COMMAND_LINE_TARGETS ++ ++if should_dagger: ++ env.Tool("dagger") ++ ++incremental_link = Tool('incremental_link') ++if incremental_link.exists(env): ++ incremental_link(env) ++ ++def checkErrorCodes(): ++ import buildscripts.errorcodes as x ++ if x.check_error_codes() == False: ++ env.FatalError("next id to use: {0}", x.get_next_code()) ++ ++checkErrorCodes() ++ ++# Resource Files are Windows specific ++def env_windows_resource_file(env, path): ++ if env.TargetOSIs('windows'): ++ return [ env.RES(path) ] ++ else: ++ return [] ++ ++env.AddMethod(env_windows_resource_file, 'WindowsResourceFile') ++ ++# --- lint ---- ++ ++def doLint( env , target , source ): ++ import buildscripts.eslint ++ if not buildscripts.eslint.lint(None, dirmode=True, glob=["jstests/", "src/mongo/"]): ++ raise Exception("ESLint errors") ++ ++ import buildscripts.clang_format ++ if not buildscripts.clang_format.lint_all(None): ++ raise Exception("clang-format lint errors") ++ ++ import buildscripts.pylinters ++ buildscripts.pylinters.lint_all(None, {}, []) ++ ++ import buildscripts.lint ++ if not buildscripts.lint.run_lint( [ "src/mongo/" ] ): ++ raise Exception( "lint errors" ) ++ ++env.Alias( "lint" , [] , [ doLint ] ) ++env.AlwaysBuild( "lint" ) ++ ++ ++# ---- INSTALL ------- ++ ++def getSystemInstallName(): ++ arch_name = env.subst('$MONGO_DISTARCH') ++ ++ # We need to make sure the directory names inside dist tarballs are permanently ++ # consistent, even if the target OS name used in scons is different. Any differences ++ # between the names used by env.TargetOSIs/env.GetTargetOSName should be added ++ # to the translation dictionary below. ++ os_name_translations = { ++ 'windows': 'win32', ++ 'macOS': 'osx' ++ } ++ os_name = env.GetTargetOSName() ++ os_name = os_name_translations.get(os_name, os_name) ++ n = os_name + "-" + arch_name ++ ++ if len(mongo_modules): ++ n += "-" + "-".join(m.name for m in mongo_modules) ++ ++ dn = env.subst('$MONGO_DISTMOD') ++ if len(dn) > 0: ++ n = n + "-" + dn ++ ++ return n ++ ++# This function will add the version.txt file to the source tarball ++# so that versioning will work without having the git repo available. ++def add_version_to_distsrc(env, archive): ++ version_file_path = env.subst("$MONGO_DIST_SRC_PREFIX") + "version.json" ++ if version_file_path not in archive: ++ version_data = { ++ 'version': env['MONGO_VERSION'], ++ 'githash': env['MONGO_GIT_HASH'], ++ } ++ archive.append_file_contents( ++ version_file_path, ++ json.dumps( ++ version_data, ++ sort_keys=True, ++ indent=4, ++ separators=(',', ': ') ++ ) ++ ) ++ ++env.AddDistSrcCallback(add_version_to_distsrc) ++ ++env['SERVER_DIST_BASENAME'] = env.subst('mongodb-%s-$MONGO_DISTNAME' % (getSystemInstallName())) ++ ++module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules) ++ ++# The following symbols are exported for use in subordinate SConscript files. ++# Ideally, the SConscript files would be purely declarative. They would only ++# import build environment objects, and would contain few or no conditional ++# statements or branches. ++# ++# Currently, however, the SConscript files do need some predicates for ++# conditional decision making that hasn't been moved up to this SConstruct file, ++# and they are exported here, as well. ++Export("get_option") ++Export("has_option") ++Export("use_system_version_of_library") ++Export("serverJs") ++Export("usemozjs") ++Export('module_sconscripts') ++Export("debugBuild optBuild") ++Export("wiredtiger") ++Export("mmapv1") ++Export("mobile_se") ++Export("endian") ++Export("ssl_provider") ++Export("free_monitoring") ++ ++def injectMongoIncludePaths(thisEnv): ++ thisEnv.AppendUnique(CPPPATH=['$BUILD_DIR']) ++env.AddMethod(injectMongoIncludePaths, 'InjectMongoIncludePaths') ++ ++def injectModule(env, module, **kwargs): ++ injector = env['MODULE_INJECTORS'].get(module) ++ if injector: ++ return injector(env, **kwargs) ++ return env ++env.AddMethod(injectModule, 'InjectModule') ++ ++compileCommands = env.CompilationDatabase('compile_commands.json') ++compileDb = env.Alias("compiledb", compileCommands) ++ ++# Microsoft Visual Studio Project generation for code browsing ++vcxprojFile = env.Command( ++ "mongodb.vcxproj", ++ compileCommands, ++ r"$PYTHON buildscripts\make_vcxproj.py mongodb") ++vcxproj = env.Alias("vcxproj", vcxprojFile) ++ ++distSrc = env.DistSrc("mongodb-src-${MONGO_VERSION}.tar") ++env.NoCache(distSrc) ++env.Alias("distsrc-tar", distSrc) ++ ++distSrcGzip = env.GZip( ++ target="mongodb-src-${MONGO_VERSION}.tgz", ++ source=[distSrc]) ++env.NoCache(distSrcGzip) ++env.Alias("distsrc-tgz", distSrcGzip) ++ ++distSrcZip = env.DistSrc("mongodb-src-${MONGO_VERSION}.zip") ++env.NoCache(distSrcZip) ++env.Alias("distsrc-zip", distSrcZip) ++ ++env.Alias("distsrc", "distsrc-tgz") ++ ++# Defaults for SCons provided flags. SetOption only sets the option to our value ++# if the user did not provide it. So for any flag here if it's explicitly passed ++# the values below set with SetOption will be overwritten. ++# ++# Default j to the number of CPUs on the system. Note: in containers this ++# reports the number of CPUs for the host system. We're relying on the standard ++# library here and perhaps in a future version of Python it will instead report ++# the correct number when in a container. ++try: ++ env.SetOption('num_jobs', multiprocessing.cpu_count()) ++# On some platforms (like Windows) on Python 2.7 multiprocessing.cpu_count ++# is not implemented. After we upgrade to Python 3.4+ we can use alternative ++# methods that are cross-platform. ++except NotImplementedError: ++ pass ++ ++ ++# Do this as close to last as possible before reading SConscripts, so ++# that any tools that may have injected other things via emitters are included ++# among the side effect adornments. ++# ++# TODO: Move this to a tool. ++if has_option('jlink'): ++ jlink = get_option('jlink') ++ if jlink <= 0: ++ env.FatalError("The argument to jlink must be a positive integer or float") ++ elif jlink < 1 and jlink > 0: ++ jlink = env.GetOption('num_jobs') * jlink ++ jlink = round(jlink) ++ if jlink < 1.0: ++ print("Computed jlink value was less than 1; Defaulting to 1") ++ jlink = 1.0 ++ ++ jlink = int(jlink) ++ target_builders = ['Program', 'SharedLibrary', 'LoadableModule'] ++ ++ # A bound map of stream (as in stream of work) name to side-effect ++ # file. Since SCons will not allow tasks with a shared side-effect ++ # to execute concurrently, this gives us a way to limit link jobs ++ # independently of overall SCons concurrency. ++ jlink_stream_map = dict() ++ ++ def jlink_emitter(target, source, env): ++ name = str(target[0]) ++ se_name = "#jlink-stream" + str(hash(name) % jlink) ++ se_node = jlink_stream_map.get(se_name, None) ++ if not se_node: ++ se_node = env.Entry(se_name) ++ # This may not be necessary, but why chance it ++ env.NoCache(se_node) ++ jlink_stream_map[se_name] = se_node ++ env.SideEffect(se_node, target) ++ return (target, source) ++ ++ for target_builder in target_builders: ++ builder = env['BUILDERS'][target_builder] ++ base_emitter = builder.emitter ++ new_emitter = SCons.Builder.ListEmitter([base_emitter, jlink_emitter]) ++ builder.emitter = new_emitter ++ ++# Keep this late in the game so that we can investigate attributes set by all the tools that have run. ++if has_option("cache"): ++ if get_option("cache") == "nolinked": ++ def noCacheEmitter(target, source, env): ++ for t in target: ++ try: ++ if getattr(t.attributes, 'thin_archive', False): ++ continue ++ except(AttributeError): ++ pass ++ env.NoCache(t) ++ return target, source ++ ++ def addNoCacheEmitter(builder): ++ origEmitter = builder.emitter ++ if SCons.Util.is_Dict(origEmitter): ++ for k,v in origEmitter: ++ origEmitter[k] = SCons.Builder.ListEmitter([v, noCacheEmitter]) ++ elif SCons.Util.is_List(origEmitter): ++ origEmitter.append(noCacheEmitter) ++ else: ++ builder.emitter = SCons.Builder.ListEmitter([origEmitter, noCacheEmitter]) ++ ++ addNoCacheEmitter(env['BUILDERS']['Program']) ++ addNoCacheEmitter(env['BUILDERS']['StaticLibrary']) ++ addNoCacheEmitter(env['BUILDERS']['SharedLibrary']) ++ addNoCacheEmitter(env['BUILDERS']['LoadableModule']) ++ ++env.SConscript( ++ dirs=[ ++ 'src', ++ ], ++ duplicate=False, ++ exports=[ ++ 'env', ++ ], ++ variant_dir='$BUILD_DIR', ++) ++ ++all = env.Alias('all', ['core', 'tools', 'dbtest', 'unittests', 'integration_tests', 'benchmarks']) ++ ++# run the Dagger tool if it's installed ++if should_dagger: ++ dependencyDb = env.Alias("dagger", env.Dagger('library_dependency_graph.json')) ++ # Require everything to be built before trying to extract build dependency information ++ env.Requires(dependencyDb, all) ++ ++# We don't want installing files to cause them to flow into the cache, ++# since presumably we can re-install them from the origin if needed. ++env.NoCache(env.FindInstalledFiles()) ++ ++# Declare the cache prune target ++cachePrune = env.Command( ++ target="#cache-prune", ++ source=[ ++ "#buildscripts/scons_cache_prune.py", ++ ], ++ action="$PYTHON ${SOURCES[0]} --cache-dir=${CACHE_DIR.abspath} --cache-size=${CACHE_SIZE} --prune-ratio=${CACHE_PRUNE_TARGET/100.00}", ++ CACHE_DIR=env.Dir(cacheDir), ++) ++ ++env.AlwaysBuild(cachePrune) ++env.Alias('cache-prune', cachePrune) ++ ++# Substitute environment variables in any build targets so that we can ++# say, for instance: ++# ++# > scons --prefix=/foo/bar '$INSTALL_DIR' ++# or ++# > scons \$BUILD_DIR/mongo/base ++# ++# That way, you can reference targets under the variant dir or install ++# path via an invariant name. ++# ++# We need to replace the values in the BUILD_TARGETS object in-place ++# because SCons wants it to be a particular object. ++for i, s in enumerate(BUILD_TARGETS): ++ BUILD_TARGETS[i] = env.subst(s) +diff -upNr mongo-r4.0.23.orig/site_scons/libdeps.py mongo-r4.0.23/site_scons/libdeps.py +--- mongo-r4.0.23.orig/site_scons/libdeps.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/libdeps.py 2021-03-17 01:21:05.956000000 +0000 +@@ -133,7 +133,7 @@ def __get_libdeps(node): + marked.add(n.target_node) + tsorted.append(n.target_node) + +- except DependencyCycleError, e: ++ except DependencyCycleError as e: + if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]: + e.cycle_nodes.insert(0, n.target_node) + raise +@@ -161,7 +161,7 @@ def __get_syslibdeps(node): + for lib in __get_libdeps(node): + for syslib in node.get_env().Flatten(lib.get_env().get(syslibdeps_env_var, [])): + if syslib: +- if type(syslib) in (str, unicode) and syslib.startswith(missing_syslibdep): ++ if type(syslib) in (str, str) and syslib.startswith(missing_syslibdep): + print("Target '%s' depends on the availability of a " + "system provided library for '%s', " + "but no suitable library was found during configuration." % +@@ -220,7 +220,7 @@ def get_syslibdeps(source, target, env, + # they're believed to represent library short names, that should be prefixed with -l + # or the compiler-specific equivalent. I.e., 'm' becomes '-lm', but 'File("m.a") is passed + # through whole cloth. +- if type(d) in (str, unicode): ++ if type(d) in (str, str): + result.append('%s%s%s' % (lib_link_prefix, d, lib_link_suffix)) + else: + result.append(d) +diff -upNr mongo-r4.0.23.orig/site_scons/libdeps.py.orig mongo-r4.0.23/site_scons/libdeps.py.orig +--- mongo-r4.0.23.orig/site_scons/libdeps.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/site_scons/libdeps.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,397 @@ ++"""Extension to SCons providing advanced static library dependency tracking. ++ ++These modifications to a build environment, which can be attached to ++StaticLibrary and Program builders via a call to setup_environment(env), ++cause the build system to track library dependencies through static libraries, ++and to add them to the link command executed when building programs. ++ ++For example, consider a program 'try' that depends on a lib 'tc', which in ++turn uses a symbol from a lib 'tb' which in turn uses a library from 'ta'. ++Without this package, the Program declaration for "try" looks like this: ++ ++Program('try', ['try.c', 'path/to/${LIBPREFIX}tc${LIBSUFFIX}', ++ 'path/to/${LIBPREFIX}tc${LIBSUFFIX}', ++ 'path/to/${LIBPREFIX}tc${LIBSUFFIX}',]) ++ ++With this library, we can instead write the following ++ ++Program('try', ['try.c'], LIBDEPS=['path/to/tc']) ++StaticLibrary('tc', ['c.c'], LIBDEPS=['path/to/tb']) ++StaticLibrary('tb', ['b.c'], LIBDEPS=['path/to/ta']) ++StaticLibrary('ta', ['a.c']) ++ ++And the build system will figure out that it needs to link libta.a and libtb.a ++when building 'try'. ++ ++A StaticLibrary S may also declare programs or libraries, [L1, ...] to be dependent ++upon S by setting LIBDEPS_DEPENDENTS=[L1, ...], using the same syntax as is used ++for LIBDEPS, except that the libraries and programs will not have LIBPREFIX/LIBSUFFIX ++automatically added when missing. ++""" ++ ++# Copyright (c) 2010, Corensic Inc., All Rights Reserved. ++# ++# Permission is hereby granted, free of charge, to any person obtaining ++# a copy of this software and associated documentation files (the ++# "Software"), to deal in the Software without restriction, including ++# without limitation the rights to use, copy, modify, merge, publish, ++# distribute, sublicense, and/or sell copies of the Software, and to ++# permit persons to whom the Software is furnished to do so, subject to ++# the following conditions: ++# ++# The above copyright notice and this permission notice shall be included ++# in all copies or substantial portions of the Software. ++# ++# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY ++# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE ++# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND ++# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE ++# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION ++# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION ++# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ++ ++import os ++ ++import SCons.Errors ++import SCons.Scanner ++import SCons.Util ++ ++libdeps_env_var = 'LIBDEPS' ++syslibdeps_env_var = 'SYSLIBDEPS' ++missing_syslibdep = 'MISSING_LIBDEP_' ++ ++class dependency(object): ++ Public, Private, Interface = range(3) ++ ++ def __init__(self, value, deptype): ++ self.target_node = value ++ self.dependency_type = deptype ++ ++ def __str__(self): ++ return str(self.target_node) ++ ++dependency_visibility_ignored = { ++ dependency.Public : dependency.Public, ++ dependency.Private : dependency.Public, ++ dependency.Interface : dependency.Public, ++} ++ ++dependency_visibility_honored = { ++ dependency.Public : dependency.Public, ++ dependency.Private : dependency.Private, ++ dependency.Interface : dependency.Interface, ++} ++ ++class DependencyCycleError(SCons.Errors.UserError): ++ """Exception representing a cycle discovered in library dependencies.""" ++ ++ def __init__(self, first_node ): ++ super(DependencyCycleError, self).__init__() ++ self.cycle_nodes = [first_node] ++ ++ def __str__(self): ++ return "Library dependency cycle detected: " + " => ".join(str(n) for n in self.cycle_nodes) ++ ++def __get_sorted_direct_libdeps(node): ++ direct_sorted = getattr(node.attributes, "libdeps_direct_sorted", False) ++ if not direct_sorted: ++ direct = getattr(node.attributes, 'libdeps_direct', []) ++ direct_sorted = sorted(direct, key=lambda t: str(t.target_node)) ++ setattr(node.attributes, "libdeps_direct_sorted", direct_sorted) ++ return direct_sorted ++ ++def __get_libdeps(node): ++ ++ """Given a SCons Node, return its library dependencies, topologically sorted. ++ ++ Computes the dependencies if they're not already cached. ++ """ ++ ++ cached_var_name = libdeps_env_var + '_cached' ++ ++ if hasattr(node.attributes, cached_var_name): ++ return getattr(node.attributes, cached_var_name) ++ ++ tsorted = [] ++ marked = set() ++ ++ def visit(n): ++ if getattr(n.target_node.attributes, 'libdeps_exploring', False): ++ raise DependencyCycleError(n.target_node) ++ ++ n.target_node.attributes.libdeps_exploring = True ++ try: ++ ++ if n.target_node in marked: ++ return ++ ++ try: ++ for child in __get_sorted_direct_libdeps(n.target_node): ++ if child.dependency_type != dependency.Private: ++ visit(child) ++ ++ marked.add(n.target_node) ++ tsorted.append(n.target_node) ++ ++ except DependencyCycleError, e: ++ if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]: ++ e.cycle_nodes.insert(0, n.target_node) ++ raise ++ ++ finally: ++ n.target_node.attributes.libdeps_exploring = False ++ ++ for child in __get_sorted_direct_libdeps(node): ++ if child.dependency_type != dependency.Interface: ++ visit(child) ++ ++ tsorted.reverse() ++ setattr(node.attributes, cached_var_name, tsorted) ++ ++ return tsorted ++ ++def __get_syslibdeps(node): ++ """ Given a SCons Node, return its system library dependencies. ++ ++ These are the depencencies listed with SYSLIBDEPS, and are linked using -l. ++ """ ++ cached_var_name = syslibdeps_env_var + '_cached' ++ if not hasattr(node.attributes, cached_var_name): ++ syslibdeps = node.get_env().Flatten(node.get_env().get(syslibdeps_env_var, [])) ++ for lib in __get_libdeps(node): ++ for syslib in node.get_env().Flatten(lib.get_env().get(syslibdeps_env_var, [])): ++ if syslib: ++ if type(syslib) in (str, unicode) and syslib.startswith(missing_syslibdep): ++ print("Target '%s' depends on the availability of a " ++ "system provided library for '%s', " ++ "but no suitable library was found during configuration." % ++ (str(node), syslib[len(missing_syslibdep):])) ++ node.get_env().Exit(1) ++ syslibdeps.append(syslib) ++ setattr(node.attributes, cached_var_name, syslibdeps) ++ return getattr(node.attributes, cached_var_name) ++ ++def __missing_syslib(name): ++ return missing_syslibdep + name ++ ++def update_scanner(builder): ++ """Update the scanner for "builder" to also scan library dependencies.""" ++ ++ old_scanner = builder.target_scanner ++ ++ if old_scanner: ++ path_function = old_scanner.path_function ++ def new_scanner(node, env, path=()): ++ result = old_scanner.function(node, env, path) ++ result.extend(__get_libdeps(node)) ++ return result ++ else: ++ path_function = None ++ def new_scanner(node, env, path=()): ++ return __get_libdeps(node) ++ ++ builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, ++ path_function=path_function) ++ ++def get_libdeps(source, target, env, for_signature): ++ """Implementation of the special _LIBDEPS environment variable. ++ ++ Expands to the library dependencies for a target. ++ """ ++ ++ target = env.Flatten([target]) ++ return __get_libdeps(target[0]) ++ ++def get_libdeps_objs(source, target, env, for_signature): ++ objs = [] ++ for lib in get_libdeps(source, target, env, for_signature): ++ # This relies on Node.sources being order stable build-to-build. ++ objs.extend(lib.sources) ++ return objs ++ ++def get_syslibdeps(source, target, env, for_signature): ++ deps = __get_syslibdeps(target[0]) ++ lib_link_prefix = env.subst('$LIBLINKPREFIX') ++ lib_link_suffix = env.subst('$LIBLINKSUFFIX') ++ result = [] ++ for d in deps: ++ # Elements of syslibdeps are either strings (str or unicode), or they're File objects. ++ # If they're File objects, they can be passed straight through. If they're strings, ++ # they're believed to represent library short names, that should be prefixed with -l ++ # or the compiler-specific equivalent. I.e., 'm' becomes '-lm', but 'File("m.a") is passed ++ # through whole cloth. ++ if type(d) in (str, unicode): ++ result.append('%s%s%s' % (lib_link_prefix, d, lib_link_suffix)) ++ else: ++ result.append(d) ++ return result ++ ++def __append_direct_libdeps(node, prereq_nodes): ++ # We do not bother to decorate nodes that are not actual Objects ++ if type(node) == str: ++ return ++ if getattr(node.attributes, 'libdeps_direct', None) is None: ++ node.attributes.libdeps_direct = [] ++ node.attributes.libdeps_direct.extend(prereq_nodes) ++ ++def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibility_ignored, ignore_progdeps=False): ++ ++ def libdeps_emitter(target, source, env): ++ """SCons emitter that takes values from the LIBDEPS environment variable and ++ converts them to File node objects, binding correct path information into ++ those File objects. ++ ++ Emitters run on a particular "target" node during the initial execution of ++ the SConscript file, rather than during the later build phase. When they ++ run, the "env" environment's working directory information is what you ++ expect it to be -- that is, the working directory is considered to be the ++ one that contains the SConscript file. This allows specification of ++ relative paths to LIBDEPS elements. ++ ++ This emitter also adds LIBSUFFIX and LIBPREFIX appropriately. ++ ++ NOTE: For purposes of LIBDEPS_DEPENDENTS propagation, only the first member ++ of the "target" list is made a prerequisite of the elements of LIBDEPS_DEPENDENTS. ++ """ ++ ++ lib_builder = env['BUILDERS'][dependency_builder] ++ lib_node_factory = lib_builder.target_factory or env.File ++ ++ prog_builder = env['BUILDERS']['Program'] ++ prog_node_factory = prog_builder.target_factory or env.File ++ ++ prereqs = [dependency(l, dependency_map[dependency.Public]) for l in env.get(libdeps_env_var, []) if l] ++ prereqs.extend(dependency(l, dependency_map[dependency.Interface]) for l in env.get(libdeps_env_var + '_INTERFACE', []) if l) ++ prereqs.extend(dependency(l, dependency_map[dependency.Private]) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l) ++ ++ for prereq in prereqs: ++ prereqWithIxes = SCons.Util.adjustixes( ++ prereq.target_node, lib_builder.get_prefix(env), lib_builder.get_suffix(env)) ++ prereq.target_node = lib_node_factory(prereqWithIxes) ++ ++ for t in target: ++ # target[0] must be a Node and not a string, or else libdeps will fail to ++ # work properly. ++ __append_direct_libdeps(t, prereqs) ++ ++ for dependent in env.get('LIBDEPS_DEPENDENTS', []): ++ if dependent is None: ++ continue ++ ++ visibility = dependency.Private ++ if isinstance(dependent, tuple): ++ visibility = dependent[1] ++ dependent = dependent[0] ++ ++ dependentWithIxes = SCons.Util.adjustixes( ++ dependent, lib_builder.get_prefix(env), lib_builder.get_suffix(env)) ++ dependentNode = lib_node_factory(dependentWithIxes) ++ __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) ++ ++ if not ignore_progdeps: ++ for dependent in env.get('PROGDEPS_DEPENDENTS', []): ++ if dependent is None: ++ continue ++ ++ visibility = dependency.Public ++ if isinstance(dependent, tuple): ++ # TODO: Error here? Non-public PROGDEPS_DEPENDENTS probably are meaningless ++ visibility = dependent[1] ++ dependent = dependent[0] ++ ++ dependentWithIxes = SCons.Util.adjustixes( ++ dependent, prog_builder.get_prefix(env), prog_builder.get_suffix(env)) ++ dependentNode = prog_node_factory(dependentWithIxes) ++ __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])]) ++ ++ return target, source ++ ++ return libdeps_emitter ++ ++def expand_libdeps_tags(source, target, env, for_signature): ++ results = [] ++ for expansion in env.get('LIBDEPS_TAG_EXPANSIONS', []): ++ results.append(expansion(source, target, env, for_signature)) ++ return results ++ ++def setup_environment(env, emitting_shared=False): ++ """Set up the given build environment to do LIBDEPS tracking.""" ++ ++ try: ++ env['_LIBDEPS'] ++ except KeyError: ++ env['_LIBDEPS'] = '$_LIBDEPS_LIBS' ++ ++ env['_LIBDEPS_TAGS'] = expand_libdeps_tags ++ env['_LIBDEPS_GET_LIBS'] = get_libdeps ++ env['_LIBDEPS_OBJS'] = get_libdeps_objs ++ env['_SYSLIBDEPS'] = get_syslibdeps ++ ++ env[libdeps_env_var] = SCons.Util.CLVar() ++ env[syslibdeps_env_var] = SCons.Util.CLVar() ++ ++ # We need a way for environments to alter just which libdeps ++ # emitter they want, without altering the overall program or ++ # library emitter which may have important effects. The ++ # subsitution rules for emitters are a little strange, so build ++ # ourselves a little trampoline to use below so we don't have to ++ # deal with it. ++ def make_indirect_emitter(variable): ++ def indirect_emitter(target, source, env): ++ return env[variable](target, source, env) ++ return indirect_emitter ++ ++ env.Append( ++ LIBDEPS_LIBEMITTER=make_libdeps_emitter('StaticLibrary'), ++ LIBEMITTER=make_indirect_emitter('LIBDEPS_LIBEMITTER'), ++ ++ LIBDEPS_SHAREMITTER=make_libdeps_emitter('SharedArchive', ignore_progdeps=True), ++ SHAREMITTER=make_indirect_emitter('LIBDEPS_SHAREMITTER'), ++ ++ LIBDEPS_SHLIBEMITTER=make_libdeps_emitter('SharedLibrary', dependency_visibility_honored), ++ SHLIBEMITTER=make_indirect_emitter('LIBDEPS_SHLIBEMITTER'), ++ ++ LIBDEPS_PROGEMITTER=make_libdeps_emitter('SharedLibrary' if emitting_shared else 'StaticLibrary'), ++ PROGEMITTER=make_indirect_emitter('LIBDEPS_PROGEMITTER'), ++ ) ++ ++ def expand_libdeps_with_extraction_flags(source, target, env, for_signature): ++ result = [] ++ libs = get_libdeps(source, target, env, for_signature) ++ for lib in libs: ++ if 'init-no-global-side-effects' in env.Entry(lib).get_env().get('LIBDEPS_TAGS', []): ++ result.append(str(lib)) ++ else: ++ result.extend(env.subst('$LINK_WHOLE_ARCHIVE_LIB_START' ++ '$TARGET' ++ '$LINK_WHOLE_ARCHIVE_LIB_END', target=lib).split()) ++ return result ++ ++ env['_LIBDEPS_LIBS_WITH_TAGS'] = expand_libdeps_with_extraction_flags ++ ++ env['_LIBDEPS_LIBS'] = ('$LINK_WHOLE_ARCHIVE_START ' ++ '$LINK_LIBGROUP_START ' ++ '$_LIBDEPS_LIBS_WITH_TAGS ' ++ '$LINK_LIBGROUP_END ' ++ '$LINK_WHOLE_ARCHIVE_END') ++ ++ env.Prepend(_LIBFLAGS='$_LIBDEPS_TAGS $_LIBDEPS $_SYSLIBDEPS ') ++ for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'SharedArchive'): ++ try: ++ update_scanner(env['BUILDERS'][builder_name]) ++ except KeyError: ++ pass ++ ++def setup_conftests(conf): ++ def FindSysLibDep(context, name, libs, **kwargs): ++ var = "LIBDEPS_" + name.upper() + "_SYSLIBDEP" ++ kwargs['autoadd'] = False ++ for lib in libs: ++ result = context.sconf.CheckLib(lib, **kwargs) ++ context.did_show_result = 1 ++ if result: ++ context.env[var] = lib ++ return context.Result(result) ++ context.env[var] = __missing_syslib(name) ++ return context.Result(result) ++ conf.AddTest('FindSysLibDep', FindSysLibDep) +diff -upNr mongo-r4.0.23.orig/site_scons/mongo/generators.py mongo-r4.0.23/site_scons/mongo/generators.py +--- mongo-r4.0.23.orig/site_scons/mongo/generators.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/mongo/generators.py 2021-03-17 01:21:05.956000000 +0000 +@@ -1,6 +1,6 @@ + # -*- mode: python; -*- + +-import md5 ++import hashlib + + # Default and alternative generator definitions go here. + +@@ -44,7 +44,7 @@ def default_variant_dir_generator(target + + # Hash the named options and their values, and take the first 8 characters of the hash as + # the variant name +- hasher = md5.md5() ++ hasher = hashlib.md5() + for option in variant_options: + hasher.update(option) + hasher.update(str(env.GetOption(option))) +diff -upNr mongo-r4.0.23.orig/site_scons/mongo/__init__.py mongo-r4.0.23/site_scons/mongo/__init__.py +--- mongo-r4.0.23.orig/site_scons/mongo/__init__.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/mongo/__init__.py 2021-03-17 01:21:05.956000000 +0000 +@@ -5,4 +5,4 @@ + def print_build_failures(): + from SCons.Script import GetBuildFailures + for bf in GetBuildFailures(): +- print "%s failed: %s" % (bf.node, bf.errstr) ++ print("%s failed: %s" % (bf.node, bf.errstr)) +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/dagger/dagger.py mongo-r4.0.23/site_scons/site_tools/dagger/dagger.py +--- mongo-r4.0.23.orig/site_scons/site_tools/dagger/dagger.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/dagger/dagger.py 2021-03-17 01:21:05.956000000 +0000 +@@ -40,8 +40,8 @@ import sys + + import SCons + +-import graph +-import graph_consts ++from . import graph ++from . import graph_consts + + + LIB_DB = [] # Stores every SCons library nodes +@@ -240,7 +240,7 @@ def write_obj_db(target, source, env): + for obj in OBJ_DB: + __generate_file_rels(obj, g) + +- for exe in EXE_DB.keys(): ++ for exe in list(EXE_DB.keys()): + __generate_exe_rels(exe, g) + + # target is given as a list of target SCons nodes - this builder is only responsible for +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph_consts.py mongo-r4.0.23/site_scons/site_tools/dagger/graph_consts.py +--- mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph_consts.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/dagger/graph_consts.py 2021-03-17 01:21:05.956000000 +0000 +@@ -17,8 +17,8 @@ NODE_SYM = 2 + NODE_FILE = 3 + NODE_EXE = 4 + +-RELATIONSHIP_TYPES = range(1, 9) +-NODE_TYPES = range(1, 5) ++RELATIONSHIP_TYPES = list(range(1, 9)) ++NODE_TYPES = list(range(1, 5)) + + + """Error/query codes""" +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph.py mongo-r4.0.23/site_scons/site_tools/dagger/graph.py +--- mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/dagger/graph.py 2021-03-17 01:21:05.956000000 +0000 +@@ -4,11 +4,13 @@ import abc + import json + import copy + +-import graph_consts ++from . import graph_consts + + if sys.version_info >= (3, 0): + basestring = str + ++ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) ++ + class Graph(object): + """Graph class for storing the build dependency graph. The graph stores the + directed edges as a nested dict of { RelationshipType: {From_Node: Set of +@@ -141,7 +143,7 @@ class Graph(object): + node_dict["id"] = id + node_dict["node"] = {} + +- for property, value in vars(node).iteritems(): ++ for property, value in vars(node).items(): + if isinstance(value, set): + node_dict["node"][property] = list(value) + else: +@@ -170,10 +172,9 @@ class Graph(object): + sum(len(x) for x in self._edges.values()), hash(self)) + + +-class NodeInterface(object): ++class NodeInterface(ABC): + """Abstract base class for all Node Objects - All nodes must have an id and name + """ +- __metaclass__ = abc.ABCMeta + + @abc.abstractproperty + def id(self): +@@ -190,7 +191,7 @@ class NodeLib(NodeInterface): + def __init__(self, id, name, input=None): + if isinstance(input, dict): + should_fail = False +- for k, v in input.iteritems(): ++ for k, v in input.items(): + try: + if isinstance(v, list): + setattr(self, k, set(v)) +@@ -310,7 +311,7 @@ class NodeSymbol(NodeInterface): + if isinstance(input, dict): + should_fail = False + +- for k, v in input.iteritems(): ++ for k, v in input.items(): + try: + if isinstance(v, list): + setattr(self, k, set(v)) +@@ -435,7 +436,7 @@ class NodeFile(NodeInterface): + def __init__(self, id, name, input=None): + if isinstance(input, dict): + should_fail = False +- for k, v in input.iteritems(): ++ for k, v in input.items(): + try: + if isinstance(v, list): + setattr(self, k, set(v)) +@@ -551,7 +552,7 @@ class NodeExe(NodeInterface): + def __init__(self, id, name, input=None): + if isinstance(input, dict): + should_fail = False +- for k, v in input.iteritems(): ++ for k, v in input.items(): + try: + if isinstance(v, list): + setattr(self, k, set(v)) +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph_test.py mongo-r4.0.23/site_scons/site_tools/dagger/graph_test.py +--- mongo-r4.0.23.orig/site_scons/site_tools/dagger/graph_test.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/dagger/graph_test.py 2021-03-17 01:21:05.956000000 +0000 +@@ -5,8 +5,8 @@ from JSON + + import json + import unittest +-import graph +-import graph_consts ++from . import graph ++from . import graph_consts + + + def generate_graph(): +@@ -122,15 +122,15 @@ class TestGraphMethods(unittest.TestCase + node = graph.NodeLib("test_node", "test_node") + self.g._nodes = {"test_node": node} + +- self.assertEquals(self.g.get_node("test_node"), node) ++ self.assertEqual(self.g.get_node("test_node"), node) + +- self.assertEquals(self.g.get_node("missing_node"), None) ++ self.assertEqual(self.g.get_node("missing_node"), None) + + def test_add_node(self): + node = graph.NodeLib("test_node", "test_node") + self.g.add_node(node) + +- self.assertEquals(self.g.get_node("test_node"), node) ++ self.assertEqual(self.g.get_node("test_node"), node) + + self.assertRaises(ValueError, self.g.add_node, node) + +@@ -153,16 +153,16 @@ class TestGraphMethods(unittest.TestCase + self.g.add_edge(graph_consts.LIB_FIL, self.from_node_lib.id, + self.to_node_file.id) + +- self.assertEquals(self.g.edges[graph_consts.LIB_LIB][ ++ self.assertEqual(self.g.edges[graph_consts.LIB_LIB][ + self.from_node_lib.id], set([self.to_node_lib.id])) + +- self.assertEquals(self.g.edges[graph_consts.LIB_SYM][ ++ self.assertEqual(self.g.edges[graph_consts.LIB_SYM][ + self.from_node_lib.id], set([self.to_node_sym.id])) + +- self.assertEquals(self.g.edges[graph_consts.LIB_FIL][ ++ self.assertEqual(self.g.edges[graph_consts.LIB_FIL][ + self.from_node_lib.id], set([self.to_node_file.id])) + +- self.assertEquals(self.to_node_lib.dependent_libs, ++ self.assertEqual(self.to_node_lib.dependent_libs, + set([self.from_node_lib.id])) + + def test_add_edge_files(self): +@@ -173,14 +173,14 @@ class TestGraphMethods(unittest.TestCase + self.g.add_edge(graph_consts.FIL_LIB, self.from_node_file.id, + self.to_node_lib.id) + +- self.assertEquals(self.g.edges[graph_consts.FIL_FIL][ ++ self.assertEqual(self.g.edges[graph_consts.FIL_FIL][ + self.from_node_file.id], set([self.to_node_file.id])) +- self.assertEquals(self.g.edges[graph_consts.FIL_SYM][ ++ self.assertEqual(self.g.edges[graph_consts.FIL_SYM][ + self.from_node_file.id], set([self.to_node_sym.id])) +- self.assertEquals(self.g.edges[graph_consts.FIL_LIB][ ++ self.assertEqual(self.g.edges[graph_consts.FIL_LIB][ + self.from_node_file.id], set([self.to_node_lib.id])) + +- self.assertEquals(self.to_node_file.dependent_files, ++ self.assertEqual(self.to_node_file.dependent_files, + set([self.from_node_file.id])) + + def test_export_to_json(self): +@@ -188,7 +188,7 @@ class TestGraphMethods(unittest.TestCase + generated_graph.export_to_json("export_test.json") + generated = open("export_test.json", "r") + correct = open("test_graph.json", "r") +- self.assertEquals(json.load(generated), json.load(correct)) ++ self.assertEqual(json.load(generated), json.load(correct)) + generated.close() + correct.close() + +@@ -205,7 +205,7 @@ class TestGraphMethods(unittest.TestCase + self.assertNodeEquals( + graph_fromJSON.get_node(id), correct_graph.get_node(id)) + +- self.assertEquals(graph_fromJSON.edges, correct_graph.edges) ++ self.assertEqual(graph_fromJSON.edges, correct_graph.edges) + + + if __name__ == '__main__': +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/dagger/__init__.py mongo-r4.0.23/site_scons/site_tools/dagger/__init__.py +--- mongo-r4.0.23.orig/site_scons/site_tools/dagger/__init__.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/dagger/__init__.py 2021-03-17 01:21:05.956000000 +0000 +@@ -5,7 +5,7 @@ import logging + + import SCons + +-import dagger ++from . import dagger + + def generate(env, **kwargs): + """The entry point for our tool. However, the builder for +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/distsrc.py mongo-r4.0.23/site_scons/site_tools/distsrc.py +--- mongo-r4.0.23.orig/site_scons/site_tools/distsrc.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/distsrc.py 2021-03-17 01:21:05.956000000 +0000 +@@ -20,7 +20,7 @@ import shutil + import tarfile + import time + import zipfile +-import StringIO ++import io + + from distutils.spawn import find_executable + +@@ -82,7 +82,7 @@ class DistSrcTarArchive(DistSrcArchive): + + def append_file_contents(self, filename, file_contents, + mtime=time.time(), +- mode=0644, ++ mode=0o644, + uname="root", + gname="root"): + file_metadata = tarfile.TarInfo(name=filename) +@@ -91,7 +91,7 @@ class DistSrcTarArchive(DistSrcArchive): + file_metadata.uname = uname + file_metadata.gname = gname + file_metadata.size = len(file_contents) +- file_buf = StringIO.StringIO(file_contents) ++ file_buf = io.StringIO(file_contents) + if self.archive_mode == 'r': + self.archive_file.close() + self.archive_file = tarfile.open( +@@ -119,7 +119,7 @@ class DistSrcZipArchive(DistSrcArchive): + name=key, + size=item_data.file_size, + mtime=time.mktime(fixed_time), +- mode=0775 if is_dir else 0664, ++ mode=0o775 if is_dir else 0o664, + type=tarfile.DIRTYPE if is_dir else tarfile.REGTYPE, + uid=0, + gid=0, +@@ -129,7 +129,7 @@ class DistSrcZipArchive(DistSrcArchive): + + def append_file_contents(self, filename, file_contents, + mtime=time.time(), +- mode=0644, ++ mode=0o644, + uname="root", + gname="root"): + self.archive_file.writestr(filename, file_contents) +@@ -139,7 +139,7 @@ class DistSrcZipArchive(DistSrcArchive): + + def build_error_action(msg): + def error_stub(target=None, source=None, env=None): +- print msg ++ print(msg) + env.Exit(1) + return [ error_stub ] + +@@ -162,7 +162,7 @@ def distsrc_action_generator(source, tar + + target_ext = str(target[0])[-3:] + if not target_ext in [ 'zip', 'tar' ]: +- print "Invalid file format for distsrc. Must be tar or zip file" ++ print("Invalid file format for distsrc. Must be tar or zip file") + env.Exit(1) + + git_cmd = "\"%s\" archive --format %s --output %s --prefix ${MONGO_DIST_SRC_PREFIX} HEAD" % ( +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/icecream.py mongo-r4.0.23/site_scons/site_tools/icecream.py +--- mongo-r4.0.23.orig/site_scons/site_tools/icecream.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/icecream.py 2021-03-17 01:21:05.956000000 +0000 +@@ -99,7 +99,7 @@ def generate(env): + suffixes = _CSuffixes + _CXXSuffixes + for object_builder in SCons.Tool.createObjBuilders(env): + emitterdict = object_builder.builder.emitter +- for suffix in emitterdict.iterkeys(): ++ for suffix in emitterdict.keys(): + if not suffix in suffixes: + continue + base = emitterdict[suffix] +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/idl_tool.py mongo-r4.0.23/site_scons/site_tools/idl_tool.py +--- mongo-r4.0.23.orig/site_scons/site_tools/idl_tool.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/idl_tool.py 2021-03-17 01:21:05.956000000 +0000 +@@ -47,7 +47,7 @@ def idl_scanner(node, env, path): + + deps_list = deps_str.splitlines() + +- nodes_deps_list = [ env.File(d) for d in deps_list] ++ nodes_deps_list = [ env.File(d.decode("utf-8")) for d in deps_list] + nodes_deps_list.extend(env.Glob('#buildscripts/idl/*.py')) + nodes_deps_list.extend(env.Glob('#buildscripts/idl/idl/*.py')) + +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/jstoh.py mongo-r4.0.23/site_scons/site_tools/jstoh.py +--- mongo-r4.0.23.orig/site_scons/site_tools/jstoh.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/jstoh.py 2021-03-17 01:21:05.956000000 +0000 +@@ -1,3 +1,5 @@ ++from __future__ import unicode_literals ++ + import os + import sys + +@@ -39,7 +41,7 @@ def jsToHeader(target, source): + + text = '\n'.join(h) + +- with open(outFile, 'wb') as out: ++ with open(outFile, 'w') as out: + try: + out.write(text) + finally: +@@ -48,7 +50,7 @@ def jsToHeader(target, source): + + if __name__ == "__main__": + if len(sys.argv) < 3: +- print "Must specify [target] [source] " ++ print("Must specify [target] [source] ") + sys.exit(1) + + jsToHeader(sys.argv[1], sys.argv[2:]) +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/mongo_benchmark.py mongo-r4.0.23/site_scons/site_tools/mongo_benchmark.py +--- mongo-r4.0.23.orig/site_scons/site_tools/mongo_benchmark.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/mongo_benchmark.py 2021-03-17 01:21:05.956000000 +0000 +@@ -14,7 +14,7 @@ def benchmark_list_builder_action(env, t + ofile = open(str(target[0]), 'wb') + try: + for s in _benchmarks: +- print '\t' + str(s) ++ print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/mongo_integrationtest.py mongo-r4.0.23/site_scons/site_tools/mongo_integrationtest.py +--- mongo-r4.0.23.orig/site_scons/site_tools/mongo_integrationtest.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/mongo_integrationtest.py 2021-03-17 01:21:05.956000000 +0000 +@@ -12,10 +12,10 @@ def register_integration_test(env, test) + env.Alias('$INTEGRATION_TEST_ALIAS', installed_test) + + def integration_test_list_builder_action(env, target, source): +- ofile = open(str(target[0]), 'wb') ++ ofile = open(str(target[0]), 'w') + try: + for s in _integration_tests: +- print '\t' + str(s) ++ print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/mongo_unittest.py mongo-r4.0.23/site_scons/site_tools/mongo_unittest.py +--- mongo-r4.0.23.orig/site_scons/site_tools/mongo_unittest.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/mongo_unittest.py 2021-03-17 01:21:05.956000000 +0000 +@@ -11,10 +11,10 @@ def register_unit_test(env, test): + env.Alias('$UNITTEST_ALIAS', test) + + def unit_test_list_builder_action(env, target, source): +- ofile = open(str(target[0]), 'wb') ++ ofile = open(str(target[0]), 'w') + try: + for s in _unittests: +- print '\t' + str(s) ++ print('\t' + str(s)) + ofile.write('%s\n' % s) + finally: + ofile.close() +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/split_dwarf.py mongo-r4.0.23/site_scons/site_tools/split_dwarf.py +--- mongo-r4.0.23.orig/site_scons/site_tools/split_dwarf.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/split_dwarf.py 2021-03-17 01:21:05.956000000 +0000 +@@ -52,7 +52,7 @@ def generate(env): + + for object_builder in SCons.Tool.createObjBuilders(env): + emitterdict = object_builder.builder.emitter +- for suffix in emitterdict.iterkeys(): ++ for suffix in emitterdict.keys(): + if not suffix in suffixes: + continue + base = emitterdict[suffix] +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/thin_archive.py mongo-r4.0.23/site_scons/site_tools/thin_archive.py +--- mongo-r4.0.23.orig/site_scons/site_tools/thin_archive.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/thin_archive.py 2021-03-17 01:23:49.176000000 +0000 +@@ -41,7 +41,7 @@ def exists(env): + for line in pipe.stdout: + if found: + continue # consume all data +- found = re.search(r'^GNU ar|^LLVM', line) ++ found = re.search(b'^GNU ar|^LLVM', line) + + return bool(found) + +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/thin_archive.py.orig mongo-r4.0.23/site_scons/site_tools/thin_archive.py.orig +--- mongo-r4.0.23.orig/site_scons/site_tools/thin_archive.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/thin_archive.py.orig 2021-03-17 01:21:05.956000000 +0000 +@@ -0,0 +1,91 @@ ++# Copyright 2016 MongoDB Inc. ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++import SCons ++ ++import re ++import subprocess ++ ++def exists(env): ++ if not 'AR' in env: ++ return False ++ ++ ar = env.subst(env['AR']) ++ if not ar: ++ return False ++ ++ # If the user has done anything confusing with ARFLAGS, bail out. We want to find ++ # an item in ARFLAGS of the exact form 'rc'. ++ if not "rc" in env['ARFLAGS']: ++ return False ++ ++ pipe = SCons.Action._subproc(env, SCons.Util.CLVar(ar) + ['--version'], ++ stdin = 'devnull', ++ stderr = 'devnull', ++ stdout = subprocess.PIPE) ++ if pipe.wait() != 0: ++ return False ++ ++ found = False ++ for line in pipe.stdout: ++ if found: ++ continue # consume all data ++ found = re.search(r'^GNU ar|^LLVM', line) ++ ++ return bool(found) ++ ++def _add_emitter(builder): ++ base_emitter = builder.emitter ++ ++ def new_emitter(target, source, env): ++ for t in target: ++ setattr(t.attributes, "thin_archive", True) ++ return (target, source) ++ ++ new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) ++ builder.emitter = new_emitter ++ ++def _add_scanner(builder): ++ old_scanner = builder.target_scanner ++ path_function = old_scanner.path_function ++ ++ def new_scanner(node, env, path): ++ old_results = old_scanner(node, env, path) ++ new_results = [] ++ for base in old_results: ++ new_results.append(base) ++ if getattr(env.Entry(base).attributes, "thin_archive", None): ++ new_results.extend(base.children()) ++ return new_results ++ ++ builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function) ++ ++def generate(env): ++ if not exists(env): ++ return ++ ++ env['ARFLAGS'] = SCons.Util.CLVar([arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']]) ++ ++ def noop_action(env, target, source): ++ pass ++ ++ # Disable running ranlib, since we added 's' above ++ env['RANLIBCOM'] = noop_action ++ env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET' ++ ++ for builder in ['StaticLibrary', 'SharedArchive']: ++ _add_emitter(env['BUILDERS'][builder]) ++ ++ for builder in ['SharedLibrary', 'LoadableModule', 'Program']: ++ _add_scanner(env['BUILDERS'][builder]) +diff -upNr mongo-r4.0.23.orig/site_scons/site_tools/xcode.py mongo-r4.0.23/site_scons/site_tools/xcode.py +--- mongo-r4.0.23.orig/site_scons/site_tools/xcode.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/site_scons/site_tools/xcode.py 2021-03-17 01:21:05.956000000 +0000 +@@ -9,4 +9,4 @@ def generate(env): + + if 'DEVELOPER_DIR' in os.environ: + env['ENV']['DEVELOPER_DIR'] = os.environ['DEVELOPER_DIR'] +- print "NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands" ++ print("NOTE: Xcode detected; propagating DEVELOPER_DIR from shell environment to subcommands") +diff -upNr mongo-r4.0.23.orig/src/mongo/base/generate_error_codes.py mongo-r4.0.23/src/mongo/base/generate_error_codes.py +--- mongo-r4.0.23.orig/src/mongo/base/generate_error_codes.py 2021-02-09 17:06:15.000000000 +0000 ++++ mongo-r4.0.23/src/mongo/base/generate_error_codes.py 2021-03-17 01:21:05.956000000 +0000 +@@ -27,6 +27,8 @@ + # exception statement from all source files in the program, then also delete + # it in the license file. + ++from __future__ import unicode_literals ++ + """Generate error_codes.{h,cpp} from error_codes.err. + + Format of error_codes.err: +@@ -99,7 +101,7 @@ def main(argv): + categories=error_classes, + ) + +- with open(output, 'wb') as outfile: ++ with open(output, 'w') as outfile: + outfile.write(text) + + def die(message=None): +diff -upNr mongo-r4.0.23.orig/src/mongo/base/generate_error_codes.py.orig mongo-r4.0.23/src/mongo/base/generate_error_codes.py.orig +--- mongo-r4.0.23.orig/src/mongo/base/generate_error_codes.py.orig 1970-01-01 00:00:00.000000000 +0000 ++++ mongo-r4.0.23/src/mongo/base/generate_error_codes.py.orig 2021-02-09 17:06:15.000000000 +0000 +@@ -0,0 +1,183 @@ ++#!/usr/bin/env python ++# ++# Copyright (C) 2018-present MongoDB, Inc. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the Server Side Public License, version 1, ++# as published by MongoDB, Inc. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# Server Side Public License for more details. ++# ++# You should have received a copy of the Server Side Public License ++# along with this program. If not, see ++# . ++# ++# As a special exception, the copyright holders give permission to link the ++# code of portions of this program with the OpenSSL library under certain ++# conditions as described in each individual source file and distribute ++# linked combinations including the program with the OpenSSL library. You ++# must comply with the Server Side Public License in all respects for ++# all of the code used other than as permitted herein. If you modify file(s) ++# with this exception, you may extend this exception to your version of the ++# file(s), but you are not obligated to do so. If you do not wish to do so, ++# delete this exception statement from your version. If you delete this ++# exception statement from all source files in the program, then also delete ++# it in the license file. ++ ++"""Generate error_codes.{h,cpp} from error_codes.err. ++ ++Format of error_codes.err: ++ ++error_code("symbol1", code1) ++error_code("symbol2", code2) ++... ++error_class("class1", ["symbol1", "symbol2, ..."]) ++ ++Usage: ++ python generate_error_codes.py