ansible: improve Python 3 compatibility (#1929)

* ansible: improve Python 3 compatibility

* fixup: review comments

Author: cclauss

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+exclude=.venv
+max-line-length=250
+import-order-style=google
+ignore=E111

.gitignore

@@ -12,3 +12,5 @@ setup/*/host_vars/release-*
 ansible/host_vars/*
 !ansible/host_vars/README.md
 !ansible/host_vars/*-template
+.venv
+Pipfile.lock

ansible/plugins/inventory/nodejs_yaml.py

@@ -23,35 +23,32 @@
 #
 
 from __future__ import print_function
+
 import argparse
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
-try:
-    from future_builtins import filter # Python 2
-except ImportError:
-    pass # Python 3
 import json
-import yaml
 import os
-import sys
 import subprocess
+import sys
 
+import yaml
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
 
 valid = {
-  # taken from nodejs/node.git: ./configure
-  'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc',
-           'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'),
-
-  # valid roles - add as necessary
-  'type': ('infra', 'release', 'test'),
-
-  # providers - validated for consistency
-  'provider': ('azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc',
-               'macstadium', 'marist', 'mininodes', 'msft', 'osuosl',
-               'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer',
-               'packetnet', 'nearform')
+    # taken from nodejs/node.git: ./configure
+    'arch': ('armv6l', 'armv7l', 'arm64', 'ia32', 'mips', 'mipsel', 'ppc',
+             'ppc64', 'x32', 'x64', 'x86', 's390', 's390x'),
+
+    # valid roles - add as necessary
+    'type': ('infra', 'release', 'test'),
+
+    # providers - validated for consistency
+    'provider': ('azure', 'digitalocean', 'joyent', 'ibm', 'linuxonecc',
+                 'macstadium', 'marist', 'mininodes', 'msft', 'osuosl',
+                 'rackspace', 'requireio', 'scaleway', 'softlayer', 'voxer',
+                 'packetnet', 'nearform')
 }
 DECRYPT_TOOL = "gpg"
 INVENTORY_FILENAME = "inventory.yml"
@@ -97,15 +94,15 @@ def main():
 # https://stackoverflow.com/a/7205107
 def merge(a, b, path=None):
     "merges b into a"
-    if path is None: path = []
+    path = path or []
     for key in b:
         if key in a:
             if isinstance(a[key], dict) and isinstance(b[key], dict):
                 merge(a[key], b[key], path + [str(key)])
             elif isinstance(a[key], list) and isinstance(b[key], list):
                 a[key] = sorted(set(a[key]).union(b[key]))
             elif a[key] == b[key]:
-                pass # same leaf value
+                pass  # same leaf value
             else:
                 raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
         else:
@@ -167,7 +164,7 @@ def load_yaml_file(file_name):
     # get inventory
     with open(file_name, 'r') as stream:
         try:
-            hosts = yaml.load(stream)
+            hosts = yaml.safe_load(stream)
 
         except yaml.YAMLError as exc:
             print(exc)
@@ -186,11 +183,11 @@ def load_yaml_secrets(file_name):
         print("WARNING: cannot load %s" % file_name, file=sys.stderr)
         return None
 
-    return yaml.load(stdout)
+    return yaml.safe_load(stdout)
 
 
 def parse_yaml(hosts, config):
-    """Parses host information from the output of yaml.load"""
+    """Parses host information from the output of yaml.safe_load"""
 
     export = {'_meta': {'hostvars': {}}}
 
@@ -279,9 +276,11 @@ def parse_host(host):
 
 
 def has_metadata(info):
-    """Checks for metadata in variables. These are separated from the "key"
-       metadata by underscore. Not used anywhere at the moment for anything
-       other than descriptiveness"""
+    """
+    Checks for metadata in variables. These are separated from the "key"
+    metadata by underscore. Not used anywhere at the moment for anything
+    other than descriptiveness
+    """
 
     metadata = info.split('_', 1)
 

ansible/plugins/library/remmina_config.py

@@ -23,15 +23,17 @@
 #
 
 from __future__ import print_function
-from ansible.module_utils.basic import *
-from jinja2 import Environment
+
+import base64
 import os
+
+from ansible.module_utils.basic import AnsibleModule
+from Crypto.Cipher import DES3
+from jinja2 import Environment
 try:
-    import configparser # Python 3
+    import configparser  # Python 3
 except ImportError:
-    import ConfigParser as configparser # Python 2
-import base64
-from Crypto.Cipher import DES3
+    import ConfigParser as configparser  # Python 2
 
 
 host_template = \

ansible/plugins/library/ssh_config.py

@@ -22,11 +22,13 @@
 # IN THE SOFTWARE.
 #
 
-from ansible.module_utils.basic import *
-from jinja2 import Environment
 import os
 import re
 
+from ansible.module_utils.basic import AnsibleModule
+from jinja2 import Environment
+
+
 pre_match = '# begin: node.js template'
 post_match = '# end: node.js template'
 match = re.compile(r'^' + re.escape(pre_match) + '(.*)' + re.escape(post_match),
@@ -99,8 +101,7 @@ def main():
                          path)
 
     if not is_templatable(path, contents):
-        module.fail_json(msg='Your ssh config lacks template stubs. ' +
-                             'Check README.md for instructions.')
+        module.fail_json(msg='Your ssh config lacks template stubs. Check README.md for instructions.')
 
     rendered = '{}{}{}'.format(
         pre_match,

jenkins/scripts/coverage/generate-index-html.py

@@ -7,8 +7,7 @@
   index_csv = filter(lambda line: line, index.read().split('\n'))
 
 with open('out/index.html', 'w') as out:
-  out.write(
-'''
+  out.write('''
 <!DOCTYPE html>
 <html>
   <head>
@@ -28,6 +27,7 @@
     <link rel="stylesheet" href="https://nodejs.org/layouts/css/styles.css" media="all">
     <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Sans+Pro:400,600">
     <style>
+        #logo { margin-bottom: 1rem; }
         main { margin-bottom: 2rem; }
         .table-header,
         .table-row {
@@ -64,6 +64,11 @@
     </style>
   </head>
   <body>
+  <header>
+    <div class="container" id="logo">
+      <img src="https://nodejs.org/static/images/logos/nodejs-new-white-pantone.png" alt="node.js">
+    </div>
+  </header>
   <div id="main">
     <div class="container">
       <h1>Node.js Nightly Code Coverage</h1>
@@ -90,6 +95,7 @@
               <div><div class="cell-header">JS Coverage</div><div class="cell-value"><a href="coverage-{1}/index.html">{2:05.2f}&nbsp;%</a></div></div>
               <div><div class="cell-header">C++ Coverage</div><div class="cell-value"><a href="coverage-{1}/cxxcoverage.html">{3:05.2f}&nbsp;%</a></div></div>
             </div>'''.format(date, sha, float(jscov), float(cxxcov)))
+
   out.write('''
           </div>
         </div>

setup/www/tools/metrics/country-lookup.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
-import sys
 import csv
-import geoip2.database
 import os
+import sys
+
+import geoip2.database
 
 reader = geoip2.database.Reader(os.path.dirname(os.path.realpath(__file__)) + '/GeoLite2-City.mmdb')
 
@@ -28,11 +29,10 @@
         country = georec.country.iso_code
       if georec.subdivisions.most_specific.iso_code:
         region = georec.subdivisions.most_specific.iso_code
-  except:
+  except Exception:
     pass
 
   row.insert(1, country.encode('utf-8'))
   row.insert(2, region.encode('utf-8'))
 
   logFileWriter.writerow(row)
-