diff options
author | Neil Williams <neil.williams@linaro.org> | 2018-10-01 08:33:51 +0100 |
---|---|---|
committer | Neil Williams <neil.williams@linaro.org> | 2018-10-01 16:14:00 +0100 |
commit | d353200b923472e09039ef7d7d59cbbad1967d33 (patch) | |
tree | b33dad5249fa109f07b21a88758acaeba2983b59 | |
parent | b69f7c11e41e0094d49b2fe2f60a8bb8fb6ee019 (diff) |
Add overrides for bandit false positives
Reduce the noise in the SAST report further.
Override bandit warnings in lava-dispatcher as bandit is
overly based on a web UI mindset.
Add overrides and fixes for files in ./share/
Signed-off-by: Neil Williams <neil.williams@linaro.org>
39 files changed, 124 insertions, 138 deletions
diff --git a/dashboard_app/xmlrpc.py b/dashboard_app/xmlrpc.py index 45a8767fd..a1c61eaff 100644 --- a/dashboard_app/xmlrpc.py +++ b/dashboard_app/xmlrpc.py @@ -20,26 +20,13 @@ XMP-RPC API """ -import datetime import logging -import re -import hashlib -import os -import subprocess -import xmlrpc.client -from django.contrib.auth.models import User, Group -from django.core.urlresolvers import reverse -from django.db import IntegrityError from linaro_django_xmlrpc.models import ( ExposedAPI, Mapper, xml_rpc_signature, ) -from lava_scheduler_app.models import ( - TestJob, -) - class errors: """ diff --git a/doc/v2/conf.py b/doc/v2/conf.py index a97805aca..61d9e23b8 100644 --- a/doc/v2/conf.py +++ b/doc/v2/conf.py @@ -13,7 +13,7 @@ import sys import os -import subprocess +import subprocess # nosec - internal import sphinx_bootstrap_theme # If extensions (or modules to document with autodoc) are in another directory, @@ -57,7 +57,7 @@ copyright = u'2010-2018, Linaro Limited' # built documents. # # The short X.Y version. -version = subprocess.Popen(r'./version.py', cwd=r'../..', stdout=subprocess.PIPE).stdout.read().rstrip().decode('utf-8') +version = subprocess.Popen(r'./version.py', cwd=r'../..', stdout=subprocess.PIPE).stdout.read().rstrip().decode('utf-8') # nosec - internal # The full version, including alpha/beta/rc tags. release = version diff --git a/lava_common/utils.py b/lava_common/utils.py index 9ef9b357b..f80911063 100644 --- a/lava_common/utils.py +++ b/lava_common/utils.py @@ -19,7 +19,7 @@ # with this program; if not, see <http://www.gnu.org/licenses>. import os -import subprocess +import subprocess # nosec dpkg def debian_package_arch(pkg): @@ -30,7 +30,7 @@ def debian_package_arch(pkg): """ changelog = '/usr/share/doc/%s/changelog.Debian.gz' % pkg if os.path.exists(changelog): - deb_arch = subprocess.check_output(( + deb_arch = subprocess.check_output(( # nosec dpkg-query 'dpkg-query', '-W', "-f=${Architecture}\n", "%s" % pkg)).strip().decode('utf-8', errors="replace") return deb_arch @@ -45,7 +45,7 @@ def debian_package_version(pkg, split): """ changelog = '/usr/share/doc/%s/changelog.Debian.gz' % pkg if os.path.exists(changelog): - deb_version = subprocess.check_output(( + deb_version = subprocess.check_output(( # nosec dpkg-query 'dpkg-query', '-W', "-f=${Version}\n", "%s" % pkg)).strip().decode('utf-8', errors="replace") # example version returned would be '2016.11' diff --git a/lava_dispatcher/action.py b/lava_dispatcher/action.py index c9dd2de5f..73524352b 100644 --- a/lava_dispatcher/action.py +++ b/lava_dispatcher/action.py @@ -24,7 +24,7 @@ from functools import reduce import time import types import traceback -import subprocess +import subprocess # nosec - internal from collections import OrderedDict from nose.tools import nottest from lava_common.timeout import Timeout @@ -478,7 +478,7 @@ class Action: # pylint: disable=too-many-instance-attributes,too-many-public-me command_list = ['nice'] + [str(s) for s in command_list] self.logger.debug("%s", ' '.join(command_list)) try: - log = subprocess.check_output(command_list, stderr=subprocess.STDOUT, + log = subprocess.check_output(command_list, stderr=subprocess.STDOUT, # nosec - internal cwd=cwd) log = log.decode('utf-8', errors="replace") # pylint: disable=redefined-variable-type except subprocess.CalledProcessError as exc: diff --git a/lava_dispatcher/actions/deploy/apply_overlay.py b/lava_dispatcher/actions/deploy/apply_overlay.py index 66ce304eb..e74c09081 100644 --- a/lava_dispatcher/actions/deploy/apply_overlay.py +++ b/lava_dispatcher/actions/deploy/apply_overlay.py @@ -20,7 +20,7 @@ import os import shutil -import subprocess +import subprocess # nosec - internal use. from lava_dispatcher.action import ( Action, Pipeline, @@ -247,7 +247,7 @@ class ApplyOverlayTftp(Action): # not be removed if umount fails. directory = mkdtemp(autoremove=False) try: - subprocess.check_output(['mount', '-t', 'nfs', nfs_address, directory]) + subprocess.check_output(['mount', '-t', 'nfs', nfs_address, directory]) # nosec - internal. except subprocess.CalledProcessError as exc: raise JobError(exc) elif self.parameters.get('ramdisk') is not None: @@ -279,7 +279,7 @@ class ApplyOverlayTftp(Action): self.logger.debug("[%s] Applying overlay %s to directory %s", namespace, overlay_file, directory) untar_file(overlay_file, directory) if nfs_address: - subprocess.check_output(['umount', directory]) + subprocess.check_output(['umount', directory]) # nosec - internal. os.rmdir(directory) # fails if the umount fails return connection @@ -549,8 +549,7 @@ class CompressRamdisk(Action): ramdisk_data, ramdisk_dir) cmd = "find . | cpio --create --format='newc' > %s" % ramdisk_data try: - # safe to use shell=True here, no external arguments - log = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + log = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) # nosec - safe to use shell=True here, no external arguments log = log.decode("utf-8", errors="replace") except OSError as exc: raise InfrastructureError('Unable to create cpio filesystem: %s' % exc) diff --git a/lava_dispatcher/actions/deploy/docker.py b/lava_dispatcher/actions/deploy/docker.py index 004373030..6da65c361 100644 --- a/lava_dispatcher/actions/deploy/docker.py +++ b/lava_dispatcher/actions/deploy/docker.py @@ -19,7 +19,7 @@ # with this program; if not, see <http://www.gnu.org/licenses>. import re -import subprocess +import subprocess # nosec - internal from lava_common.exceptions import InfrastructureError, JobError from lava_dispatcher.action import Pipeline @@ -42,10 +42,10 @@ class DockerAction(DeployAction): # Print docker version try: - out = subprocess.check_output(["docker", "version", "-f", "{{.Server.Version}}"]) + out = subprocess.check_output(["docker", "version", "-f", "{{.Server.Version}}"]) # nosec - internal out = out.decode("utf-8", errors="replace").strip("\n") self.logger.debug("docker server, installed at version: %s", out) - out = subprocess.check_output(["docker", "version", "-f", "{{.Client.Version}}"]) + out = subprocess.check_output(["docker", "version", "-f", "{{.Client.Version}}"]) # nosec - internal out = out.decode("utf-8", errors="replace").strip("\n") self.logger.debug("docker client, installed at version: %s", out) except subprocess.CalledProcessError as exc: diff --git a/lava_dispatcher/actions/deploy/download.py b/lava_dispatcher/actions/deploy/download.py index 53682dd0a..0f087d989 100644 --- a/lava_dispatcher/actions/deploy/download.py +++ b/lava_dispatcher/actions/deploy/download.py @@ -30,7 +30,7 @@ import shutil import time import hashlib import requests -import subprocess +import subprocess # nosec - verified. from lava_dispatcher.power import ResetDevice from lava_dispatcher.protocols.lxc import LxcProtocol from lava_dispatcher.actions.deploy import DeployAction @@ -212,7 +212,7 @@ class DownloadHandler(Action): # pylint: disable=too-many-instance-attributes connection = super().run(connection, max_end_time) # self.cookies = self.job.context.config.lava_cookies # FIXME: work out how to restore - md5 = hashlib.md5() + md5 = hashlib.md5() # nosec - not being used for cryptography. sha256 = hashlib.sha256() # Create a fresh directory if the old one has been removed by a previous cleanup @@ -286,7 +286,7 @@ class DownloadHandler(Action): # pylint: disable=too-many-instance-attributes if compression and decompress_command: try: with open(fname, 'wb') as dwnld_file: - proc = subprocess.Popen([decompress_command], + proc = subprocess.Popen([decompress_command], # nosec - internal. stdin=subprocess.PIPE, stdout=dwnld_file) except OSError as exc: @@ -502,7 +502,7 @@ class ScpDownloadAction(DownloadHandler): def validate(self): super().validate() try: - size = subprocess.check_output(['nice', 'ssh', + size = subprocess.check_output(['nice', 'ssh', # nosec - internal. self.url.netloc, 'stat', '-c', '%s', self.url.path], @@ -514,7 +514,7 @@ class ScpDownloadAction(DownloadHandler): def reader(self): process = None try: - process = subprocess.Popen( + process = subprocess.Popen( # nosec - internal. ['nice', 'ssh', self.url.netloc, 'cat', self.url.path], stdout=subprocess.PIPE ) @@ -605,7 +605,7 @@ class QCowConversionAction(Action): self.logger.debug("Converting downloaded image from qcow2 to raw") try: - subprocess.check_output(['qemu-img', 'convert', + subprocess.check_output(['qemu-img', 'convert', # nosec - checked. '-f', 'qcow2', '-O', 'raw', origin, fname], stderr=subprocess.STDOUT) diff --git a/lava_dispatcher/actions/deploy/overlay.py b/lava_dispatcher/actions/deploy/overlay.py index ac6fe5452..26589b8fa 100644 --- a/lava_dispatcher/actions/deploy/overlay.py +++ b/lava_dispatcher/actions/deploy/overlay.py @@ -223,7 +223,7 @@ class MultinodeOverlayAction(OverlayAction): # Multinode-only self.lava_multi_node_test_dir = os.path.realpath( '%s/../../lava_test_shell/multi_node' % os.path.dirname(__file__)) - self.lava_multi_node_cache_file = '/tmp/lava_multi_node_cache.txt' + self.lava_multi_node_cache_file = '/tmp/lava_multi_node_cache.txt' # nosec - on the DUT self.role = None self.protocol = MultinodeProtocol.name @@ -316,7 +316,7 @@ class VlandOverlayAction(OverlayAction): # vland-only self.lava_vland_test_dir = os.path.realpath( '%s/../../lava_test_shell/vland' % os.path.dirname(__file__)) - self.lava_vland_cache_file = '/tmp/lava_vland_cache.txt' + self.lava_vland_cache_file = '/tmp/lava_vland_cache.txt' # nosec - on the DUT self.params = {} self.sysfs = [] self.tags = [] diff --git a/lava_dispatcher/actions/deploy/testdef.py b/lava_dispatcher/actions/deploy/testdef.py index b77786e7b..400536eca 100644 --- a/lava_dispatcher/actions/deploy/testdef.py +++ b/lava_dispatcher/actions/deploy/testdef.py @@ -452,7 +452,7 @@ class InlineRepoAction(RepoAction): # pylint: disable=too-many-public-methods # Grab the inline test definition testdef = self.parameters['repository'] - sha1 = hashlib.sha1() + sha1 = hashlib.sha1() # nosec - not used for cryptography # Dump the test definition and compute the sha1 yaml_file = os.path.join(runner_path, self.parameters['path']) diff --git a/lava_dispatcher/deployment_data.py b/lava_dispatcher/deployment_data.py index 0223f41d3..a7b899063 100644 --- a/lava_dispatcher/deployment_data.py +++ b/lava_dispatcher/deployment_data.py @@ -164,9 +164,9 @@ lede = { # pylint: disable=invalid-name # for lava-test-shell 'distro': 'lede', 'lava_test_sh_cmd': '/bin/sh', - 'lava_test_dir': '/tmp/lava-%s', + 'lava_test_dir': '/tmp/lava-%s', # nosec - on the DUT 'lava_test_results_part_attr': 'root_part', - 'lava_test_results_dir': '/tmp/lava-results-%s', + 'lava_test_results_dir': '/tmp/lava-results-%s', # nosec - on the DUT 'lava_test_shell_file': None, } diff --git a/lava_dispatcher/job.py b/lava_dispatcher/job.py index 00b01ead5..18d246cb8 100644 --- a/lava_dispatcher/job.py +++ b/lava_dispatcher/job.py @@ -155,7 +155,7 @@ class Job: # pylint: disable=too-many-instance-attributes # Create the sub-directory tmp_dir = tempfile.mkdtemp(prefix=action_name + '-', dir=base_dir) - os.chmod(tmp_dir, 0o755) + os.chmod(tmp_dir, 0o755) # nosec - automatic cleanup. return tmp_dir def _validate(self): diff --git a/lava_dispatcher/protocols/lxc.py b/lava_dispatcher/protocols/lxc.py index d953f666c..536e210a1 100644 --- a/lava_dispatcher/protocols/lxc.py +++ b/lava_dispatcher/protocols/lxc.py @@ -24,7 +24,7 @@ import os import yaml import logging import traceback -import subprocess +import subprocess # nosec - internal from lava_dispatcher.connection import Protocol from lava_common.exceptions import ( InfrastructureError, @@ -142,7 +142,7 @@ class LxcProtocol(Protocol): # pylint: disable=too-many-instance-attributes def _call_handler(self, command): try: self.logger.debug("%s protocol: executing '%s'", self.name, command) - output = subprocess.check_output(command.split(' '), + output = subprocess.check_output(command.split(' '), # nosec - internal stderr=subprocess.STDOUT) if output: self.logger.debug(output) diff --git a/lava_dispatcher/test/conftest.py b/lava_dispatcher/test/conftest.py index f76552783..ec7600f35 100644 --- a/lava_dispatcher/test/conftest.py +++ b/lava_dispatcher/test/conftest.py @@ -26,8 +26,8 @@ import requests @pytest.fixture(autouse=True) def no_requests(monkeypatch, request): def get(url, allow_redirects, stream): - assert allow_redirects is True - assert stream is True + assert allow_redirects is True # nosec - unit test support + assert stream is True # nosec - unit test support res = requests.Response() res.status_code = requests.codes.OK res.close = lambda: None diff --git a/lava_dispatcher/test/fake_coordinator.py b/lava_dispatcher/test/fake_coordinator.py index 818205387..a655ebd00 100644 --- a/lava_dispatcher/test/fake_coordinator.py +++ b/lava_dispatcher/test/fake_coordinator.py @@ -65,27 +65,27 @@ class TestSocket: def send(self, data): if self.header: self.header = False - assert(int(data, 16) < 0xFFFE) + assert(int(data, 16) < 0xFFFE) # nosec - unit test support self.log.info("\tCoordinator header: %d bytes" % int(data, 16)) else: try: json_data = json.loads(data) except ValueError: - assert False + assert False # nosec - unit test support if not self.response: - assert(json_data['response'] == "nack") + assert(json_data['response'] == "nack") # nosec - unit test support self.header = True return - assert 'response' in json_data + assert 'response' in json_data # nosec - unit test support self.log.info("\tCoordinator response: '%s'" % json_data['response']) self.log.info("\tdebug: %s" % self.response['response']) - assert(json_data['response'] == self.response['response']) + assert(json_data['response'] == self.response['response']) # nosec - unit test support self.passes += 1 if self.message: # we are expecting a message back. - assert 'message' in json_data + assert 'message' in json_data # nosec - unit test support self.log.info("\tCoordinator received a message: '%s'" % (json.dumps(json_data['message']))) - assert(json_data['message'] == self.message) + assert(json_data['message'] == self.message) # nosec - unit test support self.passes += 1 else: # actual calls will discriminate between dict and string replies diff --git a/lava_dispatcher/test/test_compression.py b/lava_dispatcher/test/test_compression.py index cd5d4b7b8..49310150e 100644 --- a/lava_dispatcher/test/test_compression.py +++ b/lava_dispatcher/test/test_compression.py @@ -57,7 +57,7 @@ class TestDecompression(StdoutTestCase): output = httpaction.get_namespace_data(action='download-action', label=httpaction.key, key='file') outputfile = output.split('/')[-1] sha256hash = hashlib.sha256() - md5sumhash = hashlib.md5() + md5sumhash = hashlib.md5() # nosec - not used for cryptography with open(output, 'rb', buffering=0) as f: for b in iter(lambda: f.read(128 * 1024), b''): sha256hash.update(b) @@ -94,5 +94,5 @@ class TestDecompression(StdoutTestCase): # Call decompress_file, we only need it to create the command required, # it doesn't need to complete successfully. with self.assertRaises(InfrastructureError): - decompress_file("/tmp/test.xz", "zip") + decompress_file("/tmp/test.xz", "zip") # nosec - unit test only. self.assertEqual(copy_of_command_map, decompress_command_map) diff --git a/lava_dispatcher/test/test_defs.py b/lava_dispatcher/test/test_defs.py index a5fd7a1e6..b16ed2c8a 100644 --- a/lava_dispatcher/test/test_defs.py +++ b/lava_dispatcher/test/test_defs.py @@ -27,7 +27,7 @@ import shutil import pexpect import tempfile import unittest -import subprocess +import subprocess # nosec - unit test support. from nose.tools import nottest from lava_dispatcher.power import FinalizeAction from lava_dispatcher.parser import JobParser @@ -387,7 +387,7 @@ def check_rpcinfo(server='127.0.0.1'): returns True on failure. """ try: - subprocess.check_output(['/usr/sbin/rpcinfo', '-u', server, 'nfs', '3']) + subprocess.check_output(['/usr/sbin/rpcinfo', '-u', server, 'nfs', '3']) # nosec - unit test support. except (OSError, subprocess.CalledProcessError): return True return False diff --git a/lava_dispatcher/test/test_iso.py b/lava_dispatcher/test/test_iso.py index 90a54b868..bbfe3cfea 100644 --- a/lava_dispatcher/test/test_iso.py +++ b/lava_dispatcher/test/test_iso.py @@ -93,11 +93,11 @@ class TestIsoJob(StdoutTestCase): '/usr/bin/qemu-system-x86_64', '-nographic', '-enable-kvm', '-cpu host', '-net nic,model=virtio,macaddr=52:54:00:12:34:58 -net user', '-m 2048', ' -drive format=raw,file={emptyimage} ', '-boot c'] - substitutions = {'{emptyimage}': '/tmp/tmp.00000/hd.img'} + substitutions = {'{emptyimage}': '/tmp/tmp.00000/hd.img'} # nosec unit test support. sub_command = substitute(sub_command, substitutions) self.assertNotIn('{emptyimage}', sub_command) - self.assertNotIn('/tmp/tmp.00000/hd.img', sub_command) - self.assertIn('/tmp/tmp.00000/hd.img', ' '.join(sub_command)) + self.assertNotIn('/tmp/tmp.00000/hd.img', sub_command) # nosec unit test support. + self.assertIn('/tmp/tmp.00000/hd.img', ' '.join(sub_command)) # nosec unit test support. def test_timeout_inheritance(self): """ diff --git a/lava_dispatcher/test/test_multinode.py b/lava_dispatcher/test/test_multinode.py index 14249d0c9..8aa87df5a 100644 --- a/lava_dispatcher/test/test_multinode.py +++ b/lava_dispatcher/test/test_multinode.py @@ -170,7 +170,7 @@ class TestMultinode(StdoutTestCase): # pylint: disable=too-many-public-methods self.assertEqual(client_protocol.parameters['protocols'][client_protocol.name]['roles']['kvm02'], 'server') self.assertEqual(server_protocol.parameters['protocols'][client_protocol.name]['roles']['kvm01'], 'client') self.assertEqual(server_protocol.parameters['protocols'][client_protocol.name]['roles']['kvm02'], 'server') - self.assertEqual(client_multinode.lava_multi_node_cache_file, '/tmp/lava_multi_node_cache.txt') + self.assertEqual(client_multinode.lava_multi_node_cache_file, '/tmp/lava_multi_node_cache.txt') # nosec - replicating DUT behaviour. self.assertIsNotNone(client_multinode.lava_multi_node_test_dir) self.assertTrue(os.path.exists(client_multinode.lava_multi_node_test_dir)) @@ -589,7 +589,7 @@ class TestProtocol(StdoutTestCase): # pylint: disable=too-many-public-methods def send(self, msg): if self.header: self.header = False - assert(int(msg, 16) < 0xFFFE) + assert(int(msg, 16) < 0xFFFE) # nosec - unit test support else: message = json.loads(msg) self.coord.dataReceived(message) diff --git a/lava_dispatcher/test/test_utils.py b/lava_dispatcher/test/test_utils.py index 0e016d0da..a99bc4f83 100644 --- a/lava_dispatcher/test/test_utils.py +++ b/lava_dispatcher/test/test_utils.py @@ -20,7 +20,7 @@ import os import shutil -import subprocess +import subprocess # nosec - unit test support. import tempfile import unittest @@ -45,12 +45,12 @@ class TestGit(StdoutTestCase): # pylint: disable=too-many-public-methods os.chdir(self.tmpdir) # Create a Git repository with two commits - subprocess.check_output(['git', 'init', 'git']) + subprocess.check_output(['git', 'init', 'git']) # nosec - unit test support. os.chdir('git') with open('test.txt', 'w') as testfile: testfile.write("Some data") - subprocess.check_output(['git', 'add', 'test.txt']) - subprocess.check_output(['git', 'commit', 'test.txt', '-m', 'First commit'], + subprocess.check_output(['git', 'add', 'test.txt']) # nosec - unit test support. + subprocess.check_output(['git', 'commit', 'test.txt', '-m', 'First commit'], # nosec - unit test support. env={'GIT_COMMITTER_DATE': 'Fri Oct 24 14:40:36 CEST 2014', 'GIT_AUTHOR_DATE': 'Fri Oct 24 14:40:36 CEST 2014', 'GIT_AUTHOR_NAME': 'Foo Bar', @@ -59,8 +59,8 @@ class TestGit(StdoutTestCase): # pylint: disable=too-many-public-methods 'GIT_COMMITTER_EMAIL': 'foo@example.com'}) with open('second.txt', 'w') as datafile: datafile.write("Some more data") - subprocess.check_output(['git', 'add', 'second.txt']) - subprocess.check_output(['git', 'commit', 'second.txt', '-m', 'Second commit'], + subprocess.check_output(['git', 'add', 'second.txt']) # nosec - unit test support. + subprocess.check_output(['git', 'commit', 'second.txt', '-m', 'Second commit'], # nosec - unit test support. env={'GIT_COMMITTER_DATE': 'Fri Oct 24 14:40:38 CEST 2014', 'GIT_AUTHOR_DATE': 'Fri Oct 24 14:40:38 CEST 2014', 'GIT_AUTHOR_NAME': 'Foo Bar', @@ -68,11 +68,11 @@ class TestGit(StdoutTestCase): # pylint: disable=too-many-public-methods 'GIT_COMMITTER_NAME': 'Foo Bar', 'GIT_COMMITTER_EMAIL': 'foo@example.com'}) - subprocess.check_output(['git', 'checkout', '-q', '-b', 'testing']) + subprocess.check_output(['git', 'checkout', '-q', '-b', 'testing']) # nosec - unit test support. with open('third.txt', 'w') as datafile: datafile.write("333") - subprocess.check_output(['git', 'add', 'third.txt']) - subprocess.check_output(['git', 'commit', 'third.txt', '-m', 'Third commit'], + subprocess.check_output(['git', 'add', 'third.txt']) # nosec - unit test support. + subprocess.check_output(['git', 'commit', 'third.txt', '-m', 'Third commit'], # nosec - unit test support. env={'GIT_COMMITTER_DATE': 'Thu Sep 1 10:14:29 CEST 2016', 'GIT_AUTHOR_DATE': 'Thu Sep 1 10:14:29 CEST 2016', 'GIT_AUTHOR_NAME': 'Foo Bar', @@ -80,7 +80,7 @@ class TestGit(StdoutTestCase): # pylint: disable=too-many-public-methods 'GIT_COMMITTER_NAME': 'Foo Bar', 'GIT_COMMITTER_EMAIL': 'foo@example.com'}) - subprocess.check_output(['git', 'checkout', '-q', 'master']) + subprocess.check_output(['git', 'checkout', '-q', 'master']) # nosec - unit test support. # Go into the tempdir os.chdir('..') @@ -145,22 +145,22 @@ class TestBzr(StdoutTestCase): # pylint: disable=too-many-public-methods 'BZR_LOG': os.path.join(self.tmpdir, "bzr.log")} # Create a Git repository with two commits - subprocess.check_output(['bzr', 'init', 'repo'], + subprocess.check_output(['bzr', 'init', 'repo'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) os.chdir('repo') - subprocess.check_output(['bzr', 'whoami', 'lava-ci@example.com'], + subprocess.check_output(['bzr', 'whoami', 'lava-ci@example.com'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) with open('test.txt', 'w') as datafile: datafile.write("Some data") - subprocess.check_output(['bzr', 'add', 'test.txt'], + subprocess.check_output(['bzr', 'add', 'test.txt'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) - subprocess.check_output(['bzr', 'commit', 'test.txt', '-m', 'First commit'], + subprocess.check_output(['bzr', 'commit', 'test.txt', '-m', 'First commit'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) with open('second.txt', 'w') as datafile: datafile.write("Some more data") - subprocess.check_output(['bzr', 'add', 'second.txt'], + subprocess.check_output(['bzr', 'add', 'second.txt'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) - subprocess.check_output(['bzr', 'commit', 'second.txt', '-m', 'Second commit'], + subprocess.check_output(['bzr', 'commit', 'second.txt', '-m', 'Second commit'], # nosec - unit test support. env=self.env, stderr=subprocess.STDOUT) # Go back into the tempdir diff --git a/lava_dispatcher/utils/compression.py b/lava_dispatcher/utils/compression.py index dfa6b7f3c..476a8f55e 100644 --- a/lava_dispatcher/utils/compression.py +++ b/lava_dispatcher/utils/compression.py @@ -25,7 +25,7 @@ # vexpress recovery images: any compression though usually zip import os -import subprocess +import subprocess # nosec - internal use. import tarfile from lava_common.exceptions import ( @@ -61,7 +61,7 @@ def compress_file(infile, compression): cmd = compress_command_map[compression][:] cmd.append(infile) try: - subprocess.check_output(cmd) + subprocess.check_output(cmd) # nosec - internal use. return "%s.%s" % (infile, compression) except (OSError, subprocess.CalledProcessError) as exc: raise InfrastructureError('unable to compress file %s: %s' % (infile, exc)) @@ -84,7 +84,7 @@ def decompress_file(infile, compression): if infile.endswith(compression): outfile = infile[:-(len(compression) + 1)] try: - subprocess.check_output(cmd) + subprocess.check_output(cmd) # nosec - internal use. return outfile except (OSError, subprocess.CalledProcessError) as exc: raise InfrastructureError('unable to decompress file %s: %s' % (infile, exc)) diff --git a/lava_dispatcher/utils/filesystem.py b/lava_dispatcher/utils/filesystem.py index 3a009f8ba..4c3d9edce 100644 --- a/lava_dispatcher/utils/filesystem.py +++ b/lava_dispatcher/utils/filesystem.py @@ -24,7 +24,7 @@ import shutil import tarfile import tempfile import guestfs -import subprocess +import subprocess # nosec - internal use. import glob import logging import magic @@ -55,13 +55,13 @@ def rmtree(directory): % (directory, exc)) -def mkdtemp(autoremove=True, basedir='/tmp'): +def mkdtemp(autoremove=True, basedir='/tmp'): # nosec - internal use. """ returns a temporary directory that's deleted when the process exits """ tmpdir = tempfile.mkdtemp(dir=basedir) - os.chmod(tmpdir, 0o755) + os.chmod(tmpdir, 0o755) # nosec - internal use. if autoremove: atexit.register(rmtree, tmpdir) return tmpdir @@ -357,7 +357,7 @@ def copy_overlay_to_sparse_fs(image, overlay): if not is_sparse_image(image): raise JobError("Image is not an Android sparse image: %s" % image) - subprocess.check_output(['/usr/bin/simg2img', image, ext4_img], + subprocess.check_output(['/usr/bin/simg2img', image, ext4_img], # nosec - internal use. stderr=subprocess.STDOUT) guest.add_drive(ext4_img) _launch_guestfs(guest) @@ -379,7 +379,7 @@ def copy_overlay_to_sparse_fs(image, overlay): guest.umount(devices[0]) if int(available) is 0 or percent == '100%': raise JobError("No space in image after applying overlay: %s" % image) - subprocess.check_output(['/usr/bin/img2simg', ext4_img, image], + subprocess.check_output(['/usr/bin/img2simg', ext4_img, image], # nosec - internal use. stderr=subprocess.STDOUT) os.remove(ext4_img) diff --git a/lava_dispatcher/utils/network.py b/lava_dispatcher/utils/network.py index 08aa0ef09..bc146188b 100644 --- a/lava_dispatcher/utils/network.py +++ b/lava_dispatcher/utils/network.py @@ -28,7 +28,7 @@ import os import netifaces import random import socket -import subprocess +import subprocess # nosec - internal use. from lava_common.exceptions import InfrastructureError from lava_common.constants import ( XNBD_PORT_RANGE_MIN, @@ -71,7 +71,7 @@ def rpcinfo_nfs(server, version=3): :return: None if success, message if fail """ with open(os.devnull, 'w') as devnull: - proc = subprocess.Popen(['/usr/sbin/rpcinfo', '-u', server, 'nfs', "%s" % version], stdout=devnull, stderr=subprocess.PIPE) + proc = subprocess.Popen(['/usr/sbin/rpcinfo', '-u', server, 'nfs', "%s" % version], stdout=devnull, stderr=subprocess.PIPE) # nosec - internal use. msg = proc.communicate() if msg[1]: return "%s %s" % (server, msg[1]) diff --git a/lava_dispatcher/utils/vcs.py b/lava_dispatcher/utils/vcs.py index 8202200c1..ffb86a914 100644 --- a/lava_dispatcher/utils/vcs.py +++ b/lava_dispatcher/utils/vcs.py @@ -21,7 +21,7 @@ import logging import os import shutil -import subprocess +import subprocess # nosec - internal use. import yaml from lava_common.exceptions import InfrastructureError @@ -53,18 +53,18 @@ class BzrHelper(VCSHelper): try: if revision is not None: logger.debug("Running '%s branch -r %s %s'", self.binary, str(revision), self.url) - subprocess.check_output([self.binary, 'branch', '-r', + subprocess.check_output([self.binary, 'branch', '-r', # nosec - internal use. str(revision), self.url, dest_path], stderr=subprocess.STDOUT, env=env) commit_id = revision else: logger.debug("Running '%s branch %s'", self.binary, self.url) - subprocess.check_output([self.binary, 'branch', self.url, + subprocess.check_output([self.binary, 'branch', self.url, # nosec - internal use. dest_path], stderr=subprocess.STDOUT, env=env) os.chdir(dest_path) - commit_id = subprocess.check_output(['bzr', 'revno'], + commit_id = subprocess.check_output(['bzr', 'revno'], # nosec - internal use. env=env).strip().decode('utf-8', errors="replace") except subprocess.CalledProcessError as exc: @@ -112,16 +112,16 @@ class GitHelper(VCSHelper): cmd_args.append("--depth=1") logger.debug("Running '%s'", " ".join(cmd_args)) - subprocess.check_output(cmd_args, stderr=subprocess.STDOUT) + subprocess.check_output(cmd_args, stderr=subprocess.STDOUT) # nosec - internal use. if revision is not None: logger.debug("Running '%s checkout %s", self.binary, str(revision)) - subprocess.check_output([self.binary, '-C', dest_path, + subprocess.check_output([self.binary, '-C', dest_path, # nosec - internal use. 'checkout', str(revision)], stderr=subprocess.STDOUT) - commit_id = subprocess.check_output([self.binary, '-C', dest_path, + commit_id = subprocess.check_output([self.binary, '-C', dest_path, # nosec - internal use. 'log', '-1', '--pretty=%H'], stderr=subprocess.STDOUT).strip() diff --git a/lava_results_app/dbutils.py b/lava_results_app/dbutils.py index b505b8d83..31f22cbb1 100644 --- a/lava_results_app/dbutils.py +++ b/lava_results_app/dbutils.py @@ -208,7 +208,7 @@ def _get_job_metadata(job): retval = {} # Add original_definition checksum to metadata retval.update({ - 'definition-checksum': hashlib.md5( + 'definition-checksum': hashlib.md5( # nosec - not used for crypto job.original_definition.encode('utf-8')).hexdigest() }) # Add lava-server-version to metadata diff --git a/lava_scheduler_app/checks.py b/lava_scheduler_app/checks.py index a3690602a..dfa6201ab 100644 --- a/lava_scheduler_app/checks.py +++ b/lava_scheduler_app/checks.py @@ -19,7 +19,7 @@ import os from pwd import getpwuid import stat -import subprocess +import subprocess # nosec system from django.core.checks import ( Error, @@ -110,7 +110,7 @@ def check_permissions(app_configs, **kwargs): def _package_status(name, errors, info=False): try: - out = subprocess.check_output(["dpkg-query", "--status", name], + out = subprocess.check_output(["dpkg-query", "--status", name], # nosec system stderr=subprocess.STDOUT).decode("utf-8").split("\n") if out[1] != "Status: install ok installed": errors.append(Error('not installed correctly', obj=name)) @@ -163,13 +163,13 @@ def check_services(app_configs, **kwargs): for service in services: try: - subprocess.check_call(['systemctl', '-q', 'is-active', service]) + subprocess.check_call(['systemctl', '-q', 'is-active', service]) # nosec system except subprocess.CalledProcessError: errors.append(Error("%s service is not active." % service, obj="lava service")) for service in optional: try: - subprocess.check_call(['systemctl', '-q', 'is-active', service]) + subprocess.check_call(['systemctl', '-q', 'is-active', service]) # nosec system except subprocess.CalledProcessError: errors.append(Info("%s service is not active." % service, obj="lava service")) return errors diff --git a/lava_scheduler_app/migrations/0027_device_dict_onto_filesystem.py b/lava_scheduler_app/migrations/0027_device_dict_onto_filesystem.py index 7628055e9..47cb918e2 100644 --- a/lava_scheduler_app/migrations/0027_device_dict_onto_filesystem.py +++ b/lava_scheduler_app/migrations/0027_device_dict_onto_filesystem.py @@ -3,7 +3,7 @@ from django.db import migrations import base64 import errno import os -import pickle +import pickle # nosec - migration no longer in active use import pprint @@ -50,7 +50,7 @@ def migrate_device_dict_to_filesystem(apps, schema_editor): hostname = device_dict.kee.replace('__KV_STORE_::lava_scheduler_app.models.DeviceDictionary:', '') value64 = device_dict.value valuepickled = base64.b64decode(value64) - value = pickle.loads(valuepickled) + value = pickle.loads(valuepickled) # nosec - no longer in active use DDT[hostname] = devicedictionary_to_jinja2(value['parameters'], value['parameters']['extends']) # Dump the device dictionaries to file system diff --git a/lava_scheduler_app/tests/test_menus.py b/lava_scheduler_app/tests/test_menus.py index 5d20e1355..16d2d5489 100644 --- a/lava_scheduler_app/tests/test_menus.py +++ b/lava_scheduler_app/tests/test_menus.py @@ -7,7 +7,7 @@ from lava_scheduler_app.tests.test_submission import TestCaseWithFactory class YamlMenuFactory(YamlFactory): def make_fake_mustang_device(self, hostname='fakemustang1'): # pylint: disable=no-self-use - assert hostname == 'fakemustang1' + assert hostname == 'fakemustang1' # nosec - unit test support def make_job_data(self, actions=None, **kw): sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'mustang-menu-ramdisk.yaml') diff --git a/lava_scheduler_app/tests/test_pipeline.py b/lava_scheduler_app/tests/test_pipeline.py index 9675f4f78..e27a3059e 100644 --- a/lava_scheduler_app/tests/test_pipeline.py +++ b/lava_scheduler_app/tests/test_pipeline.py @@ -4,7 +4,7 @@ import yaml import jinja2 import unittest import logging -import subprocess +import subprocess # nosec unit test support from nose.tools import nottest from lava_scheduler_app.models import ( Device, @@ -547,7 +547,7 @@ def check_rpcinfo(server='127.0.0.1'): returns True on failure. """ try: - subprocess.check_output(['/usr/sbin/rpcinfo', '-u', server, 'nfs', '3']) + subprocess.check_output(['/usr/sbin/rpcinfo', '-u', server, 'nfs', '3']) # nosec unit test except (OSError, subprocess.CalledProcessError): return True return False diff --git a/lava_scheduler_app/utils.py b/lava_scheduler_app/utils.py index e17e27844..86c78e71b 100644 --- a/lava_scheduler_app/utils.py +++ b/lava_scheduler_app/utils.py @@ -24,7 +24,7 @@ import errno import ldap import logging import os -import subprocess +import subprocess # nosec verified import yaml from collections import OrderedDict @@ -244,7 +244,7 @@ def send_irc_notification(nick, recipient, message, nick, nick, nick, recipient, message, server, port) - proc = subprocess.Popen(['/bin/bash', '-c', netcat_cmd], + proc = subprocess.Popen(['/bin/bash', '-c', netcat_cmd], # nosec managed. stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/lava_server/api.py b/lava_server/api.py index 34be22807..441bf0592 100644 --- a/lava_server/api.py +++ b/lava_server/api.py @@ -19,7 +19,7 @@ # along with LAVA. If not, see <http://www.gnu.org/licenses/>. import os -import subprocess +import subprocess # nosec internal import xmlrpc.client import yaml @@ -80,7 +80,7 @@ class LavaSystemAPI(SystemAPI): changelog = '/usr/share/doc/lava-server/changelog.Debian.gz' if os.path.exists(changelog): - deb_version = subprocess.check_output(( + deb_version = subprocess.check_output(( # nosec internal 'dpkg-query', '-W', "-f=${Version}\n", "lava-server")).strip().decode('utf-8') return deb_version diff --git a/man/conf.py b/man/conf.py index d25024db5..ab7e65a6c 100644 --- a/man/conf.py +++ b/man/conf.py @@ -13,7 +13,7 @@ import sys import os -import subprocess +import subprocess # nosec - internal # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -61,7 +61,7 @@ copyright = u'2010-2018, Linaro Limited' # built documents. # # The short X.Y version. -version = subprocess.Popen(r'./version.py', cwd=r'..', stdout=subprocess.PIPE).stdout.read().decode('utf-8') +version = subprocess.Popen(r'./version.py', cwd=r'..', stdout=subprocess.PIPE).stdout.read().decode('utf-8') # nosec - internal # The full version, including alpha/beta/rc tags. release = version diff --git a/share/dev.py b/share/dev.py index 3533d21e6..8aeacb4b2 100644 --- a/share/dev.py +++ b/share/dev.py @@ -21,7 +21,7 @@ import argparse import simplejson import os -import subprocess +import subprocess # nosec - internal import sys @@ -38,7 +38,7 @@ def handle_on(options): # Check that the sources are already present if not os.path.exists("lava-server/.git"): print("Downloading the sources") - subprocess.check_call(["git", "clone", options.url]) + subprocess.check_call(["git", "clone", options.url]) # nosec - internal os.chdir("/usr/lib/python3/dist-packages") # Making backups @@ -95,7 +95,7 @@ def _restart(): print("Restarting the services:") for service in services: print("* %s" % service) - subprocess.check_call(["service", service, "restart"]) + subprocess.check_call(["service", service, "restart"]) # nosec - internal def main(): @@ -116,7 +116,7 @@ def main(): options = parser.parse_args() # Check that we are running this script on a debian machine - out = subprocess.check_output(["lsb_release", "--id"], + out = subprocess.check_output(["lsb_release", "--id"], # nosec - internal stderr=subprocess.STDOUT).decode("utf-8") if out != "Distributor ID:\tDebian\n": print("Not running on a Debian system") diff --git a/share/download-test-suites-api.py b/share/download-test-suites-api.py index 43b319b2c..eec5291b6 100755 --- a/share/download-test-suites-api.py +++ b/share/download-test-suites-api.py @@ -66,14 +66,14 @@ def main(): '%s/yaml_summary' % (job_results_url), params={'user': args.username, 'token': args.token}) - for test_suite in yaml.load(summary_response.content): + for test_suite in yaml.safe_load(summary_response.content): if test_suite['name'] in args.testsuites: print("Test case list for suite '%s':" % test_suite["name"]) suite_response = requests.get( '%s/%s/yaml' % (job_results_url, test_suite['name']), params={'user': args.username, 'token': args.token}) - for test_case in yaml.load(suite_response.content): + for test_case in yaml.safe_load(suite_response.content): print("- %s" % test_case["name"]) diff --git a/share/javascript.py b/share/javascript.py index 35ccb6b93..aef7e92e9 100755 --- a/share/javascript.py +++ b/share/javascript.py @@ -22,7 +22,7 @@ import os import sys import yaml import argparse -import subprocess +import subprocess # nosec - internal # pylint: disable=too-many-branches @@ -110,8 +110,8 @@ def uglify(os_name, data, dirname, remove=False, simulate=False): if not simulate: try: - subprocess.check_call( - ['uglifyjs', orig_path, '-o', + subprocess.check_call( # nosec - internal + ['/usr/bin/uglifyjs', orig_path, '-o', dest_path, '-c', '-m'], stderr=open(os.devnull, 'wb')) except Exception as e: @@ -148,7 +148,7 @@ def main(): action='store_true', help='Only echo the commands') args = parser.parse_args() - data = yaml.load(open(args.filename, 'r')) + data = yaml.safe_load(open(args.filename, 'r')) # only have data for debian-based packages so far. dependencies = handle_embedded('debian', data, os.getcwd(), args.simulate) dep_list = [] diff --git a/share/lava_lxc_device_add.py b/share/lava_lxc_device_add.py index 4d069f367..ed926ed74 100755 --- a/share/lava_lxc_device_add.py +++ b/share/lava_lxc_device_add.py @@ -29,7 +29,7 @@ import os import argparse import logging -import subprocess +import subprocess # nosec - internal import sys import time import syslog @@ -111,7 +111,7 @@ def main(): lxc_cmd = ['lxc-device', '-n', lxc_name, 'add', device] try: - output = subprocess.check_output(lxc_cmd, stderr=subprocess.STDOUT) + output = subprocess.check_output(lxc_cmd, stderr=subprocess.STDOUT) # nosec - internal output = output.decode("utf-8", errors="replace") logger.debug(output) logger.info("[%s] device %s added", uniq_str, device) diff --git a/share/postinst.py b/share/postinst.py index 7da56fd1f..adbb9a45b 100755 --- a/share/postinst.py +++ b/share/postinst.py @@ -25,7 +25,7 @@ import pwd import random import shutil import glob -import subprocess +import subprocess # nosec - controlled inputs. import sys from lava_server.settings.config_file import ConfigFile @@ -54,7 +54,7 @@ def psql_run(cmd_list, failure_msg): def run(cmd_list, failure_msg): print(" ".join(cmd_list)) try: - ret = subprocess.check_call(cmd_list) + ret = subprocess.check_call(cmd_list) # nosec - internal. except subprocess.CalledProcessError: print(failure_msg) # all failures are fatal during setup @@ -84,7 +84,7 @@ def db_setup(config, pg_admin_username, pg_admin_password): except psycopg2.ProgrammingError as exc: print(exc) - cursor.execute("SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s')" % config.LAVA_DB_NAME) + cursor.execute("SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s')" % config.LAVA_DB_NAME) # nosec - not accessible. db_existed_before = cursor.fetchone()[0] if not db_existed_before: @@ -216,7 +216,7 @@ def configure(): # Allow lavaserver to write to all the log files # setgid on LAVA_LOGS directory - os.chmod(LAVA_LOGS, 0o2775) + os.chmod(LAVA_LOGS, 0o2775) # nosec - group permissive. # Allow users in the adm group to read all logs with open("%s/django.log" % LAVA_LOGS, 'w+') as logfile: diff --git a/share/release-queue.py b/share/release-queue.py index 0a8cb5571..8fdf268aa 100755 --- a/share/release-queue.py +++ b/share/release-queue.py @@ -21,7 +21,7 @@ import argparse import re -import subprocess +import subprocess # nosec - local import time @@ -35,8 +35,8 @@ class Commit: self.commit_id = commit_id self.change_id = change_id - self.obj = subprocess.check_output(['git', 'cat-file', '-p', self.commit_id]).decode('utf-8') - self.hash = subprocess.check_output(['git', 'rev-parse', '--short', self.commit_id]).decode('utf-8').strip() + self.obj = subprocess.check_output(['git', 'cat-file', '-p', self.commit_id]).decode('utf-8') # nosec - internal + self.hash = subprocess.check_output(['git', 'rev-parse', '--short', self.commit_id]).decode('utf-8').strip() # nosec - internal break_next_time = False for line in self.obj.split('\n'): @@ -63,8 +63,8 @@ class Commit: def get_change_ids(branch): results = [] - subprocess.check_output(["git", "checkout", branch], stderr=subprocess.STDOUT) - lines = subprocess.check_output(["git", "log"]).decode('utf-8') + subprocess.check_output(["git", "checkout", branch], stderr=subprocess.STDOUT) # nosec - internal + lines = subprocess.check_output(["git", "log"]).decode('utf-8') # nosec - internal for line in lines.split('\n'): if "Change-Id" in line: m = change_id_pattern.match(line) @@ -85,7 +85,7 @@ def main(): # Check the current working directory try: - subprocess.check_call(["git", "rev-parse"]) + subprocess.check_call(["git", "rev-parse"]) # nosec - internal except subprocess.CalledProcessError: print("Ensure this script is run from the git working copy.") return 1 @@ -99,8 +99,8 @@ def main(): diff.sort() # Go back to master - subprocess.check_output(["git", "checkout", master_branch], stderr=subprocess.STDOUT) - lines = subprocess.check_output(["git", "log"]).decode('utf-8') + subprocess.check_output(["git", "checkout", master_branch], stderr=subprocess.STDOUT) # nosec - internal + lines = subprocess.check_output(["git", "log"]).decode('utf-8') # nosec - internal # List the missing commits current_hash = '' diff --git a/share/render-template.py b/share/render-template.py index 04ff21fd8..aaa2f9b64 100755 --- a/share/render-template.py +++ b/share/render-template.py @@ -65,11 +65,11 @@ def main(): help='Path to the device-types template folder') args = parser.parse_args() - env = Environment( + env = Environment( # nosec rendering to YAML loader=FileSystemLoader( [os.path.join(args.path, 'devices'), os.path.join(args.path, 'device-types')]), - trim_blocks=True) + trim_blocks=True, autoescape=False) if not os.path.exists(os.path.join(args.path, 'devices', "%s.jinja2" % args.device)): print("Cannot find %s device configuration file" % args.device) return diff --git a/version.py b/version.py index 7b88340b1..648d97547 100755 --- a/version.py +++ b/version.py @@ -19,7 +19,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # -import subprocess +import subprocess # nosec - internal import os @@ -38,9 +38,9 @@ def version_tag(): """ describe = None if os.path.exists("./.git/"): - return subprocess.check_output(['git', 'describe']).strip().decode('utf-8') + return subprocess.check_output(['git', 'describe']).strip().decode('utf-8') # nosec - internal if os.path.exists('debian/changelog'): - return subprocess.check_output(('dpkg-parsechangelog', '--show-field', + return subprocess.check_output(('dpkg-parsechangelog', '--show-field', # nosec - internal 'Version')).strip().decode('utf-8').split('-')[0] |