From b0c32bd3a3b6ba0089e2330ee2e59a8d91fbc8fc Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 6 May 2018 22:22:13 +0200 Subject: [PATCH 01/55] [connection] Init new connection module Requires django-netjsonconfig 0.8.1 Allows to - connect to devices using different protocols - SSH protocol supported by default - centrally launch update config operations when config/templates are changed Signed-off-by: Federico Capoano --- openwisp_controller/config/tests/__init__.py | 2 + openwisp_controller/connection/__init__.py | 1 + openwisp_controller/connection/admin.py | 35 +++ openwisp_controller/connection/apps.py | 14 ++ .../connection/connectors/__init__.py | 0 .../connection/connectors/openwrt/__init__.py | 0 .../connection/connectors/openwrt/ssh.py | 6 + .../connection/connectors/ssh.py | 103 +++++++++ .../connection/migrations/0001_initial.py | 74 +++++++ .../connection/migrations/__init__.py | 0 openwisp_controller/connection/models.py | 167 ++++++++++++++ .../connection/tests/__init__.py | 0 .../connection/tests/test_models.py | 203 ++++++++++++++++++ openwisp_controller/connection/utils.py | 27 +++ setup.py | 5 +- tests/settings.py | 23 ++ tests/test-key.rsa | 15 ++ 17 files changed, 674 insertions(+), 1 deletion(-) create mode 100644 openwisp_controller/connection/__init__.py create mode 100644 openwisp_controller/connection/admin.py create mode 100644 openwisp_controller/connection/apps.py create mode 100644 openwisp_controller/connection/connectors/__init__.py create mode 100644 openwisp_controller/connection/connectors/openwrt/__init__.py create mode 100644 openwisp_controller/connection/connectors/openwrt/ssh.py create mode 100644 openwisp_controller/connection/connectors/ssh.py create mode 100644 openwisp_controller/connection/migrations/0001_initial.py create mode 100644 openwisp_controller/connection/migrations/__init__.py create mode 100644 openwisp_controller/connection/models.py create mode 100644 openwisp_controller/connection/tests/__init__.py create mode 100644 openwisp_controller/connection/tests/test_models.py create mode 100644 openwisp_controller/connection/utils.py create mode 100644 tests/test-key.rsa diff --git a/openwisp_controller/config/tests/__init__.py b/openwisp_controller/config/tests/__init__.py index 6edd6e0b6..cba2205f2 100644 --- a/openwisp_controller/config/tests/__init__.py +++ b/openwisp_controller/config/tests/__init__.py @@ -19,4 +19,6 @@ def _create_config(self, **kwargs): if 'device' not in kwargs: kwargs['device'] = self._create_device(name='test-device', organization=kwargs.get('organization', None)) + if 'organization' not in kwargs: + kwargs['organization'] = kwargs['device'].organization return super(CreateConfigTemplateMixin, self)._create_config(**kwargs) diff --git a/openwisp_controller/connection/__init__.py b/openwisp_controller/connection/__init__.py new file mode 100644 index 000000000..07f6254df --- /dev/null +++ b/openwisp_controller/connection/__init__.py @@ -0,0 +1 @@ +default_app_config = 'openwisp_controller.connection.apps.ConnectionConfig' diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py new file mode 100644 index 000000000..9175fca03 --- /dev/null +++ b/openwisp_controller/connection/admin.py @@ -0,0 +1,35 @@ +from django.contrib import admin + +from openwisp_utils.admin import MultitenantOrgFilter, TimeReadonlyAdminMixin + +from ..admin import MultitenantAdminMixin +from ..config.admin import DeviceAdmin +from .models import Credentials, DeviceConnection, DeviceIp + + +@admin.register(Credentials) +class CredentialsAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin): + list_display = ('name', 'organization', 'connector', 'created', 'modified') + list_filter = [('organization', MultitenantOrgFilter), + 'connector'] + list_select_related = ('organization',) + + +class DeviceIpInline(admin.TabularInline): + model = DeviceIp + exclude = ('created', 'modified') + extra = 0 + + def get_queryset(self, request): + qs = super(DeviceIpInline, self).get_queryset(request) + return qs.order_by('priority') + + +class DeviceConnectionInline(admin.StackedInline): + model = DeviceConnection + exclude = ['params', 'created', 'modified'] + readonly_fields = ['is_working', 'failure_reason', 'last_attempt'] + extra = 0 + + +DeviceAdmin.inlines += [DeviceConnectionInline, DeviceIpInline] diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py new file mode 100644 index 000000000..c25b8ae3c --- /dev/null +++ b/openwisp_controller/connection/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ +from django_netjsonconfig.signals import config_modified + + +class ConnectionConfig(AppConfig): + name = 'openwisp_controller.connection' + label = 'connection' + verbose_name = _('Network Device Credentials') + + def ready(self): + # connect to config_modified signal + from .models import DeviceConnection + config_modified.connect(DeviceConnection.config_modified_receiver) diff --git a/openwisp_controller/connection/connectors/__init__.py b/openwisp_controller/connection/connectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/connectors/openwrt/__init__.py b/openwisp_controller/connection/connectors/openwrt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/connectors/openwrt/ssh.py b/openwisp_controller/connection/connectors/openwrt/ssh.py new file mode 100644 index 000000000..ea646afde --- /dev/null +++ b/openwisp_controller/connection/connectors/openwrt/ssh.py @@ -0,0 +1,6 @@ +from ..ssh import Ssh + + +class OpenWrt(Ssh): + def update_config(self): + self.shell.exec_command('/etc/init.d/openwisp_config restart') diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py new file mode 100644 index 000000000..61a57863f --- /dev/null +++ b/openwisp_controller/connection/connectors/ssh.py @@ -0,0 +1,103 @@ +import ipaddress +import logging + +import paramiko +from django.core.exceptions import ObjectDoesNotExist +from django.utils.functional import cached_property +from jsonschema import validate +from jsonschema.exceptions import ValidationError as SchemaError +from scp import SCPClient + +from ..utils import get_interfaces + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + + +logger = logging.getLogger(__name__) + + +class Ssh(object): + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": False, + "required": ["username"], + "properties": { + "username": {"type": "string"}, + "password": {"type": "string"}, + "key": {"type": "string"}, + "port": {"type": "integer"}, + } + } + + def __init__(self, device_connection): + self.connection = device_connection + self.device = device_connection.device + self.shell = paramiko.SSHClient() + self.shell.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + @classmethod + def validate(cls, params): + validate(params, cls.schema) + cls.custom_validation(params) + + @classmethod + def custom_validation(cls, params): + if 'password' not in params and 'key' not in params: + raise SchemaError('Missing password or key') + + @cached_property + def _addresses(self): + deviceip_set = list(self.device.deviceip_set.all() + .only('address') + .order_by('priority')) + address_list = [] + for deviceip in deviceip_set: + address = deviceip.address + ip = ipaddress.ip_address(address) + if not ip.is_link_local: + address_list.append(address) + else: + for interface in get_interfaces(): + address_list.append('{0}%{1}'.format(address, interface)) + try: + address_list.append(self.device.config.last_ip) + except ObjectDoesNotExist: + pass + return address_list + + @cached_property + def _params(self): + params = self.connection.get_params() + if 'key' in params: + key_fileobj = StringIO(params.pop('key')) + params['pkey'] = paramiko.RSAKey.from_private_key(key_fileobj) + return params + + def connect(self): + success = False + exception = None + for address in self._addresses: + try: + self.shell.connect(address, **self._params) + except Exception as e: + exception = e + else: + success = True + break + if not success: + raise exception + + def disconnect(self): + self.shell.close() + + def update_config(self): + raise NotImplementedError() + + def upload(self, fl, remote_path): + scp = SCPClient(self.shell.get_transport()) + scp.putfo(fl, remote_path) + scp.close() diff --git a/openwisp_controller/connection/migrations/0001_initial.py b/openwisp_controller/connection/migrations/0001_initial.py new file mode 100644 index 000000000..5096ec1b8 --- /dev/null +++ b/openwisp_controller/connection/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# Generated by Django 2.0.5 on 2018-05-05 17:33 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +import openwisp_users.mixins +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('config', '0012_auto_20180219_1501'), + ('openwisp_users', '0002_auto_20180219_1409'), + ] + + operations = [ + migrations.CreateModel( + name='Credentials', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('name', models.CharField(db_index=True, max_length=64, unique=True)), + ('connector', models.CharField(choices=[('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH')], db_index=True, max_length=128, verbose_name='connection type')), + ('params', jsonfield.fields.JSONField(default=dict, help_text='global connection parameters', verbose_name='parameters')), + ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='openwisp_users.Organization', verbose_name='organization')), + ], + options={ + 'verbose_name_plural': 'Access credentials', + 'verbose_name': 'Access credentials', + }, + bases=(openwisp_users.mixins.ValidateOrgMixin, models.Model), + ), + migrations.CreateModel( + name='DeviceConnection', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('update_strategy', models.CharField(blank=True, choices=[('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH')], db_index=True, help_text='leave blank to determine automatically', max_length=128, verbose_name='update strategy')), + ('enabled', models.BooleanField(db_index=True, default=True)), + ('params', jsonfield.fields.JSONField(blank=True, default=dict, help_text='local connection parameters (will override the global parameters if specified)', verbose_name='parameters')), + ('is_working', models.NullBooleanField(default=None)), + ('last_attempt', models.DateTimeField(blank=True, null=True)), + ('failure_reason', models.CharField(blank=True, max_length=128, verbose_name='reason of failure')), + ('credentials', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='connection.Credentials')), + ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='config.Device')), + ], + options={ + 'verbose_name_plural': 'Device connections', + 'verbose_name': 'Device connection', + }, + ), + migrations.CreateModel( + name='DeviceIp', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('address', models.GenericIPAddressField(verbose_name='IP address')), + ('priority', models.PositiveSmallIntegerField()), + ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='config.Device')), + ], + options={ + 'verbose_name_plural': 'Device IP addresses', + 'verbose_name': 'Device IP', + }, + ), + ] diff --git a/openwisp_controller/connection/migrations/__init__.py b/openwisp_controller/connection/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py new file mode 100644 index 000000000..ca4fccba9 --- /dev/null +++ b/openwisp_controller/connection/models.py @@ -0,0 +1,167 @@ +import collections + +from django.core.exceptions import ValidationError +from django.db import models +from django.utils import timezone +from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.module_loading import import_string +from django.utils.translation import ugettext_lazy as _ +from django_netjsonconfig.base.base import BaseModel +from jsonfield import JSONField +from jsonschema.exceptions import ValidationError as SchemaError + +from openwisp_users.mixins import ShareableOrgMixin +from openwisp_utils.base import TimeStampedEditableModel + + +class ConnectorMixin(object): + _connector_field = 'connector' + + def clean(self): + self._validate_connector_schema() + + def _validate_connector_schema(self): + try: + self.connector_class.validate(self.get_params()) + except SchemaError as e: + raise ValidationError({'params': e.message}) + + def get_params(self): + return self.params + + @cached_property + def connector_class(self): + return import_string(getattr(self, self._connector_field)) + + @cached_property + def connector_instance(self): + return self.connector_class(self) + + +class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): + """ + Credentials for access + """ + CONNECTOR_CHOICES = ( + ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), + ) + connector = models.CharField(_('connection type'), + choices=CONNECTOR_CHOICES, + max_length=128, + db_index=True) + params = JSONField(_('parameters'), + default=dict, + help_text=_('global connection parameters'), + load_kwargs={'object_pairs_hook': collections.OrderedDict}, + dump_kwargs={'indent': 4}) + + class Meta: + verbose_name = _('Access credentials') + verbose_name_plural = verbose_name + + def __str__(self): + return '{0} ({1})'.format(self.name, self.get_connector_display()) + + +@python_2_unicode_compatible +class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): + UPDATE_STRATEGY_CHOICES = ( + ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), + ) + CONFIG_BACKEND_MAPPING = { + 'netjsonconfig.OpenWrt': UPDATE_STRATEGY_CHOICES[0][0], + } + device = models.ForeignKey('config.Device', on_delete=models.CASCADE) + credentials = models.ForeignKey(Credentials, on_delete=models.CASCADE) + update_strategy = models.CharField(_('update strategy'), + help_text=_('leave blank to determine automatically'), + choices=UPDATE_STRATEGY_CHOICES, + max_length=128, + blank=True, + db_index=True) + enabled = models.BooleanField(default=True, db_index=True) + params = JSONField(_('parameters'), + default=dict, + blank=True, + help_text=_('local connection parameters (will override ' + 'the global parameters if specified)'), + load_kwargs={'object_pairs_hook': collections.OrderedDict}, + dump_kwargs={'indent': 4}) + # usability improvements + is_working = models.NullBooleanField(default=None) + failure_reason = models.CharField(_('reason of failure'), + max_length=128, + blank=True) + last_attempt = models.DateTimeField(blank=True, null=True) + _connector_field = 'update_strategy' + + class Meta: + verbose_name = _('Device connection') + verbose_name_plural = _('Device connections') + + def clean(self): + if not self.update_strategy and hasattr(self.device, 'config'): + try: + self.update_strategy = self.CONFIG_BACKEND_MAPPING[self.device.config.backend] + except KeyError as e: + raise ValidationError({ + 'update_stragy': _('could not determine update strategy ' + ' automatically, exception: {0}'.format(e)) + }) + elif not self.update_strategy: + raise ValidationError({ + 'update_strategy': _('the update strategy can be determined automatically ' + 'only if the device has a configuration specified, ' + 'because it is inferred from the configuration backend. ' + 'Please select the update strategy manually.') + }) + self._validate_connector_schema() + + def get_params(self): + params = self.credentials.params.copy() + params.update(self.params) + return params + + def connect(self): + try: + self.connector_instance.connect() + except Exception as e: + self.is_working = False + self.failure_reason = str(e) + else: + self.is_working = True + self.failure_reason = '' + finally: + self.last_attempt = timezone.now() + self.save() + + def disconnect(self): + self.connector_instance.disconnect() + + @classmethod + def config_modified_receiver(cls, **kwargs): + """ + receiver for ``config_modified`` signal + triggers the ``update_config`` operation + """ + qs = kwargs['device'].deviceconnection_set.filter(enabled=True) + if qs.count() > 0: + conn = qs.first() + conn.connector_instance.connect() + if conn.is_working: + conn.connector_instance.update_config() + + +@python_2_unicode_compatible +class DeviceIp(TimeStampedEditableModel): + device = models.ForeignKey('config.Device', on_delete=models.CASCADE) + address = models.GenericIPAddressField(_('IP address')) + priority = models.PositiveSmallIntegerField() + + class Meta: + verbose_name = _('Device IP') + verbose_name_plural = _('Device IP addresses') + + def __str__(self): + return self.address diff --git a/openwisp_controller/connection/tests/__init__.py b/openwisp_controller/connection/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py new file mode 100644 index 000000000..2e3bc6a42 --- /dev/null +++ b/openwisp_controller/connection/tests/test_models.py @@ -0,0 +1,203 @@ +import os + +import paramiko +from django.conf import settings +from django.core.exceptions import ValidationError +from django.test import TestCase +from mockssh import Server as SshServer +from openwisp_controller.config.models import Config, Device +from openwisp_controller.config.tests import CreateConfigTemplateMixin + +from openwisp_users.tests.utils import TestOrganizationMixin + +from ..models import Credentials, DeviceConnection, DeviceIp + + +class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): + device_model = Device + config_model = Config + _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') + with open(_TEST_RSA_KEY_PATH, 'r') as f: + _SSH_PRIVATE_KEY = f.read() + + @classmethod + def setUpClass(cls): + cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) + cls.ssh_server.__enter__() + + @classmethod + def tearDownClass(cls): + try: + cls.ssh_server.__exit__() + except OSError: + pass + + def _create_credentials(self, **kwargs): + opts = dict(name='Test credentials', + connector=Credentials.CONNECTOR_CHOICES[0][0], + params={'username': 'root', + 'password': 'password', + 'port': 22}) + opts.update(kwargs) + if 'organization' not in opts: + opts['organization'] = self._create_org() + c = Credentials(**opts) + c.full_clean() + c.save() + return c + + def _create_credentials_with_key(self, username='root', port=22, **kwargs): + opts = dict(name='Test SSH Key', + params={'username': username, + 'key': self._SSH_PRIVATE_KEY, + 'port': port}) + return self._create_credentials(**opts) + + def _create_device_connection(self, **kwargs): + opts = dict(enabled=True, + params={}) + opts.update(kwargs) + if 'credentials' not in opts: + opts['credentials'] = self._create_credentials() + org = opts['credentials'].organization + if 'device' not in opts: + opts['device'] = self._create_device(organization=org) + self._create_config(device=opts['device']) + dc = DeviceConnection(**opts) + dc.full_clean() + dc.save() + return dc + + def _create_device_ip(self, **kwargs): + opts = dict(address='10.40.0.1', + priority=1) + opts.update(kwargs) + if 'device' not in opts: + dc = self._create_device_connection() + opts['device'] = dc.device + ip = DeviceIp(**opts) + ip.full_clean() + ip.save() + return ip + + +class TestModels(TestConnectionMixin, TestCase): + def test_connection_str(self): + c = Credentials(name='Dev Key', connector=Credentials.CONNECTOR_CHOICES[0][0]) + self.assertIn(c.name, str(c)) + self.assertIn(c.get_connector_display(), str(c)) + + def test_deviceip_str(self): + di = DeviceIp(address='10.40.0.1') + self.assertIn(di.address, str(di)) + + def test_device_connection_get_params(self): + dc = self._create_device_connection() + self.assertEqual(dc.get_params(), dc.credentials.params) + dc.params = {'port': 2400} + self.assertEqual(dc.get_params()['port'], 2400) + self.assertEqual(dc.get_params()['username'], 'root') + + def test_device_connection_auto_update_strategy(self): + dc = self._create_device_connection() + self.assertEqual(dc.update_strategy, dc.UPDATE_STRATEGY_CHOICES[0][0]) + + def test_device_connection_auto_update_strategy_key_error(self): + orig_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES + orig_mapping = DeviceConnection.CONFIG_BACKEND_MAPPING + DeviceConnection.UPDATE_STRATEGY_CHOICES = (('meddle', 'meddle'),) + DeviceConnection.CONFIG_BACKEND_MAPPING = {'wrong': 'wrong'} + try: + self._create_device_connection() + except ValidationError: + failed = False + else: + failed = True + # restore + DeviceConnection.UPDATE_STRATEGY_CHOICES = orig_strategy + DeviceConnection.CONFIG_BACKEND_MAPPING = orig_mapping + if failed: + self.fail('ValidationError not raised') + + def test_device_connection_auto_update_strategy_missing_config(self): + device = self._create_device(organization=self._create_org()) + self.assertFalse(hasattr(device, 'config')) + try: + self._create_device_connection(device=device) + except ValidationError as e: + self.assertIn('inferred from', str(e)) + else: + self.fail('ValidationError not raised') + + def test_device_connection_connector_instance(self): + dc = self._create_device_connection() + self.assertIsInstance(dc.connector_instance, dc.connector_class) + + def test_device_connection_ssh_key_param(self): + ckey = self._create_credentials_with_key() + dc = self._create_device_connection(credentials=ckey) + self.assertIn('pkey', dc.connector_instance._params) + self.assertIsInstance(dc.connector_instance._params['pkey'], + paramiko.rsakey.RSAKey) + self.assertNotIn('key', dc.connector_instance._params) + + def test_ssh_connect(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connect() + self.assertTrue(dc.is_working) + self.assertIsNotNone(dc.last_attempt) + self.assertEqual(dc.failure_reason, '') + try: + dc.disconnect() + except OSError: + pass + + def test_ssh_connect_failure(self): + ckey = self._create_credentials_with_key(username='wrong', + port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connect() + self.assertEqual(dc.is_working, False) + self.assertIsNotNone(dc.last_attempt) + self.assertEqual(dc.failure_reason, 'Authentication failed.') + + def test_credentials_schema(self): + # unrecognized parameter + try: + self._create_credentials(params={ + 'username': 'root', + 'password': 'password', + 'unrecognized': True + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') + # missing password or key + try: + self._create_credentials(params={ + 'username': 'root', + 'port': 22 + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') + + def test_device_connection_schema(self): + # unrecognized parameter + try: + self._create_device_connection(params={ + 'username': 'root', + 'password': 'password', + 'unrecognized': True + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') diff --git a/openwisp_controller/connection/utils.py b/openwisp_controller/connection/utils.py new file mode 100644 index 000000000..acfcf976b --- /dev/null +++ b/openwisp_controller/connection/utils.py @@ -0,0 +1,27 @@ +import array +import fcntl +import socket +import struct + + +def get_interfaces(): + """ + returns all non loopback interfaces available on the system + """ + max_possible = 128 + bytes_ = max_possible * 32 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + names = array.array('B', b'\0' * bytes_) + outbytes = struct.unpack('iL', fcntl.ioctl( + s.fileno(), + 0x8912, + struct.pack('iL', bytes_, names.buffer_info()[0]) + ))[0] + namestr = names.tostring() + interfaces = [] + for i in range(0, outbytes, 40): + name = namestr[i:i + 16].split(b'\0', 1)[0] + name = name.decode() + if name != 'lo': + interfaces.append(name) + return interfaces diff --git a/setup.py b/setup.py index 1a24d8836..fc602ad40 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,10 @@ "django-netjsonconfig>=0.8.1,<0.10.0", "openwisp-utils[users]<0.3.1", "django-loci>=0.1.1,<0.3.0", - "djangorestframework-gis>=0.12.0,<0.14.0" + "djangorestframework-gis>=0.12.0,<0.14.0", + # connections feature + "paramiko>=2.4.1,<2.5.0", + "scp>=0.13.0,<0.14.0" ], classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tests/settings.py b/tests/settings.py index e26e612d3..687e77334 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -38,6 +38,7 @@ 'openwisp_controller.pki', 'openwisp_controller.config', 'openwisp_controller.geo', + 'openwisp_controller.connection', # admin 'django.contrib.admin', 'django.forms', @@ -121,6 +122,28 @@ # during development only EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +LOGGING = { + 'version': 1, + 'filters': { + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue', + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + 'django.db.backends': { + 'level': 'DEBUG', + 'handlers': ['console'], + } + } +} + # local settings must be imported before test runner otherwise they'll be ignored try: from local_settings import * diff --git a/tests/test-key.rsa b/tests/test-key.rsa new file mode 100644 index 000000000..bdd4846ed --- /dev/null +++ b/tests/test-key.rsa @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDQwuFrDNYUCi5Doy2xrc71P06vEuNG5i2rNgIHm95IuP8WrFwZ +W/VfNviGyhA8JwmWwHco9uzgKthaMKrGKB5Oeu/Z2F6SZPdCAdamCdbCcihXZ4g1 +RGbX5wECH7UjTx0th4GV6jwRAvJM/MpVJcCkTIzBHVHOC5jYotDuTnjJdwIDAQAB +AoGAHvfp7LF4yHxCJLJ+Qs9f1i3QBFSu9oOK3s0iO/K5ZNxcqwZimzhzC+7hq00q +X2IDICPpCWCn/xEcCzURAFhPNlx0RYZUzXOiW1JL7MzLYny87UAuW+TDaS4eEV9r +YX8acLWfg+aEw/pF0FRb2AuoRClztAyNF6GJtR/ky4z7vnECQQD3NEcEL1s913HW +1yV4RHBZO8n8oH2WidXtFDstmdmAvDQv7KC8c6rPJ6VVH5IlY+WyDIzI6X1IJFew +DXhO3A8zAkEA2DBvhy5TbAOPX7wQN53SA9+z4sdhOlYwcDpq2YuYvKH3ZFIWQEAX +cTQSjvaI35jWyKNYL+8T+Pqsngd3AUNsrQJBAI1yCSx42FFDRCz0v8jYCBzW3BVD +03hed9yGlfHatRw3E/lUAQizekm72pshTGM+jMBa8/dFulycBtyCaJNe0QcCQQCQ +uoxPcWIDs7ZuHta0hQEt+rrQnS2oAj9XQqR5kwzja4LVNGcVCFMpQ/UQpFcpaYaQ +t1m4bVNvoVGiUdkHjX3ZAkEAmHvrBB2TvcPZkhuUGviIlXbIeHWZMRF7wh0wZ7SH +SZWnv9EqwFcOGqqoLhQDznTI9TmWdpkxPxLzVwnjWLT4qw== +-----END RSA PRIVATE KEY----- From 077b02c1fe001b7c3d454556a7308833563f02a7 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 7 May 2018 16:39:51 +0200 Subject: [PATCH 02/55] [connections] Execute update_config in the background --- openwisp_controller/connection/apps.py | 29 +++++++++++++++++-- .../connection/connectors/ssh.py | 7 ++++- openwisp_controller/connection/models.py | 26 ++++++++--------- openwisp_controller/connection/tasks.py | 25 ++++++++++++++++ setup.py | 4 ++- tests/__init__.py | 0 tests/manage.py | 2 +- tests/openwisp2/__init__.py | 5 ++++ tests/openwisp2/celery.py | 11 +++++++ .../{ => openwisp2}/local_settings.example.py | 0 tests/{ => openwisp2}/settings.py | 11 +++++-- tests/{ => openwisp2}/test-key.rsa | 0 tests/{ => openwisp2}/urls.py | 0 13 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 openwisp_controller/connection/tasks.py delete mode 100644 tests/__init__.py create mode 100644 tests/openwisp2/__init__.py create mode 100644 tests/openwisp2/celery.py rename tests/{ => openwisp2}/local_settings.example.py (100%) rename tests/{ => openwisp2}/settings.py (94%) rename tests/{ => openwisp2}/test-key.rsa (100%) rename tests/{ => openwisp2}/urls.py (100%) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index c25b8ae3c..7f4e97884 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,6 +1,9 @@ from django.apps import AppConfig from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified +from celery.task.control import inspect + +_TASK_NAME = 'openwisp_controller.connection.tasks.update_config' class ConnectionConfig(AppConfig): @@ -9,6 +12,26 @@ class ConnectionConfig(AppConfig): verbose_name = _('Network Device Credentials') def ready(self): - # connect to config_modified signal - from .models import DeviceConnection - config_modified.connect(DeviceConnection.config_modified_receiver) + """ + connects the ``config_modified`` signal + to the ``update_config`` celery task + which will be executed in the background + """ + from .tasks import update_config + + def config_modified_receiver(**kwargs): + device_id = kwargs['device'].id + if not self._is_update_in_progress(device_id): + update_config.delay(device_id) + + config_modified.connect(config_modified_receiver, + dispatch_uid='connection.update_config', + weak=False) + + def _is_update_in_progress(self, device_id): + # check if there's any other running task before adding it + for task_list in inspect().active().values(): + for task in task_list: + if task['name'] == _TASK_NAME and str(device_id) in task['args']: + return True + return False diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 61a57863f..bb45c5dac 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -17,6 +17,8 @@ logger = logging.getLogger(__name__) +SSH_CONNECTION_TIMEOUT = 5 +SSH_AUTH_TIMEOUT = 2 class Ssh(object): @@ -82,7 +84,10 @@ def connect(self): exception = None for address in self._addresses: try: - self.shell.connect(address, **self._params) + self.shell.connect(address, + timeout=SSH_CONNECTION_TIMEOUT, + auth_timeout=SSH_AUTH_TIMEOUT, + **self._params) except Exception as e: exception = e else: diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index ca4fccba9..f4e14aae3 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -14,6 +14,9 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel +import logging +logger = logging.getLogger(__name__) + class ConnectorMixin(object): _connector_field = 'connector' @@ -66,6 +69,7 @@ def __str__(self): @python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): + _connector_field = 'update_strategy' UPDATE_STRATEGY_CHOICES = ( ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), ) @@ -94,7 +98,6 @@ class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): max_length=128, blank=True) last_attempt = models.DateTimeField(blank=True, null=True) - _connector_field = 'update_strategy' class Meta: verbose_name = _('Device connection') @@ -139,18 +142,15 @@ def connect(self): def disconnect(self): self.connector_instance.disconnect() - @classmethod - def config_modified_receiver(cls, **kwargs): - """ - receiver for ``config_modified`` signal - triggers the ``update_config`` operation - """ - qs = kwargs['device'].deviceconnection_set.filter(enabled=True) - if qs.count() > 0: - conn = qs.first() - conn.connector_instance.connect() - if conn.is_working: - conn.connector_instance.update_config() + def update_config(self): + self.connect() + if self.is_working: + try: + self.connector_instance.update_config() + except Exception as e: + logger.exception(e) + else: + self.device.config.set_status_running() @python_2_unicode_compatible diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py new file mode 100644 index 000000000..0a4880d29 --- /dev/null +++ b/openwisp_controller/connection/tasks.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, unicode_literals + +from celery import shared_task +from time import sleep + +from ..config.models import Device + + +@shared_task +def update_config(device_id): + """ + Launches the ``update_config()`` operation + of a specific device in the background + """ + # wait for the saving operations of this device to complete + # (there may be multiple ones happening at the same time) + sleep(4) + # avoid repeating the operation multiple times + device = Device.objects.select_related('config').get(pk=device_id) + if device.config.status == 'running': + return + qs = device.deviceconnection_set.filter(device_id=device_id, enabled=True) + if qs.count() > 0: + conn = qs.first() + conn.update_config() diff --git a/setup.py b/setup.py index fc602ad40..bee535c7b 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,10 @@ "django-loci>=0.1.1,<0.3.0", "djangorestframework-gis>=0.12.0,<0.14.0", # connections feature + # TODO: make optional? "paramiko>=2.4.1,<2.5.0", - "scp>=0.13.0,<0.14.0" + "scp>=0.13.0,<0.14.0", + "celery>=4.1.0,<4.2.0" ], classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/manage.py b/tests/manage.py index 9811d22f7..dd989d36d 100755 --- a/tests/manage.py +++ b/tests/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'openwisp2.settings') from django.core.management import execute_from_command_line diff --git a/tests/openwisp2/__init__.py b/tests/openwisp2/__init__.py new file mode 100644 index 000000000..76be18a55 --- /dev/null +++ b/tests/openwisp2/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, unicode_literals + +from .celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/tests/openwisp2/celery.py b/tests/openwisp2/celery.py new file mode 100644 index 000000000..440c69813 --- /dev/null +++ b/tests/openwisp2/celery.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openwisp2.settings') + +app = Celery('openwisp2') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/tests/local_settings.example.py b/tests/openwisp2/local_settings.example.py similarity index 100% rename from tests/local_settings.example.py rename to tests/openwisp2/local_settings.example.py diff --git a/tests/settings.py b/tests/openwisp2/settings.py similarity index 94% rename from tests/settings.py rename to tests/openwisp2/settings.py index 687e77334..55c4cd099 100644 --- a/tests/settings.py +++ b/tests/openwisp2/settings.py @@ -74,7 +74,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'urls' +ROOT_URLCONF = 'openwisp2.urls' CHANNEL_LAYERS = { 'default': { @@ -122,6 +122,13 @@ # during development only EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +if not TESTING: + CELERY_BROKER_URL = 'redis://localhost/1' +else: + CELERY_TASK_ALWAYS_EAGER = True + CELERY_TASK_EAGER_PROPAGATES = True + CELERY_BROKER_URL = 'memory://' + LOGGING = { 'version': 1, 'filters': { @@ -146,6 +153,6 @@ # local settings must be imported before test runner otherwise they'll be ignored try: - from local_settings import * + from .local_settings import * except ImportError: pass diff --git a/tests/test-key.rsa b/tests/openwisp2/test-key.rsa similarity index 100% rename from tests/test-key.rsa rename to tests/openwisp2/test-key.rsa diff --git a/tests/urls.py b/tests/openwisp2/urls.py similarity index 100% rename from tests/urls.py rename to tests/openwisp2/urls.py From c39456d7ad489126cb1827e83805b76e2c4df934 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 20:10:29 +0200 Subject: [PATCH 03/55] [connections] Keep connector class and model logic decoupled --- openwisp_controller/connection/apps.py | 2 +- .../connection/connectors/ssh.py | 38 ++++------------- openwisp_controller/connection/models.py | 36 ++++++++++++++-- openwisp_controller/connection/tasks.py | 3 +- .../connection/tests/test_models.py | 42 +++++++++++++++++-- 5 files changed, 82 insertions(+), 39 deletions(-) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 7f4e97884..1f17d9b77 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,7 +1,7 @@ +from celery.task.control import inspect from django.apps import AppConfig from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified -from celery.task.control import inspect _TASK_NAME = 'openwisp_controller.connection.tasks.update_config' diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index bb45c5dac..34ee18d2e 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,15 +1,11 @@ -import ipaddress import logging import paramiko -from django.core.exceptions import ObjectDoesNotExist from django.utils.functional import cached_property from jsonschema import validate from jsonschema.exceptions import ValidationError as SchemaError from scp import SCPClient -from ..utils import get_interfaces - try: from io import StringIO except ImportError: @@ -35,9 +31,9 @@ class Ssh(object): } } - def __init__(self, device_connection): - self.connection = device_connection - self.device = device_connection.device + def __init__(self, params, addresses): + self._params = params + self.addresses = addresses self.shell = paramiko.SSHClient() self.shell.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -52,28 +48,8 @@ def custom_validation(cls, params): raise SchemaError('Missing password or key') @cached_property - def _addresses(self): - deviceip_set = list(self.device.deviceip_set.all() - .only('address') - .order_by('priority')) - address_list = [] - for deviceip in deviceip_set: - address = deviceip.address - ip = ipaddress.ip_address(address) - if not ip.is_link_local: - address_list.append(address) - else: - for interface in get_interfaces(): - address_list.append('{0}%{1}'.format(address, interface)) - try: - address_list.append(self.device.config.last_ip) - except ObjectDoesNotExist: - pass - return address_list - - @cached_property - def _params(self): - params = self.connection.get_params() + def params(self): + params = self._params.copy() if 'key' in params: key_fileobj = StringIO(params.pop('key')) params['pkey'] = paramiko.RSAKey.from_private_key(key_fileobj) @@ -82,12 +58,12 @@ def _params(self): def connect(self): success = False exception = None - for address in self._addresses: + for address in self.addresses: try: self.shell.connect(address, timeout=SSH_CONNECTION_TIMEOUT, auth_timeout=SSH_AUTH_TIMEOUT, - **self._params) + **self.params) except Exception as e: exception = e else: diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index f4e14aae3..82d4ac139 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -1,6 +1,8 @@ import collections +import ipaddress +import logging -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import models from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible @@ -14,7 +16,8 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel -import logging +from .utils import get_interfaces + logger = logging.getLogger(__name__) @@ -33,13 +36,17 @@ def _validate_connector_schema(self): def get_params(self): return self.params + def get_addresses(self): + return [] + @cached_property def connector_class(self): return import_string(getattr(self, self._connector_field)) @cached_property def connector_instance(self): - return self.connector_class(self) + return self.connector_class(params=self.get_params(), + addresses=self.get_addresses()) class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): @@ -121,6 +128,29 @@ def clean(self): }) self._validate_connector_schema() + def get_addresses(self): + """ + returns a list of ip addresses for the related device + (used to pass a list of ip addresses to a DeviceConnection instance) + """ + deviceip_set = list(self.device.deviceip_set.all() + .only('address') + .order_by('priority')) + address_list = [] + for deviceip in deviceip_set: + address = deviceip.address + ip = ipaddress.ip_address(address) + if not ip.is_link_local: + address_list.append(address) + else: + for interface in get_interfaces(): + address_list.append('{0}%{1}'.format(address, interface)) + try: + address_list.append(self.device.config.last_ip) + except ObjectDoesNotExist: + pass + return address_list + def get_params(self): params = self.credentials.params.copy() params.update(self.params) diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py index 0a4880d29..d02727dd3 100644 --- a/openwisp_controller/connection/tasks.py +++ b/openwisp_controller/connection/tasks.py @@ -1,8 +1,9 @@ from __future__ import absolute_import, unicode_literals -from celery import shared_task from time import sleep +from celery import shared_task + from ..config.models import Device diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 2e3bc6a42..42ae72145 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -11,6 +11,7 @@ from openwisp_users.tests.utils import TestOrganizationMixin from ..models import Credentials, DeviceConnection, DeviceIp +from ..utils import get_interfaces class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): @@ -136,10 +137,10 @@ def test_device_connection_connector_instance(self): def test_device_connection_ssh_key_param(self): ckey = self._create_credentials_with_key() dc = self._create_device_connection(credentials=ckey) - self.assertIn('pkey', dc.connector_instance._params) - self.assertIsInstance(dc.connector_instance._params['pkey'], + self.assertIn('pkey', dc.connector_instance.params) + self.assertIsInstance(dc.connector_instance.params['pkey'], paramiko.rsakey.RSAKey) - self.assertNotIn('key', dc.connector_instance._params) + self.assertNotIn('key', dc.connector_instance.params) def test_ssh_connect(self): ckey = self._create_credentials_with_key(port=self.ssh_server.port) @@ -201,3 +202,38 @@ def test_device_connection_schema(self): self.assertIn('params', e.message_dict) else: self.fail('ValidationError not raised') + + def _prepare_address_list_test(self, addresses): + update_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES[0][0] + device = self._create_device(organization=self._create_org()) + dc = self._create_device_connection(device=device, + update_strategy=update_strategy) + for index, address in enumerate(addresses): + self._create_device_ip(device=device, + address=address, + priority=index + 1) + return dc + + def test_address_list(self): + dc = self._prepare_address_list_test(['10.40.0.1', '192.168.40.1']) + self.assertEqual(dc.get_addresses(), [ + '10.40.0.1', + '192.168.40.1' + ]) + + def test_address_list_with_config_last_ip(self): + dc = self._prepare_address_list_test(['192.168.40.1']) + self._create_config(device=dc.device, + last_ip='10.40.0.2') + self.assertEqual(dc.get_addresses(), [ + '192.168.40.1', + '10.40.0.2', + ]) + + def test_address_list_link_local_ip(self): + ipv6_linklocal = 'fe80::2dae:a0d4:94da:7f61' + dc = self._prepare_address_list_test([ipv6_linklocal]) + address_list = dc.get_addresses() + interfaces = get_interfaces() + self.assertEqual(len(address_list), len(interfaces)) + self.assertIn(ipv6_linklocal, address_list[0]) From 18af642618228e577938d3c807858e50381b71d8 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 20:19:12 +0200 Subject: [PATCH 04/55] [connection] Fixed migration dependency on openwisp-users --- openwisp_controller/connection/migrations/0001_initial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/migrations/0001_initial.py b/openwisp_controller/connection/migrations/0001_initial.py index 5096ec1b8..2a5f43646 100644 --- a/openwisp_controller/connection/migrations/0001_initial.py +++ b/openwisp_controller/connection/migrations/0001_initial.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): dependencies = [ ('config', '0012_auto_20180219_1501'), - ('openwisp_users', '0002_auto_20180219_1409'), + ('openwisp_users', '0001_initial'), ] operations = [ From 0dc46dd81ed2a3887f257259b79929aa4ff1396f Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 21:57:01 +0200 Subject: [PATCH 05/55] [connections] Added settings - OPENWISP_CONNECTORS - OPENWISP_UPDATE_STRATEGIES - OPENWISP_CONFIG_UPDATE_MAPPING --- README.rst | 67 ++++++++++++++++++- openwisp_controller/connection/models.py | 16 ++--- openwisp_controller/connection/settings.py | 13 ++++ .../connection/tests/test_models.py | 21 +++--- 4 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 openwisp_controller/connection/settings.py diff --git a/README.rst b/README.rst index 1a5baa474..90d64d8cf 100755 --- a/README.rst +++ b/README.rst @@ -188,6 +188,65 @@ Add the following settings to ``settings.py``: urlpatterns += staticfiles_urlpatterns() +Settings +-------- + +``OPENWISP_CONNECTORS`` +~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+--------------------------------------------------------------------+ +| **type**: | ``tuple`` | ++--------------+--------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | ( | +| | ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), | +| | ) | ++--------------+--------------------------------------------------------------------+ + +Available connector classes. Connectors are python classes that specify ways +in which OpenWISP can connect to devices in order to launch commands. + +``OPENWISP_UPDATE_STRATEGIES`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+----------------------------------------------------------------------------------------+ +| **type**: | ``tuple`` | ++--------------+----------------------------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | ( | +| | ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), | +| | ) | ++--------------+----------------------------------------------------------------------------------------+ + +Available update strategies. An update strategy is a subclass of a +connector class which defines an ``update_config`` method which is +in charge of updating the configuratio of the device. + +This operation is launched in a background worker when the configuration +of a device is changed. + +It's possible to write custom update strategies and add them to this +setting to make them available in OpenWISP. + +``OPENWISP_CONFIG_UPDATE_MAPPING`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+--------------------------------------------------------------------+ +| **type**: | ``dict`` | ++--------------+--------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | { | +| | 'netjsonconfig.OpenWrt': OPENWISP_UPDATE_STRATEGIES[0][0], | +| | } | ++--------------+--------------------------------------------------------------------+ + +A dictionary that maps configuration backends to update strategies in order to +automatically determine the update strategy of a device connection if the +update strategy field is left blank by the user. + Installing for development -------------------------- @@ -209,12 +268,18 @@ Install your forked repo with `pipenv Create database: -.. code-block:: shell +.. code-block:: shell cd tests/ pipenv run ./manage.py migrate pipenv run ./manage.py createsuperuser +Launch celery worker (for background jobs): + +.. code-block:: shell + + celery -A openwisp2 worker -l info + Launch development server: .. code-block:: shell diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 82d4ac139..e77c3dcc7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -16,6 +16,7 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel +from . import settings as app_settings from .utils import get_interfaces logger = logging.getLogger(__name__) @@ -53,11 +54,8 @@ class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): """ Credentials for access """ - CONNECTOR_CHOICES = ( - ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), - ) connector = models.CharField(_('connection type'), - choices=CONNECTOR_CHOICES, + choices=app_settings.CONNECTORS, max_length=128, db_index=True) params = JSONField(_('parameters'), @@ -77,17 +75,11 @@ def __str__(self): @python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' - UPDATE_STRATEGY_CHOICES = ( - ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), - ) - CONFIG_BACKEND_MAPPING = { - 'netjsonconfig.OpenWrt': UPDATE_STRATEGY_CHOICES[0][0], - } device = models.ForeignKey('config.Device', on_delete=models.CASCADE) credentials = models.ForeignKey(Credentials, on_delete=models.CASCADE) update_strategy = models.CharField(_('update strategy'), help_text=_('leave blank to determine automatically'), - choices=UPDATE_STRATEGY_CHOICES, + choices=app_settings.UPDATE_STRATEGIES, max_length=128, blank=True, db_index=True) @@ -113,7 +105,7 @@ class Meta: def clean(self): if not self.update_strategy and hasattr(self.device, 'config'): try: - self.update_strategy = self.CONFIG_BACKEND_MAPPING[self.device.config.backend] + self.update_strategy = app_settings.CONFIG_UPDATE_MAPPING[self.device.config.backend] except KeyError as e: raise ValidationError({ 'update_stragy': _('could not determine update strategy ' diff --git a/openwisp_controller/connection/settings.py b/openwisp_controller/connection/settings.py new file mode 100644 index 000000000..0af20ada3 --- /dev/null +++ b/openwisp_controller/connection/settings.py @@ -0,0 +1,13 @@ +from django.conf import settings + +CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', ( + ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), +)) + +UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', ( + ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), +)) + +CONFIG_UPDATE_MAPPING = getattr(settings, 'OPENWISP_CONFIG_UPDATE_MAPPING', { + 'netjsonconfig.OpenWrt': UPDATE_STRATEGIES[0][0], +}) diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 42ae72145..4f8ec61c2 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -10,6 +10,7 @@ from openwisp_users.tests.utils import TestOrganizationMixin +from .. import settings as app_settings from ..models import Credentials, DeviceConnection, DeviceIp from ..utils import get_interfaces @@ -35,7 +36,7 @@ def tearDownClass(cls): def _create_credentials(self, **kwargs): opts = dict(name='Test credentials', - connector=Credentials.CONNECTOR_CHOICES[0][0], + connector=app_settings.CONNECTORS[0][0], params={'username': 'root', 'password': 'password', 'port': 22}) @@ -84,7 +85,7 @@ def _create_device_ip(self, **kwargs): class TestModels(TestConnectionMixin, TestCase): def test_connection_str(self): - c = Credentials(name='Dev Key', connector=Credentials.CONNECTOR_CHOICES[0][0]) + c = Credentials(name='Dev Key', connector=app_settings.CONNECTORS[0][0]) self.assertIn(c.name, str(c)) self.assertIn(c.get_connector_display(), str(c)) @@ -101,13 +102,13 @@ def test_device_connection_get_params(self): def test_device_connection_auto_update_strategy(self): dc = self._create_device_connection() - self.assertEqual(dc.update_strategy, dc.UPDATE_STRATEGY_CHOICES[0][0]) + self.assertEqual(dc.update_strategy, app_settings.UPDATE_STRATEGIES[0][0]) def test_device_connection_auto_update_strategy_key_error(self): - orig_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES - orig_mapping = DeviceConnection.CONFIG_BACKEND_MAPPING - DeviceConnection.UPDATE_STRATEGY_CHOICES = (('meddle', 'meddle'),) - DeviceConnection.CONFIG_BACKEND_MAPPING = {'wrong': 'wrong'} + orig_strategy = app_settings.UPDATE_STRATEGIES + orig_mapping = app_settings.CONFIG_UPDATE_MAPPING + app_settings.UPDATE_STRATEGIES = (('meddle', 'meddle'),) + app_settings.CONFIG_UPDATE_MAPPING = {'wrong': 'wrong'} try: self._create_device_connection() except ValidationError: @@ -115,8 +116,8 @@ def test_device_connection_auto_update_strategy_key_error(self): else: failed = True # restore - DeviceConnection.UPDATE_STRATEGY_CHOICES = orig_strategy - DeviceConnection.CONFIG_BACKEND_MAPPING = orig_mapping + app_settings.UPDATE_STRATEGIES = orig_strategy + app_settings.CONFIG_UPDATE_MAPPING = orig_mapping if failed: self.fail('ValidationError not raised') @@ -204,7 +205,7 @@ def test_device_connection_schema(self): self.fail('ValidationError not raised') def _prepare_address_list_test(self, addresses): - update_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES[0][0] + update_strategy = app_settings.UPDATE_STRATEGIES[0][0] device = self._create_device(organization=self._create_org()) dc = self._create_device_connection(device=device, update_strategy=update_strategy) From 704b209075ada3edcc1d81449c187538a03820df Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 22:29:10 +0200 Subject: [PATCH 06/55] [connections] Updated runtests.py to use new setting file --- runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.py b/runtests.py index 6dd62fa93..84dd794f0 100755 --- a/runtests.py +++ b/runtests.py @@ -5,7 +5,7 @@ import sys sys.path.insert(0, "tests") -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp2.settings") if __name__ == "__main__": from django.core.management import execute_from_command_line From 990b6678f87fb1ea4723333dd13b937b5beb29ac Mon Sep 17 00:00:00 2001 From: Oliver Kraitschy Date: Wed, 9 May 2018 07:32:48 +0200 Subject: [PATCH 07/55] Fix markdown in README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 90d64d8cf..9bf11ce0c 100755 --- a/README.rst +++ b/README.rst @@ -268,8 +268,8 @@ Install your forked repo with `pipenv Create database: - .. code-block:: shell + cd tests/ pipenv run ./manage.py migrate pipenv run ./manage.py createsuperuser From 96fe4502bcc56a37421a08945a9a62f7ae61b2e0 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 14:40:28 +0200 Subject: [PATCH 08/55] [connection] Made config_modified_receiver more resilient to exceptions --- openwisp_controller/connection/apps.py | 15 +++++++++++---- openwisp_controller/connection/tasks.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 1f17d9b77..1a4e84c45 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -20,17 +20,24 @@ def ready(self): from .tasks import update_config def config_modified_receiver(**kwargs): - device_id = kwargs['device'].id - if not self._is_update_in_progress(device_id): - update_config.delay(device_id) + d = kwargs['device'] + conn_count = d.deviceconnection_set.count() + # if device has no connection specified + # or update is already in progress, stop here + if conn_count < 1 or self._is_update_in_progress(d.id): + return + update_config.delay(d.id) config_modified.connect(config_modified_receiver, dispatch_uid='connection.update_config', weak=False) def _is_update_in_progress(self, device_id): + active = inspect().active() + if not active: + return False # check if there's any other running task before adding it - for task_list in inspect().active().values(): + for task_list in active.values(): for task in task_list: if task['name'] == _TASK_NAME and str(device_id) in task['args']: return True diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py index d02727dd3..689e41570 100644 --- a/openwisp_controller/connection/tasks.py +++ b/openwisp_controller/connection/tasks.py @@ -21,6 +21,6 @@ def update_config(device_id): if device.config.status == 'running': return qs = device.deviceconnection_set.filter(device_id=device_id, enabled=True) - if qs.count() > 0: - conn = qs.first() + conn = qs.first() + if conn: conn.update_config() From 4bb7d2d1b3ed07b50c8b9bc64f43b33642df034f Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 14:40:39 +0200 Subject: [PATCH 09/55] [connection] Updated MEDIA_ROOT --- tests/openwisp2/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 55c4cd099..e81727a00 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -90,7 +90,7 @@ USE_L10N = False STATIC_URL = '/static/' MEDIA_URL = '/media/' -MEDIA_ROOT = '{0}/media/'.format(BASE_DIR) +MEDIA_ROOT = '{0}/media/'.format(os.path.dirname(BASE_DIR)) TEMPLATES = [ { From b8a1e2a57e9e5914595d58a43dcfba8c5cdfa97c Mon Sep 17 00:00:00 2001 From: Oliver Kraitschy Date: Wed, 9 May 2018 15:05:52 +0200 Subject: [PATCH 10/55] [connections] Update dependencies in README.rst and add setting for mod_spatialite --- README.rst | 7 ++++--- tests/openwisp2/settings.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 9bf11ce0c..3b281d007 100755 --- a/README.rst +++ b/README.rst @@ -250,12 +250,13 @@ update strategy field is left blank by the user. Installing for development -------------------------- -Install sqlite: +Install the dependencies: .. code-block:: shell - sudo apt-get install sqlite3 libsqlite3-dev libsqlite3-mod-spatialite openssl libssl-dev - sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev + sudo apt-get install sqlite3 libsqlite3-dev openssl libssl-dev + sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev libsqlite3-mod-spatialite + sudo apt-get install redis Install your forked repo with `pipenv `_: diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index e81727a00..3cae81556 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -129,6 +129,8 @@ CELERY_TASK_EAGER_PROPAGATES = True CELERY_BROKER_URL = 'memory://' +SPATIALITE_LIBRARY_PATH = 'mod_spatialite' + LOGGING = { 'version': 1, 'filters': { From 7f3b651818ebc41e0cc8dd4ea3b92248ea9f5614 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 17:08:36 +0200 Subject: [PATCH 11/55] [connection] Fixed admin tests Added missing inline form parameters --- Pipfile | 1 + openwisp_controller/config/tests/test_admin.py | 9 +++++++++ openwisp_controller/connection/connectors/ssh.py | 5 +++-- openwisp_controller/connection/models.py | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index af36b47f4..59d8ca696 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ coverage = "*" coveralls = "*" isort = "*" flake8 = "*" +mock-ssh-server = {version = ">=0.5.0,<0.6.0"} [scripts] lint = "python -m flake8" diff --git a/openwisp_controller/config/tests/test_admin.py b/openwisp_controller/config/tests/test_admin.py index b61a0b3d7..a43800fca 100644 --- a/openwisp_controller/config/tests/test_admin.py +++ b/openwisp_controller/config/tests/test_admin.py @@ -41,6 +41,15 @@ class TestAdmin(CreateConfigTemplateMixin, TestAdminMixin, 'config-INITIAL_FORMS': 0, 'config-MIN_NUM_FORMS': 0, 'config-MAX_NUM_FORMS': 1, + # openwisp_controller.connection + 'deviceconnection_set-TOTAL_FORMS': 0, + 'deviceconnection_set-INITIAL_FORMS': 0, + 'deviceconnection_set-MIN_NUM_FORMS': 0, + 'deviceconnection_set-MAX_NUM_FORMS': 1000, + 'deviceip_set-TOTAL_FORMS': 0, + 'deviceip_set-INITIAL_FORMS': 0, + 'deviceip_set-MIN_NUM_FORMS': 0, + 'deviceip_set-MAX_NUM_FORMS': 1000, } # WARNING - WATCHOUT # this class attribute is changed dinamically diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 34ee18d2e..3174ee738 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,4 +1,5 @@ import logging +import sys import paramiko from django.utils.functional import cached_property @@ -6,9 +7,9 @@ from jsonschema.exceptions import ValidationError as SchemaError from scp import SCPClient -try: +if sys.version_info.major > 2: # pragma: nocover from io import StringIO -except ImportError: +else: # pragma: nocover from StringIO import StringIO diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index e77c3dcc7..c07ec3364 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -72,7 +72,6 @@ def __str__(self): return '{0} ({1})'.format(self.name, self.get_connector_display()) -@python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' device = models.ForeignKey('config.Device', on_delete=models.CASCADE) From 6c5964159c807c71b5d710bce6df2c58e324d553 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 17:52:25 +0200 Subject: [PATCH 12/55] [test] Removed `SPATIALITE_LIBRARY_PATH = 'mod_spatialite` Breaks the travis build. --- tests/openwisp2/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 3cae81556..e81727a00 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -129,8 +129,6 @@ CELERY_TASK_EAGER_PROPAGATES = True CELERY_BROKER_URL = 'memory://' -SPATIALITE_LIBRARY_PATH = 'mod_spatialite' - LOGGING = { 'version': 1, 'filters': { From 00493159808e52423c4851ba6ac4004b2340507d Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:32:20 +0200 Subject: [PATCH 13/55] [connection] Added exec_command method --- .../connection/connectors/openwrt/ssh.py | 2 +- .../connection/connectors/ssh.py | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/openwisp_controller/connection/connectors/openwrt/ssh.py b/openwisp_controller/connection/connectors/openwrt/ssh.py index ea646afde..1874e6f9e 100644 --- a/openwisp_controller/connection/connectors/openwrt/ssh.py +++ b/openwisp_controller/connection/connectors/openwrt/ssh.py @@ -3,4 +3,4 @@ class OpenWrt(Ssh): def update_config(self): - self.shell.exec_command('/etc/init.d/openwisp_config restart') + self.exec_command('/etc/init.d/openwisp_config restart') diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 3174ee738..c2f59d7c0 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,5 +1,7 @@ import logging +import socket import sys +from io import BytesIO import paramiko from django.utils.functional import cached_property @@ -16,6 +18,7 @@ logger = logging.getLogger(__name__) SSH_CONNECTION_TIMEOUT = 5 SSH_AUTH_TIMEOUT = 2 +SSH_COMMAND_TIMEOUT = 5 class Ssh(object): @@ -76,10 +79,54 @@ def connect(self): def disconnect(self): self.shell.close() + def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, + exit_codes=[0], raise_unexpected_exit=True): + """ + Executes a command and performs the following operations + - logs executed command + - logs standard output + - logs standard error + - aborts on exceptions + - raises socket.timeout exceptions + """ + print('$:> {0}'.format(command)) + # execute commmand + try: + stdin, stdout, stderr = self.shell.exec_command(command) + # re-raise socket.timeout to avoid being catched + # by the subsequent `except Exception as e` block + except socket.timeout: + raise socket.timeout() + # any other exception will abort the operation + except Exception as e: + logger.exception(e) + raise e + # store command exit status + exit_status = stdout.channel.recv_exit_status() + # log standard output + output = stdout.read().decode('utf8').strip() + if output: + print(output) + # log standard error + error = stderr.read().decode('utf8').strip() + if error: + print(error) + # abort the operation if any of the command + # returned with a non-zero exit status + if exit_status not in exit_codes and raise_unexpected_exit: + print('# Previus command failed, aborting upgrade...') + message = error if error else output + raise Exception(message) + return output, exit_status + def update_config(self): raise NotImplementedError() def upload(self, fl, remote_path): scp = SCPClient(self.shell.get_transport()) + if not hasattr(fl, 'getvalue'): + fl_memory = BytesIO(fl.read()) + fl.seek(0) + fl = fl_memory scp.putfo(fl, remote_path) scp.close() From 4535ba22f6749606a5a45f91cfef23f1404897b7 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:32:45 +0200 Subject: [PATCH 14/55] [connection] Close connection after update_config --- openwisp_controller/connection/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index c07ec3364..d665398ce 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -172,6 +172,7 @@ def update_config(self): logger.exception(e) else: self.device.config.set_status_running() + self.disconnect() @python_2_unicode_compatible From e3d105cb9a35bd813a9c73b047c4a9a5a6c09b74 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:33:20 +0200 Subject: [PATCH 15/55] [connection] Added default settings --- openwisp_controller/connection/settings.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/openwisp_controller/connection/settings.py b/openwisp_controller/connection/settings.py index 0af20ada3..5b4f159b4 100644 --- a/openwisp_controller/connection/settings.py +++ b/openwisp_controller/connection/settings.py @@ -1,13 +1,17 @@ from django.conf import settings -CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', ( +DEFAULT_CONNECTORS = ( ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), -)) +) -UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', ( +CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', DEFAULT_CONNECTORS) + +DEFAULT_UPDATE_STRATEGIES = ( ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), -)) +) + +UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', DEFAULT_UPDATE_STRATEGIES) CONFIG_UPDATE_MAPPING = getattr(settings, 'OPENWISP_CONFIG_UPDATE_MAPPING', { - 'netjsonconfig.OpenWrt': UPDATE_STRATEGIES[0][0], + 'netjsonconfig.OpenWrt': DEFAULT_UPDATE_STRATEGIES[0][0], }) From 43041d75f1bcdb3444d81181a74a364c19d4182e Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 25 May 2018 12:55:43 +0300 Subject: [PATCH 16/55] [connection] Made test mixins reusable (for extensions) --- openwisp_controller/connection/tests/base.py | 83 +++++++++++++++++++ .../connection/tests/test_models.py | 83 +------------------ 2 files changed, 87 insertions(+), 79 deletions(-) create mode 100644 openwisp_controller/connection/tests/base.py diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py new file mode 100644 index 000000000..a57464e4f --- /dev/null +++ b/openwisp_controller/connection/tests/base.py @@ -0,0 +1,83 @@ +import os + +from django.conf import settings +from mockssh import Server as SshServer +from openwisp_controller.config.models import Config, Device +from openwisp_controller.config.tests import CreateConfigTemplateMixin + +from openwisp_users.tests.utils import TestOrganizationMixin + +from .. import settings as app_settings +from ..models import Credentials, DeviceConnection, DeviceIp + + +class CreateConnectionsMixin(CreateConfigTemplateMixin, TestOrganizationMixin): + device_model = Device + config_model = Config + + def _create_credentials(self, **kwargs): + opts = dict(name='Test credentials', + connector=app_settings.CONNECTORS[0][0], + params={'username': 'root', + 'password': 'password', + 'port': 22}) + opts.update(kwargs) + if 'organization' not in opts: + opts['organization'] = self._create_org() + c = Credentials(**opts) + c.full_clean() + c.save() + return c + + def _create_credentials_with_key(self, username='root', port=22, **kwargs): + opts = dict(name='Test SSH Key', + params={'username': username, + 'key': self._SSH_PRIVATE_KEY, + 'port': port}) + return self._create_credentials(**opts) + + def _create_device_connection(self, **kwargs): + opts = dict(enabled=True, + params={}) + opts.update(kwargs) + if 'credentials' not in opts: + opts['credentials'] = self._create_credentials() + org = opts['credentials'].organization + if 'device' not in opts: + opts['device'] = self._create_device(organization=org) + self._create_config(device=opts['device']) + dc = DeviceConnection(**opts) + dc.full_clean() + dc.save() + return dc + + def _create_device_ip(self, **kwargs): + opts = dict(address='10.40.0.1', + priority=1) + opts.update(kwargs) + if 'device' not in opts: + dc = self._create_device_connection() + opts['device'] = dc.device + ip = DeviceIp(**opts) + ip.full_clean() + ip.save() + return ip + + +class SshServerMixin(object): + _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') + _SSH_PRIVATE_KEY = None + + @classmethod + def setUpClass(cls): + with open(cls._TEST_RSA_KEY_PATH, 'r') as f: + cls._SSH_PRIVATE_KEY = f.read() + cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) + cls.ssh_server.__enter__() + + @classmethod + def tearDownClass(cls): + try: + cls.ssh_server.__exit__() + except OSError: + pass diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 4f8ec61c2..b9d0acc72 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -1,89 +1,14 @@ -import os - import paramiko -from django.conf import settings from django.core.exceptions import ValidationError from django.test import TestCase -from mockssh import Server as SshServer -from openwisp_controller.config.models import Config, Device -from openwisp_controller.config.tests import CreateConfigTemplateMixin - -from openwisp_users.tests.utils import TestOrganizationMixin from .. import settings as app_settings -from ..models import Credentials, DeviceConnection, DeviceIp +from ..models import Credentials, DeviceIp from ..utils import get_interfaces +from .base import CreateConnectionsMixin, SshServerMixin -class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): - device_model = Device - config_model = Config - _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') - with open(_TEST_RSA_KEY_PATH, 'r') as f: - _SSH_PRIVATE_KEY = f.read() - - @classmethod - def setUpClass(cls): - cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) - cls.ssh_server.__enter__() - - @classmethod - def tearDownClass(cls): - try: - cls.ssh_server.__exit__() - except OSError: - pass - - def _create_credentials(self, **kwargs): - opts = dict(name='Test credentials', - connector=app_settings.CONNECTORS[0][0], - params={'username': 'root', - 'password': 'password', - 'port': 22}) - opts.update(kwargs) - if 'organization' not in opts: - opts['organization'] = self._create_org() - c = Credentials(**opts) - c.full_clean() - c.save() - return c - - def _create_credentials_with_key(self, username='root', port=22, **kwargs): - opts = dict(name='Test SSH Key', - params={'username': username, - 'key': self._SSH_PRIVATE_KEY, - 'port': port}) - return self._create_credentials(**opts) - - def _create_device_connection(self, **kwargs): - opts = dict(enabled=True, - params={}) - opts.update(kwargs) - if 'credentials' not in opts: - opts['credentials'] = self._create_credentials() - org = opts['credentials'].organization - if 'device' not in opts: - opts['device'] = self._create_device(organization=org) - self._create_config(device=opts['device']) - dc = DeviceConnection(**opts) - dc.full_clean() - dc.save() - return dc - - def _create_device_ip(self, **kwargs): - opts = dict(address='10.40.0.1', - priority=1) - opts.update(kwargs) - if 'device' not in opts: - dc = self._create_device_connection() - opts['device'] = dc.device - ip = DeviceIp(**opts) - ip.full_clean() - ip.save() - return ip - - -class TestModels(TestConnectionMixin, TestCase): +class TestModels(SshServerMixin, CreateConnectionsMixin, TestCase): def test_connection_str(self): c = Credentials(name='Dev Key', connector=app_settings.CONNECTORS[0][0]) self.assertIn(c.name, str(c)) @@ -127,7 +52,7 @@ def test_device_connection_auto_update_strategy_missing_config(self): try: self._create_device_connection(device=device) except ValidationError as e: - self.assertIn('inferred from', str(e)) + self.assertIn('inferred from', str(e)) else: self.fail('ValidationError not raised') From 5124449b2bc89d4e00a54b70cb06c40a30c59828 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 15 Jun 2018 08:40:53 +0200 Subject: [PATCH 17/55] [connection] Enforce multitenancy in DeviceConnection.credentials --- openwisp_controller/connection/admin.py | 10 +++++++++- openwisp_controller/connection/models.py | 6 ++++++ openwisp_controller/connection/tests/base.py | 5 ++++- .../connection/tests/test_models.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py index 9175fca03..b8c29f34a 100644 --- a/openwisp_controller/connection/admin.py +++ b/openwisp_controller/connection/admin.py @@ -25,11 +25,19 @@ def get_queryset(self, request): return qs.order_by('priority') -class DeviceConnectionInline(admin.StackedInline): +class DeviceConnectionInline(MultitenantAdminMixin, admin.StackedInline): model = DeviceConnection exclude = ['params', 'created', 'modified'] readonly_fields = ['is_working', 'failure_reason', 'last_attempt'] extra = 0 + multitenant_shared_relations = ('credentials',) + + def get_queryset(self, request): + """ + Override MultitenantAdminMixin.get_queryset() because it breaks + """ + return super(admin.StackedInline, self).get_queryset(request) + DeviceAdmin.inlines += [DeviceConnectionInline, DeviceIpInline] diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index d665398ce..39b4de6e7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -102,6 +102,12 @@ class Meta: verbose_name_plural = _('Device connections') def clean(self): + cred_org = self.credentials.organization + if cred_org and cred_org != self.device.organization: + raise ValidationError({ + 'credentials': _('The organization of these credentials doesn\'t ' + 'match the organization of the device') + }) if not self.update_strategy and hasattr(self.device, 'config'): try: self.update_strategy = app_settings.CONFIG_UPDATE_MAPPING[self.device.config.backend] diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py index a57464e4f..5b254a9e8 100644 --- a/openwisp_controller/connection/tests/base.py +++ b/openwisp_controller/connection/tests/base.py @@ -41,7 +41,10 @@ def _create_device_connection(self, **kwargs): params={}) opts.update(kwargs) if 'credentials' not in opts: - opts['credentials'] = self._create_credentials() + cred_opts = {} + if 'device' in opts: + cred_opts = {'organization': opts['device'].organization} + opts['credentials'] = self._create_credentials(**cred_opts) org = opts['credentials'].organization if 'device' not in opts: opts['device'] = self._create_device(organization=org) diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index b9d0acc72..f52b9060a 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -163,3 +163,21 @@ def test_address_list_link_local_ip(self): interfaces = get_interfaces() self.assertEqual(len(address_list), len(interfaces)) self.assertIn(ipv6_linklocal, address_list[0]) + + def test_device_connection_credential_org_validation(self): + dc = self._create_device_connection() + shared = self._create_credentials(name='cred-shared', + organization=None) + dc.credentials = shared + dc.full_clean() + # ensure credentials of other orgs aren't accepted + org2 = self._create_org(name='org2') + cred2 = self._create_credentials(name='cred2', + organization=org2) + try: + dc.credentials = cred2 + dc.full_clean() + except ValidationError as e: + self.assertIn('credentials', e.message_dict) + else: + self.fail('ValidationError not raised') From 1e7ecb7aac5a17be527bbdabc13e3c5164069aa9 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 2 Dec 2018 19:23:12 +0100 Subject: [PATCH 18/55] [connections] Added support for management interface --- openwisp_controller/config/views.py | 2 +- openwisp_controller/connection/models.py | 10 ++++----- .../connection/tests/test_models.py | 21 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/openwisp_controller/config/views.py b/openwisp_controller/config/views.py index a63b8cec9..9cabf590f 100644 --- a/openwisp_controller/config/views.py +++ b/openwisp_controller/config/views.py @@ -13,7 +13,7 @@ def get_default_templates(request, organization_id): """ user = request.user authenticated = user.is_authenticated - if callable(authenticated): + if callable(authenticated): # pragma: nocover authenticated = authenticated() if not authenticated and not user.is_staff: return HttpResponse(status=403) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 39b4de6e7..6041ca0b7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -2,7 +2,7 @@ import ipaddress import logging -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible @@ -142,10 +142,10 @@ def get_addresses(self): else: for interface in get_interfaces(): address_list.append('{0}%{1}'.format(address, interface)) - try: - address_list.append(self.device.config.last_ip) - except ObjectDoesNotExist: - pass + if self.device.management_ip: + address_list.append(self.device.management_ip) + if self.device.last_ip: + address_list.append(self.device.last_ip) return address_list def get_params(self): diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index f52b9060a..63c760b1a 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -129,9 +129,13 @@ def test_device_connection_schema(self): else: self.fail('ValidationError not raised') - def _prepare_address_list_test(self, addresses): + def _prepare_address_list_test(self, addresses, + last_ip=None, + management_ip=None): update_strategy = app_settings.UPDATE_STRATEGIES[0][0] - device = self._create_device(organization=self._create_org()) + device = self._create_device(organization=self._create_org(), + last_ip=last_ip, + management_ip=management_ip) dc = self._create_device_connection(device=device, update_strategy=update_strategy) for index, address in enumerate(addresses): @@ -147,13 +151,16 @@ def test_address_list(self): '192.168.40.1' ]) - def test_address_list_with_config_last_ip(self): - dc = self._prepare_address_list_test(['192.168.40.1']) - self._create_config(device=dc.device, - last_ip='10.40.0.2') + def test_address_list_with_device_ip(self): + dc = self._prepare_address_list_test( + ['192.168.40.1'], + management_ip='10.0.0.2', + last_ip='84.32.46.153', + ) self.assertEqual(dc.get_addresses(), [ '192.168.40.1', - '10.40.0.2', + '10.0.0.2', + '84.32.46.153' ]) def test_address_list_link_local_ip(self): From 5663926d809eaae79c58b00cf64950b47abbd5e9 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 2 Dec 2018 19:31:53 +0100 Subject: [PATCH 19/55] [connections] Updated deprecated set_status_running to set_status_applied --- openwisp_controller/connection/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 6041ca0b7..872524592 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -177,7 +177,7 @@ def update_config(self): except Exception as e: logger.exception(e) else: - self.device.config.set_status_running() + self.device.config.set_status_applied() self.disconnect() From 32f5a54c20ea6fe7389e7ea54d50d78550d512ac Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 6 Dec 2018 14:33:54 +0100 Subject: [PATCH 20/55] [connection] Added auto_add feature to Credentials --- openwisp_controller/connection/admin.py | 10 +++- openwisp_controller/connection/apps.py | 34 +++++++---- .../migrations/0002_credentials_auto_add.py | 18 ++++++ openwisp_controller/connection/models.py | 60 +++++++++++++++++++ .../connection/tests/test_models.py | 60 +++++++++++++++++++ 5 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 openwisp_controller/connection/migrations/0002_credentials_auto_add.py diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py index b8c29f34a..41fbd3254 100644 --- a/openwisp_controller/connection/admin.py +++ b/openwisp_controller/connection/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from openwisp_utils.admin import MultitenantOrgFilter, TimeReadonlyAdminMixin +from openwisp_users.multitenancy import MultitenantOrgFilter +from openwisp_utils.admin import TimeReadonlyAdminMixin from ..admin import MultitenantAdminMixin from ..config.admin import DeviceAdmin @@ -9,7 +10,12 @@ @admin.register(Credentials) class CredentialsAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin): - list_display = ('name', 'organization', 'connector', 'created', 'modified') + list_display = ('name', + 'organization', + 'connector', + 'auto_add', + 'created', + 'modified') list_filter = [('organization', MultitenantOrgFilter), 'connector'] list_select_related = ('organization',) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 1a4e84c45..6dc7b9366 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,5 +1,6 @@ from celery.task.control import inspect from django.apps import AppConfig +from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified @@ -17,22 +18,29 @@ def ready(self): to the ``update_config`` celery task which will be executed in the background """ - from .tasks import update_config + config_modified.connect(self.config_modified_receiver, + dispatch_uid='connection.update_config') + + from ..config.models import Config + from .models import Credentials - def config_modified_receiver(**kwargs): - d = kwargs['device'] - conn_count = d.deviceconnection_set.count() - # if device has no connection specified - # or update is already in progress, stop here - if conn_count < 1 or self._is_update_in_progress(d.id): - return - update_config.delay(d.id) + post_save.connect(Credentials.auto_add_credentials_to_device, + sender=Config, + dispatch_uid='connection.auto_add_credentials') - config_modified.connect(config_modified_receiver, - dispatch_uid='connection.update_config', - weak=False) + @classmethod + def config_modified_receiver(cls, **kwargs): + from .tasks import update_config + d = kwargs['device'] + conn_count = d.deviceconnection_set.count() + # if device has no connection specified + # or update is already in progress, stop here + if conn_count < 1 or cls._is_update_in_progress(d.id): + return + update_config.delay(d.id) - def _is_update_in_progress(self, device_id): + @classmethod + def _is_update_in_progress(cls, device_id): active = inspect().active() if not active: return False diff --git a/openwisp_controller/connection/migrations/0002_credentials_auto_add.py b/openwisp_controller/connection/migrations/0002_credentials_auto_add.py new file mode 100644 index 000000000..d7373f79e --- /dev/null +++ b/openwisp_controller/connection/migrations/0002_credentials_auto_add.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.3 on 2018-12-05 13:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('connection', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='credentials', + name='auto_add', + field=models.BooleanField(default=False, help_text='automatically add these credentials to the devices of this organization; if no organization is specified will be added to all the new devices', verbose_name='auto add'), + ), + ] diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 872524592..9f48fa062 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -17,6 +17,7 @@ from openwisp_utils.base import TimeStampedEditableModel from . import settings as app_settings +from ..config.models import Device from .utils import get_interfaces logger = logging.getLogger(__name__) @@ -63,6 +64,12 @@ class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): help_text=_('global connection parameters'), load_kwargs={'object_pairs_hook': collections.OrderedDict}, dump_kwargs={'indent': 4}) + auto_add = models.BooleanField(_('auto add'), + default=False, + help_text=_('automatically add these credentials ' + 'to the devices of this organization; ' + 'if no organization is specified will ' + 'be added to all the new devices')) class Meta: verbose_name = _('Access credentials') @@ -71,6 +78,59 @@ class Meta: def __str__(self): return '{0} ({1})'.format(self.name, self.get_connector_display()) + def save(self, *args, **kwargs): + super(Credentials, self).save(*args, **kwargs) + self.auto_add_to_devices() + + def auto_add_to_devices(self): + """ + When ``auto_add`` is ``True``, adds the credentials + to each relevant ``Device`` and ``DeviceConnection`` objects + """ + if not self.auto_add: + return + devices = Device.objects.all() + org = self.organization + if org: + devices = devices.filter(organization=org) + # exclude devices which have been already added + devices = devices.exclude(deviceconnection__credentials=self) + for device in devices: + conn = DeviceConnection(device=device, + credentials=self, + enabled=True) + conn.full_clean() + conn.save() + + @classmethod + def auto_add_credentials_to_device(cls, instance, created, **kwargs): + """ + Adds relevant credentials as ``DeviceConnection`` + when a device is created, this is called from a + post_save signal receiver hooked to the ``Config`` model + (why ``Config`` and not ``Device``? because at the moment + we can automatically create a DeviceConnection if we have + a ``Config`` object) + """ + if not created: + return + device = instance.device + # select credentials which + # - are flagged as auto_add + # - belong to the same organization of the device + # OR + # belong to no organization (hence are shared) + conditions = (models.Q(organization=device.organization) | + models.Q(organization=None)) + credentials = cls.objects.filter(conditions) \ + .filter(auto_add=True) + for cred in credentials: + conn = DeviceConnection(device=device, + credentials=cred, + enabled=True) + conn.full_clean() + conn.save() + class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 63c760b1a..e10d137ba 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -2,6 +2,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase +from openwisp_users.models import Organization + from .. import settings as app_settings from ..models import Credentials, DeviceIp from ..utils import get_interfaces @@ -188,3 +190,61 @@ def test_device_connection_credential_org_validation(self): self.assertIn('credentials', e.message_dict) else: self.fail('ValidationError not raised') + + def test_auto_add_to_new_device(self): + c = self._create_credentials(auto_add=True, + organization=None) + self._create_credentials(name='cred2', + auto_add=False, + organization=None) + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + + def test_auto_add_to_existing_device_on_creation(self): + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + self.assertEqual(d.deviceconnection_set.count(), 0) + c = self._create_credentials(auto_add=True, + organization=None) + org2 = Organization.objects.create(name='org2', slug='org2') + self._create_credentials(name='cred2', + auto_add=True, + organization=org2) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + self._create_credentials(name='cred3', + auto_add=False, + organization=None) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + + def test_auto_add_to_existing_device_on_edit(self): + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + self.assertEqual(d.deviceconnection_set.count(), 0) + c = self._create_credentials(auto_add=False, + organization=None) + org2 = Organization.objects.create(name='org2', slug='org2') + self._create_credentials(name='cred2', + auto_add=True, + organization=org2) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 0) + c.auto_add = True + c.full_clean() + c.save() + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + # ensure further edits are idempotent + c.name = 'changed' + c.full_clean() + c.save() + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) From 96f2ea3147516ee76a297419c8f4a593eb7aa4ed Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sat, 8 Dec 2018 18:13:10 +0100 Subject: [PATCH 21/55] [connection] Upgraded celery to 4.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bee535c7b..c2dbe51bb 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ # TODO: make optional? "paramiko>=2.4.1,<2.5.0", "scp>=0.13.0,<0.14.0", - "celery>=4.1.0,<4.2.0" + "celery>=4.2.0,<4.3.0" ], classifiers=[ 'Development Status :: 3 - Alpha', From a2590ba5fed76fe73302540cfae6f9a8fd7b9752 Mon Sep 17 00:00:00 2001 From: Noumbissi valere Gille Geovan Date: Wed, 3 Apr 2019 10:19:02 +0100 Subject: [PATCH 22/55] [test] Resolved Conflict --- openwisp_controller/config/tests/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openwisp_controller/config/tests/__init__.py b/openwisp_controller/config/tests/__init__.py index cba2205f2..b28e29812 100644 --- a/openwisp_controller/config/tests/__init__.py +++ b/openwisp_controller/config/tests/__init__.py @@ -18,7 +18,5 @@ class CreateConfigTemplateMixin(CreateTemplateMixin, CreateConfigMixin): def _create_config(self, **kwargs): if 'device' not in kwargs: kwargs['device'] = self._create_device(name='test-device', - organization=kwargs.get('organization', None)) - if 'organization' not in kwargs: - kwargs['organization'] = kwargs['device'].organization + organization=kwargs.pop('organization')) return super(CreateConfigTemplateMixin, self)._create_config(**kwargs) From a7258d25ae3c88e65715dba10668b8670b5e6597 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 6 May 2018 22:22:13 +0200 Subject: [PATCH 23/55] [connection] Init new connection module Requires django-netjsonconfig 0.8.1 Allows to - connect to devices using different protocols - SSH protocol supported by default - centrally launch update config operations when config/templates are changed Signed-off-by: Federico Capoano --- openwisp_controller/connection/__init__.py | 1 + openwisp_controller/connection/admin.py | 35 +++ openwisp_controller/connection/apps.py | 14 ++ .../connection/connectors/__init__.py | 0 .../connection/connectors/openwrt/__init__.py | 0 .../connection/connectors/openwrt/ssh.py | 6 + .../connection/connectors/ssh.py | 103 +++++++++ .../connection/migrations/0001_initial.py | 74 +++++++ .../connection/migrations/__init__.py | 0 openwisp_controller/connection/models.py | 167 ++++++++++++++ .../connection/tests/__init__.py | 0 .../connection/tests/test_models.py | 203 ++++++++++++++++++ openwisp_controller/connection/utils.py | 27 +++ setup.py | 5 +- tests/settings.py | 23 ++ tests/test-key.rsa | 15 ++ 16 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 openwisp_controller/connection/__init__.py create mode 100644 openwisp_controller/connection/admin.py create mode 100644 openwisp_controller/connection/apps.py create mode 100644 openwisp_controller/connection/connectors/__init__.py create mode 100644 openwisp_controller/connection/connectors/openwrt/__init__.py create mode 100644 openwisp_controller/connection/connectors/openwrt/ssh.py create mode 100644 openwisp_controller/connection/connectors/ssh.py create mode 100644 openwisp_controller/connection/migrations/0001_initial.py create mode 100644 openwisp_controller/connection/migrations/__init__.py create mode 100644 openwisp_controller/connection/models.py create mode 100644 openwisp_controller/connection/tests/__init__.py create mode 100644 openwisp_controller/connection/tests/test_models.py create mode 100644 openwisp_controller/connection/utils.py create mode 100644 tests/test-key.rsa diff --git a/openwisp_controller/connection/__init__.py b/openwisp_controller/connection/__init__.py new file mode 100644 index 000000000..07f6254df --- /dev/null +++ b/openwisp_controller/connection/__init__.py @@ -0,0 +1 @@ +default_app_config = 'openwisp_controller.connection.apps.ConnectionConfig' diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py new file mode 100644 index 000000000..9175fca03 --- /dev/null +++ b/openwisp_controller/connection/admin.py @@ -0,0 +1,35 @@ +from django.contrib import admin + +from openwisp_utils.admin import MultitenantOrgFilter, TimeReadonlyAdminMixin + +from ..admin import MultitenantAdminMixin +from ..config.admin import DeviceAdmin +from .models import Credentials, DeviceConnection, DeviceIp + + +@admin.register(Credentials) +class CredentialsAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin): + list_display = ('name', 'organization', 'connector', 'created', 'modified') + list_filter = [('organization', MultitenantOrgFilter), + 'connector'] + list_select_related = ('organization',) + + +class DeviceIpInline(admin.TabularInline): + model = DeviceIp + exclude = ('created', 'modified') + extra = 0 + + def get_queryset(self, request): + qs = super(DeviceIpInline, self).get_queryset(request) + return qs.order_by('priority') + + +class DeviceConnectionInline(admin.StackedInline): + model = DeviceConnection + exclude = ['params', 'created', 'modified'] + readonly_fields = ['is_working', 'failure_reason', 'last_attempt'] + extra = 0 + + +DeviceAdmin.inlines += [DeviceConnectionInline, DeviceIpInline] diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py new file mode 100644 index 000000000..c25b8ae3c --- /dev/null +++ b/openwisp_controller/connection/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ +from django_netjsonconfig.signals import config_modified + + +class ConnectionConfig(AppConfig): + name = 'openwisp_controller.connection' + label = 'connection' + verbose_name = _('Network Device Credentials') + + def ready(self): + # connect to config_modified signal + from .models import DeviceConnection + config_modified.connect(DeviceConnection.config_modified_receiver) diff --git a/openwisp_controller/connection/connectors/__init__.py b/openwisp_controller/connection/connectors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/connectors/openwrt/__init__.py b/openwisp_controller/connection/connectors/openwrt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/connectors/openwrt/ssh.py b/openwisp_controller/connection/connectors/openwrt/ssh.py new file mode 100644 index 000000000..ea646afde --- /dev/null +++ b/openwisp_controller/connection/connectors/openwrt/ssh.py @@ -0,0 +1,6 @@ +from ..ssh import Ssh + + +class OpenWrt(Ssh): + def update_config(self): + self.shell.exec_command('/etc/init.d/openwisp_config restart') diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py new file mode 100644 index 000000000..61a57863f --- /dev/null +++ b/openwisp_controller/connection/connectors/ssh.py @@ -0,0 +1,103 @@ +import ipaddress +import logging + +import paramiko +from django.core.exceptions import ObjectDoesNotExist +from django.utils.functional import cached_property +from jsonschema import validate +from jsonschema.exceptions import ValidationError as SchemaError +from scp import SCPClient + +from ..utils import get_interfaces + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + + +logger = logging.getLogger(__name__) + + +class Ssh(object): + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": False, + "required": ["username"], + "properties": { + "username": {"type": "string"}, + "password": {"type": "string"}, + "key": {"type": "string"}, + "port": {"type": "integer"}, + } + } + + def __init__(self, device_connection): + self.connection = device_connection + self.device = device_connection.device + self.shell = paramiko.SSHClient() + self.shell.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + @classmethod + def validate(cls, params): + validate(params, cls.schema) + cls.custom_validation(params) + + @classmethod + def custom_validation(cls, params): + if 'password' not in params and 'key' not in params: + raise SchemaError('Missing password or key') + + @cached_property + def _addresses(self): + deviceip_set = list(self.device.deviceip_set.all() + .only('address') + .order_by('priority')) + address_list = [] + for deviceip in deviceip_set: + address = deviceip.address + ip = ipaddress.ip_address(address) + if not ip.is_link_local: + address_list.append(address) + else: + for interface in get_interfaces(): + address_list.append('{0}%{1}'.format(address, interface)) + try: + address_list.append(self.device.config.last_ip) + except ObjectDoesNotExist: + pass + return address_list + + @cached_property + def _params(self): + params = self.connection.get_params() + if 'key' in params: + key_fileobj = StringIO(params.pop('key')) + params['pkey'] = paramiko.RSAKey.from_private_key(key_fileobj) + return params + + def connect(self): + success = False + exception = None + for address in self._addresses: + try: + self.shell.connect(address, **self._params) + except Exception as e: + exception = e + else: + success = True + break + if not success: + raise exception + + def disconnect(self): + self.shell.close() + + def update_config(self): + raise NotImplementedError() + + def upload(self, fl, remote_path): + scp = SCPClient(self.shell.get_transport()) + scp.putfo(fl, remote_path) + scp.close() diff --git a/openwisp_controller/connection/migrations/0001_initial.py b/openwisp_controller/connection/migrations/0001_initial.py new file mode 100644 index 000000000..5096ec1b8 --- /dev/null +++ b/openwisp_controller/connection/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# Generated by Django 2.0.5 on 2018-05-05 17:33 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +import openwisp_users.mixins +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('config', '0012_auto_20180219_1501'), + ('openwisp_users', '0002_auto_20180219_1409'), + ] + + operations = [ + migrations.CreateModel( + name='Credentials', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('name', models.CharField(db_index=True, max_length=64, unique=True)), + ('connector', models.CharField(choices=[('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH')], db_index=True, max_length=128, verbose_name='connection type')), + ('params', jsonfield.fields.JSONField(default=dict, help_text='global connection parameters', verbose_name='parameters')), + ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='openwisp_users.Organization', verbose_name='organization')), + ], + options={ + 'verbose_name_plural': 'Access credentials', + 'verbose_name': 'Access credentials', + }, + bases=(openwisp_users.mixins.ValidateOrgMixin, models.Model), + ), + migrations.CreateModel( + name='DeviceConnection', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('update_strategy', models.CharField(blank=True, choices=[('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH')], db_index=True, help_text='leave blank to determine automatically', max_length=128, verbose_name='update strategy')), + ('enabled', models.BooleanField(db_index=True, default=True)), + ('params', jsonfield.fields.JSONField(blank=True, default=dict, help_text='local connection parameters (will override the global parameters if specified)', verbose_name='parameters')), + ('is_working', models.NullBooleanField(default=None)), + ('last_attempt', models.DateTimeField(blank=True, null=True)), + ('failure_reason', models.CharField(blank=True, max_length=128, verbose_name='reason of failure')), + ('credentials', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='connection.Credentials')), + ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='config.Device')), + ], + options={ + 'verbose_name_plural': 'Device connections', + 'verbose_name': 'Device connection', + }, + ), + migrations.CreateModel( + name='DeviceIp', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('address', models.GenericIPAddressField(verbose_name='IP address')), + ('priority', models.PositiveSmallIntegerField()), + ('device', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='config.Device')), + ], + options={ + 'verbose_name_plural': 'Device IP addresses', + 'verbose_name': 'Device IP', + }, + ), + ] diff --git a/openwisp_controller/connection/migrations/__init__.py b/openwisp_controller/connection/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py new file mode 100644 index 000000000..ca4fccba9 --- /dev/null +++ b/openwisp_controller/connection/models.py @@ -0,0 +1,167 @@ +import collections + +from django.core.exceptions import ValidationError +from django.db import models +from django.utils import timezone +from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.module_loading import import_string +from django.utils.translation import ugettext_lazy as _ +from django_netjsonconfig.base.base import BaseModel +from jsonfield import JSONField +from jsonschema.exceptions import ValidationError as SchemaError + +from openwisp_users.mixins import ShareableOrgMixin +from openwisp_utils.base import TimeStampedEditableModel + + +class ConnectorMixin(object): + _connector_field = 'connector' + + def clean(self): + self._validate_connector_schema() + + def _validate_connector_schema(self): + try: + self.connector_class.validate(self.get_params()) + except SchemaError as e: + raise ValidationError({'params': e.message}) + + def get_params(self): + return self.params + + @cached_property + def connector_class(self): + return import_string(getattr(self, self._connector_field)) + + @cached_property + def connector_instance(self): + return self.connector_class(self) + + +class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): + """ + Credentials for access + """ + CONNECTOR_CHOICES = ( + ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), + ) + connector = models.CharField(_('connection type'), + choices=CONNECTOR_CHOICES, + max_length=128, + db_index=True) + params = JSONField(_('parameters'), + default=dict, + help_text=_('global connection parameters'), + load_kwargs={'object_pairs_hook': collections.OrderedDict}, + dump_kwargs={'indent': 4}) + + class Meta: + verbose_name = _('Access credentials') + verbose_name_plural = verbose_name + + def __str__(self): + return '{0} ({1})'.format(self.name, self.get_connector_display()) + + +@python_2_unicode_compatible +class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): + UPDATE_STRATEGY_CHOICES = ( + ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), + ) + CONFIG_BACKEND_MAPPING = { + 'netjsonconfig.OpenWrt': UPDATE_STRATEGY_CHOICES[0][0], + } + device = models.ForeignKey('config.Device', on_delete=models.CASCADE) + credentials = models.ForeignKey(Credentials, on_delete=models.CASCADE) + update_strategy = models.CharField(_('update strategy'), + help_text=_('leave blank to determine automatically'), + choices=UPDATE_STRATEGY_CHOICES, + max_length=128, + blank=True, + db_index=True) + enabled = models.BooleanField(default=True, db_index=True) + params = JSONField(_('parameters'), + default=dict, + blank=True, + help_text=_('local connection parameters (will override ' + 'the global parameters if specified)'), + load_kwargs={'object_pairs_hook': collections.OrderedDict}, + dump_kwargs={'indent': 4}) + # usability improvements + is_working = models.NullBooleanField(default=None) + failure_reason = models.CharField(_('reason of failure'), + max_length=128, + blank=True) + last_attempt = models.DateTimeField(blank=True, null=True) + _connector_field = 'update_strategy' + + class Meta: + verbose_name = _('Device connection') + verbose_name_plural = _('Device connections') + + def clean(self): + if not self.update_strategy and hasattr(self.device, 'config'): + try: + self.update_strategy = self.CONFIG_BACKEND_MAPPING[self.device.config.backend] + except KeyError as e: + raise ValidationError({ + 'update_stragy': _('could not determine update strategy ' + ' automatically, exception: {0}'.format(e)) + }) + elif not self.update_strategy: + raise ValidationError({ + 'update_strategy': _('the update strategy can be determined automatically ' + 'only if the device has a configuration specified, ' + 'because it is inferred from the configuration backend. ' + 'Please select the update strategy manually.') + }) + self._validate_connector_schema() + + def get_params(self): + params = self.credentials.params.copy() + params.update(self.params) + return params + + def connect(self): + try: + self.connector_instance.connect() + except Exception as e: + self.is_working = False + self.failure_reason = str(e) + else: + self.is_working = True + self.failure_reason = '' + finally: + self.last_attempt = timezone.now() + self.save() + + def disconnect(self): + self.connector_instance.disconnect() + + @classmethod + def config_modified_receiver(cls, **kwargs): + """ + receiver for ``config_modified`` signal + triggers the ``update_config`` operation + """ + qs = kwargs['device'].deviceconnection_set.filter(enabled=True) + if qs.count() > 0: + conn = qs.first() + conn.connector_instance.connect() + if conn.is_working: + conn.connector_instance.update_config() + + +@python_2_unicode_compatible +class DeviceIp(TimeStampedEditableModel): + device = models.ForeignKey('config.Device', on_delete=models.CASCADE) + address = models.GenericIPAddressField(_('IP address')) + priority = models.PositiveSmallIntegerField() + + class Meta: + verbose_name = _('Device IP') + verbose_name_plural = _('Device IP addresses') + + def __str__(self): + return self.address diff --git a/openwisp_controller/connection/tests/__init__.py b/openwisp_controller/connection/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py new file mode 100644 index 000000000..2e3bc6a42 --- /dev/null +++ b/openwisp_controller/connection/tests/test_models.py @@ -0,0 +1,203 @@ +import os + +import paramiko +from django.conf import settings +from django.core.exceptions import ValidationError +from django.test import TestCase +from mockssh import Server as SshServer +from openwisp_controller.config.models import Config, Device +from openwisp_controller.config.tests import CreateConfigTemplateMixin + +from openwisp_users.tests.utils import TestOrganizationMixin + +from ..models import Credentials, DeviceConnection, DeviceIp + + +class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): + device_model = Device + config_model = Config + _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') + with open(_TEST_RSA_KEY_PATH, 'r') as f: + _SSH_PRIVATE_KEY = f.read() + + @classmethod + def setUpClass(cls): + cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) + cls.ssh_server.__enter__() + + @classmethod + def tearDownClass(cls): + try: + cls.ssh_server.__exit__() + except OSError: + pass + + def _create_credentials(self, **kwargs): + opts = dict(name='Test credentials', + connector=Credentials.CONNECTOR_CHOICES[0][0], + params={'username': 'root', + 'password': 'password', + 'port': 22}) + opts.update(kwargs) + if 'organization' not in opts: + opts['organization'] = self._create_org() + c = Credentials(**opts) + c.full_clean() + c.save() + return c + + def _create_credentials_with_key(self, username='root', port=22, **kwargs): + opts = dict(name='Test SSH Key', + params={'username': username, + 'key': self._SSH_PRIVATE_KEY, + 'port': port}) + return self._create_credentials(**opts) + + def _create_device_connection(self, **kwargs): + opts = dict(enabled=True, + params={}) + opts.update(kwargs) + if 'credentials' not in opts: + opts['credentials'] = self._create_credentials() + org = opts['credentials'].organization + if 'device' not in opts: + opts['device'] = self._create_device(organization=org) + self._create_config(device=opts['device']) + dc = DeviceConnection(**opts) + dc.full_clean() + dc.save() + return dc + + def _create_device_ip(self, **kwargs): + opts = dict(address='10.40.0.1', + priority=1) + opts.update(kwargs) + if 'device' not in opts: + dc = self._create_device_connection() + opts['device'] = dc.device + ip = DeviceIp(**opts) + ip.full_clean() + ip.save() + return ip + + +class TestModels(TestConnectionMixin, TestCase): + def test_connection_str(self): + c = Credentials(name='Dev Key', connector=Credentials.CONNECTOR_CHOICES[0][0]) + self.assertIn(c.name, str(c)) + self.assertIn(c.get_connector_display(), str(c)) + + def test_deviceip_str(self): + di = DeviceIp(address='10.40.0.1') + self.assertIn(di.address, str(di)) + + def test_device_connection_get_params(self): + dc = self._create_device_connection() + self.assertEqual(dc.get_params(), dc.credentials.params) + dc.params = {'port': 2400} + self.assertEqual(dc.get_params()['port'], 2400) + self.assertEqual(dc.get_params()['username'], 'root') + + def test_device_connection_auto_update_strategy(self): + dc = self._create_device_connection() + self.assertEqual(dc.update_strategy, dc.UPDATE_STRATEGY_CHOICES[0][0]) + + def test_device_connection_auto_update_strategy_key_error(self): + orig_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES + orig_mapping = DeviceConnection.CONFIG_BACKEND_MAPPING + DeviceConnection.UPDATE_STRATEGY_CHOICES = (('meddle', 'meddle'),) + DeviceConnection.CONFIG_BACKEND_MAPPING = {'wrong': 'wrong'} + try: + self._create_device_connection() + except ValidationError: + failed = False + else: + failed = True + # restore + DeviceConnection.UPDATE_STRATEGY_CHOICES = orig_strategy + DeviceConnection.CONFIG_BACKEND_MAPPING = orig_mapping + if failed: + self.fail('ValidationError not raised') + + def test_device_connection_auto_update_strategy_missing_config(self): + device = self._create_device(organization=self._create_org()) + self.assertFalse(hasattr(device, 'config')) + try: + self._create_device_connection(device=device) + except ValidationError as e: + self.assertIn('inferred from', str(e)) + else: + self.fail('ValidationError not raised') + + def test_device_connection_connector_instance(self): + dc = self._create_device_connection() + self.assertIsInstance(dc.connector_instance, dc.connector_class) + + def test_device_connection_ssh_key_param(self): + ckey = self._create_credentials_with_key() + dc = self._create_device_connection(credentials=ckey) + self.assertIn('pkey', dc.connector_instance._params) + self.assertIsInstance(dc.connector_instance._params['pkey'], + paramiko.rsakey.RSAKey) + self.assertNotIn('key', dc.connector_instance._params) + + def test_ssh_connect(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connect() + self.assertTrue(dc.is_working) + self.assertIsNotNone(dc.last_attempt) + self.assertEqual(dc.failure_reason, '') + try: + dc.disconnect() + except OSError: + pass + + def test_ssh_connect_failure(self): + ckey = self._create_credentials_with_key(username='wrong', + port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connect() + self.assertEqual(dc.is_working, False) + self.assertIsNotNone(dc.last_attempt) + self.assertEqual(dc.failure_reason, 'Authentication failed.') + + def test_credentials_schema(self): + # unrecognized parameter + try: + self._create_credentials(params={ + 'username': 'root', + 'password': 'password', + 'unrecognized': True + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') + # missing password or key + try: + self._create_credentials(params={ + 'username': 'root', + 'port': 22 + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') + + def test_device_connection_schema(self): + # unrecognized parameter + try: + self._create_device_connection(params={ + 'username': 'root', + 'password': 'password', + 'unrecognized': True + }) + except ValidationError as e: + self.assertIn('params', e.message_dict) + else: + self.fail('ValidationError not raised') diff --git a/openwisp_controller/connection/utils.py b/openwisp_controller/connection/utils.py new file mode 100644 index 000000000..acfcf976b --- /dev/null +++ b/openwisp_controller/connection/utils.py @@ -0,0 +1,27 @@ +import array +import fcntl +import socket +import struct + + +def get_interfaces(): + """ + returns all non loopback interfaces available on the system + """ + max_possible = 128 + bytes_ = max_possible * 32 + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + names = array.array('B', b'\0' * bytes_) + outbytes = struct.unpack('iL', fcntl.ioctl( + s.fileno(), + 0x8912, + struct.pack('iL', bytes_, names.buffer_info()[0]) + ))[0] + namestr = names.tostring() + interfaces = [] + for i in range(0, outbytes, 40): + name = namestr[i:i + 16].split(b'\0', 1)[0] + name = name.decode() + if name != 'lo': + interfaces.append(name) + return interfaces diff --git a/setup.py b/setup.py index 1a24d8836..fc602ad40 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,10 @@ "django-netjsonconfig>=0.8.1,<0.10.0", "openwisp-utils[users]<0.3.1", "django-loci>=0.1.1,<0.3.0", - "djangorestframework-gis>=0.12.0,<0.14.0" + "djangorestframework-gis>=0.12.0,<0.14.0", + # connections feature + "paramiko>=2.4.1,<2.5.0", + "scp>=0.13.0,<0.14.0" ], classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tests/settings.py b/tests/settings.py index e26e612d3..687e77334 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -38,6 +38,7 @@ 'openwisp_controller.pki', 'openwisp_controller.config', 'openwisp_controller.geo', + 'openwisp_controller.connection', # admin 'django.contrib.admin', 'django.forms', @@ -121,6 +122,28 @@ # during development only EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +LOGGING = { + 'version': 1, + 'filters': { + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue', + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + 'django.db.backends': { + 'level': 'DEBUG', + 'handlers': ['console'], + } + } +} + # local settings must be imported before test runner otherwise they'll be ignored try: from local_settings import * diff --git a/tests/test-key.rsa b/tests/test-key.rsa new file mode 100644 index 000000000..bdd4846ed --- /dev/null +++ b/tests/test-key.rsa @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDQwuFrDNYUCi5Doy2xrc71P06vEuNG5i2rNgIHm95IuP8WrFwZ +W/VfNviGyhA8JwmWwHco9uzgKthaMKrGKB5Oeu/Z2F6SZPdCAdamCdbCcihXZ4g1 +RGbX5wECH7UjTx0th4GV6jwRAvJM/MpVJcCkTIzBHVHOC5jYotDuTnjJdwIDAQAB +AoGAHvfp7LF4yHxCJLJ+Qs9f1i3QBFSu9oOK3s0iO/K5ZNxcqwZimzhzC+7hq00q +X2IDICPpCWCn/xEcCzURAFhPNlx0RYZUzXOiW1JL7MzLYny87UAuW+TDaS4eEV9r +YX8acLWfg+aEw/pF0FRb2AuoRClztAyNF6GJtR/ky4z7vnECQQD3NEcEL1s913HW +1yV4RHBZO8n8oH2WidXtFDstmdmAvDQv7KC8c6rPJ6VVH5IlY+WyDIzI6X1IJFew +DXhO3A8zAkEA2DBvhy5TbAOPX7wQN53SA9+z4sdhOlYwcDpq2YuYvKH3ZFIWQEAX +cTQSjvaI35jWyKNYL+8T+Pqsngd3AUNsrQJBAI1yCSx42FFDRCz0v8jYCBzW3BVD +03hed9yGlfHatRw3E/lUAQizekm72pshTGM+jMBa8/dFulycBtyCaJNe0QcCQQCQ +uoxPcWIDs7ZuHta0hQEt+rrQnS2oAj9XQqR5kwzja4LVNGcVCFMpQ/UQpFcpaYaQ +t1m4bVNvoVGiUdkHjX3ZAkEAmHvrBB2TvcPZkhuUGviIlXbIeHWZMRF7wh0wZ7SH +SZWnv9EqwFcOGqqoLhQDznTI9TmWdpkxPxLzVwnjWLT4qw== +-----END RSA PRIVATE KEY----- From 792ae823f4785ae586a27cfe8c8cc01b6d34ac47 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 7 May 2018 16:39:51 +0200 Subject: [PATCH 24/55] [connections] Execute update_config in the background --- openwisp_controller/connection/apps.py | 29 +++++++++++++++++-- .../connection/connectors/ssh.py | 7 ++++- openwisp_controller/connection/models.py | 26 ++++++++--------- openwisp_controller/connection/tasks.py | 25 ++++++++++++++++ setup.py | 4 ++- tests/__init__.py | 0 tests/manage.py | 2 +- tests/openwisp2/__init__.py | 5 ++++ tests/openwisp2/celery.py | 11 +++++++ .../{ => openwisp2}/local_settings.example.py | 0 tests/{ => openwisp2}/settings.py | 11 +++++-- tests/{ => openwisp2}/test-key.rsa | 0 tests/{ => openwisp2}/urls.py | 0 13 files changed, 99 insertions(+), 21 deletions(-) create mode 100644 openwisp_controller/connection/tasks.py delete mode 100644 tests/__init__.py create mode 100644 tests/openwisp2/__init__.py create mode 100644 tests/openwisp2/celery.py rename tests/{ => openwisp2}/local_settings.example.py (100%) rename tests/{ => openwisp2}/settings.py (94%) rename tests/{ => openwisp2}/test-key.rsa (100%) rename tests/{ => openwisp2}/urls.py (100%) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index c25b8ae3c..7f4e97884 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,6 +1,9 @@ from django.apps import AppConfig from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified +from celery.task.control import inspect + +_TASK_NAME = 'openwisp_controller.connection.tasks.update_config' class ConnectionConfig(AppConfig): @@ -9,6 +12,26 @@ class ConnectionConfig(AppConfig): verbose_name = _('Network Device Credentials') def ready(self): - # connect to config_modified signal - from .models import DeviceConnection - config_modified.connect(DeviceConnection.config_modified_receiver) + """ + connects the ``config_modified`` signal + to the ``update_config`` celery task + which will be executed in the background + """ + from .tasks import update_config + + def config_modified_receiver(**kwargs): + device_id = kwargs['device'].id + if not self._is_update_in_progress(device_id): + update_config.delay(device_id) + + config_modified.connect(config_modified_receiver, + dispatch_uid='connection.update_config', + weak=False) + + def _is_update_in_progress(self, device_id): + # check if there's any other running task before adding it + for task_list in inspect().active().values(): + for task in task_list: + if task['name'] == _TASK_NAME and str(device_id) in task['args']: + return True + return False diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 61a57863f..bb45c5dac 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -17,6 +17,8 @@ logger = logging.getLogger(__name__) +SSH_CONNECTION_TIMEOUT = 5 +SSH_AUTH_TIMEOUT = 2 class Ssh(object): @@ -82,7 +84,10 @@ def connect(self): exception = None for address in self._addresses: try: - self.shell.connect(address, **self._params) + self.shell.connect(address, + timeout=SSH_CONNECTION_TIMEOUT, + auth_timeout=SSH_AUTH_TIMEOUT, + **self._params) except Exception as e: exception = e else: diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index ca4fccba9..f4e14aae3 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -14,6 +14,9 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel +import logging +logger = logging.getLogger(__name__) + class ConnectorMixin(object): _connector_field = 'connector' @@ -66,6 +69,7 @@ def __str__(self): @python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): + _connector_field = 'update_strategy' UPDATE_STRATEGY_CHOICES = ( ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), ) @@ -94,7 +98,6 @@ class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): max_length=128, blank=True) last_attempt = models.DateTimeField(blank=True, null=True) - _connector_field = 'update_strategy' class Meta: verbose_name = _('Device connection') @@ -139,18 +142,15 @@ def connect(self): def disconnect(self): self.connector_instance.disconnect() - @classmethod - def config_modified_receiver(cls, **kwargs): - """ - receiver for ``config_modified`` signal - triggers the ``update_config`` operation - """ - qs = kwargs['device'].deviceconnection_set.filter(enabled=True) - if qs.count() > 0: - conn = qs.first() - conn.connector_instance.connect() - if conn.is_working: - conn.connector_instance.update_config() + def update_config(self): + self.connect() + if self.is_working: + try: + self.connector_instance.update_config() + except Exception as e: + logger.exception(e) + else: + self.device.config.set_status_running() @python_2_unicode_compatible diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py new file mode 100644 index 000000000..0a4880d29 --- /dev/null +++ b/openwisp_controller/connection/tasks.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, unicode_literals + +from celery import shared_task +from time import sleep + +from ..config.models import Device + + +@shared_task +def update_config(device_id): + """ + Launches the ``update_config()`` operation + of a specific device in the background + """ + # wait for the saving operations of this device to complete + # (there may be multiple ones happening at the same time) + sleep(4) + # avoid repeating the operation multiple times + device = Device.objects.select_related('config').get(pk=device_id) + if device.config.status == 'running': + return + qs = device.deviceconnection_set.filter(device_id=device_id, enabled=True) + if qs.count() > 0: + conn = qs.first() + conn.update_config() diff --git a/setup.py b/setup.py index fc602ad40..bee535c7b 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,10 @@ "django-loci>=0.1.1,<0.3.0", "djangorestframework-gis>=0.12.0,<0.14.0", # connections feature + # TODO: make optional? "paramiko>=2.4.1,<2.5.0", - "scp>=0.13.0,<0.14.0" + "scp>=0.13.0,<0.14.0", + "celery>=4.1.0,<4.2.0" ], classifiers=[ 'Development Status :: 3 - Alpha', diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/manage.py b/tests/manage.py index 9811d22f7..dd989d36d 100755 --- a/tests/manage.py +++ b/tests/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'openwisp2.settings') from django.core.management import execute_from_command_line diff --git a/tests/openwisp2/__init__.py b/tests/openwisp2/__init__.py new file mode 100644 index 000000000..76be18a55 --- /dev/null +++ b/tests/openwisp2/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import, unicode_literals + +from .celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/tests/openwisp2/celery.py b/tests/openwisp2/celery.py new file mode 100644 index 000000000..440c69813 --- /dev/null +++ b/tests/openwisp2/celery.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'openwisp2.settings') + +app = Celery('openwisp2') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/tests/local_settings.example.py b/tests/openwisp2/local_settings.example.py similarity index 100% rename from tests/local_settings.example.py rename to tests/openwisp2/local_settings.example.py diff --git a/tests/settings.py b/tests/openwisp2/settings.py similarity index 94% rename from tests/settings.py rename to tests/openwisp2/settings.py index 687e77334..55c4cd099 100644 --- a/tests/settings.py +++ b/tests/openwisp2/settings.py @@ -74,7 +74,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'urls' +ROOT_URLCONF = 'openwisp2.urls' CHANNEL_LAYERS = { 'default': { @@ -122,6 +122,13 @@ # during development only EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +if not TESTING: + CELERY_BROKER_URL = 'redis://localhost/1' +else: + CELERY_TASK_ALWAYS_EAGER = True + CELERY_TASK_EAGER_PROPAGATES = True + CELERY_BROKER_URL = 'memory://' + LOGGING = { 'version': 1, 'filters': { @@ -146,6 +153,6 @@ # local settings must be imported before test runner otherwise they'll be ignored try: - from local_settings import * + from .local_settings import * except ImportError: pass diff --git a/tests/test-key.rsa b/tests/openwisp2/test-key.rsa similarity index 100% rename from tests/test-key.rsa rename to tests/openwisp2/test-key.rsa diff --git a/tests/urls.py b/tests/openwisp2/urls.py similarity index 100% rename from tests/urls.py rename to tests/openwisp2/urls.py From 0ac107af8e0edf69dae969a3d41398d4925a27aa Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 20:10:29 +0200 Subject: [PATCH 25/55] [connections] Keep connector class and model logic decoupled --- openwisp_controller/connection/apps.py | 2 +- .../connection/connectors/ssh.py | 38 ++++------------- openwisp_controller/connection/models.py | 36 ++++++++++++++-- openwisp_controller/connection/tasks.py | 3 +- .../connection/tests/test_models.py | 42 +++++++++++++++++-- 5 files changed, 82 insertions(+), 39 deletions(-) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 7f4e97884..1f17d9b77 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,7 +1,7 @@ +from celery.task.control import inspect from django.apps import AppConfig from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified -from celery.task.control import inspect _TASK_NAME = 'openwisp_controller.connection.tasks.update_config' diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index bb45c5dac..34ee18d2e 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,15 +1,11 @@ -import ipaddress import logging import paramiko -from django.core.exceptions import ObjectDoesNotExist from django.utils.functional import cached_property from jsonschema import validate from jsonschema.exceptions import ValidationError as SchemaError from scp import SCPClient -from ..utils import get_interfaces - try: from io import StringIO except ImportError: @@ -35,9 +31,9 @@ class Ssh(object): } } - def __init__(self, device_connection): - self.connection = device_connection - self.device = device_connection.device + def __init__(self, params, addresses): + self._params = params + self.addresses = addresses self.shell = paramiko.SSHClient() self.shell.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -52,28 +48,8 @@ def custom_validation(cls, params): raise SchemaError('Missing password or key') @cached_property - def _addresses(self): - deviceip_set = list(self.device.deviceip_set.all() - .only('address') - .order_by('priority')) - address_list = [] - for deviceip in deviceip_set: - address = deviceip.address - ip = ipaddress.ip_address(address) - if not ip.is_link_local: - address_list.append(address) - else: - for interface in get_interfaces(): - address_list.append('{0}%{1}'.format(address, interface)) - try: - address_list.append(self.device.config.last_ip) - except ObjectDoesNotExist: - pass - return address_list - - @cached_property - def _params(self): - params = self.connection.get_params() + def params(self): + params = self._params.copy() if 'key' in params: key_fileobj = StringIO(params.pop('key')) params['pkey'] = paramiko.RSAKey.from_private_key(key_fileobj) @@ -82,12 +58,12 @@ def _params(self): def connect(self): success = False exception = None - for address in self._addresses: + for address in self.addresses: try: self.shell.connect(address, timeout=SSH_CONNECTION_TIMEOUT, auth_timeout=SSH_AUTH_TIMEOUT, - **self._params) + **self.params) except Exception as e: exception = e else: diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index f4e14aae3..82d4ac139 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -1,6 +1,8 @@ import collections +import ipaddress +import logging -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import models from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible @@ -14,7 +16,8 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel -import logging +from .utils import get_interfaces + logger = logging.getLogger(__name__) @@ -33,13 +36,17 @@ def _validate_connector_schema(self): def get_params(self): return self.params + def get_addresses(self): + return [] + @cached_property def connector_class(self): return import_string(getattr(self, self._connector_field)) @cached_property def connector_instance(self): - return self.connector_class(self) + return self.connector_class(params=self.get_params(), + addresses=self.get_addresses()) class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): @@ -121,6 +128,29 @@ def clean(self): }) self._validate_connector_schema() + def get_addresses(self): + """ + returns a list of ip addresses for the related device + (used to pass a list of ip addresses to a DeviceConnection instance) + """ + deviceip_set = list(self.device.deviceip_set.all() + .only('address') + .order_by('priority')) + address_list = [] + for deviceip in deviceip_set: + address = deviceip.address + ip = ipaddress.ip_address(address) + if not ip.is_link_local: + address_list.append(address) + else: + for interface in get_interfaces(): + address_list.append('{0}%{1}'.format(address, interface)) + try: + address_list.append(self.device.config.last_ip) + except ObjectDoesNotExist: + pass + return address_list + def get_params(self): params = self.credentials.params.copy() params.update(self.params) diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py index 0a4880d29..d02727dd3 100644 --- a/openwisp_controller/connection/tasks.py +++ b/openwisp_controller/connection/tasks.py @@ -1,8 +1,9 @@ from __future__ import absolute_import, unicode_literals -from celery import shared_task from time import sleep +from celery import shared_task + from ..config.models import Device diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 2e3bc6a42..42ae72145 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -11,6 +11,7 @@ from openwisp_users.tests.utils import TestOrganizationMixin from ..models import Credentials, DeviceConnection, DeviceIp +from ..utils import get_interfaces class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): @@ -136,10 +137,10 @@ def test_device_connection_connector_instance(self): def test_device_connection_ssh_key_param(self): ckey = self._create_credentials_with_key() dc = self._create_device_connection(credentials=ckey) - self.assertIn('pkey', dc.connector_instance._params) - self.assertIsInstance(dc.connector_instance._params['pkey'], + self.assertIn('pkey', dc.connector_instance.params) + self.assertIsInstance(dc.connector_instance.params['pkey'], paramiko.rsakey.RSAKey) - self.assertNotIn('key', dc.connector_instance._params) + self.assertNotIn('key', dc.connector_instance.params) def test_ssh_connect(self): ckey = self._create_credentials_with_key(port=self.ssh_server.port) @@ -201,3 +202,38 @@ def test_device_connection_schema(self): self.assertIn('params', e.message_dict) else: self.fail('ValidationError not raised') + + def _prepare_address_list_test(self, addresses): + update_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES[0][0] + device = self._create_device(organization=self._create_org()) + dc = self._create_device_connection(device=device, + update_strategy=update_strategy) + for index, address in enumerate(addresses): + self._create_device_ip(device=device, + address=address, + priority=index + 1) + return dc + + def test_address_list(self): + dc = self._prepare_address_list_test(['10.40.0.1', '192.168.40.1']) + self.assertEqual(dc.get_addresses(), [ + '10.40.0.1', + '192.168.40.1' + ]) + + def test_address_list_with_config_last_ip(self): + dc = self._prepare_address_list_test(['192.168.40.1']) + self._create_config(device=dc.device, + last_ip='10.40.0.2') + self.assertEqual(dc.get_addresses(), [ + '192.168.40.1', + '10.40.0.2', + ]) + + def test_address_list_link_local_ip(self): + ipv6_linklocal = 'fe80::2dae:a0d4:94da:7f61' + dc = self._prepare_address_list_test([ipv6_linklocal]) + address_list = dc.get_addresses() + interfaces = get_interfaces() + self.assertEqual(len(address_list), len(interfaces)) + self.assertIn(ipv6_linklocal, address_list[0]) From 5cfa5096a1fbd636002f39022591742e92263cd4 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 20:19:12 +0200 Subject: [PATCH 26/55] [connection] Fixed migration dependency on openwisp-users --- openwisp_controller/connection/migrations/0001_initial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/migrations/0001_initial.py b/openwisp_controller/connection/migrations/0001_initial.py index 5096ec1b8..2a5f43646 100644 --- a/openwisp_controller/connection/migrations/0001_initial.py +++ b/openwisp_controller/connection/migrations/0001_initial.py @@ -15,7 +15,7 @@ class Migration(migrations.Migration): dependencies = [ ('config', '0012_auto_20180219_1501'), - ('openwisp_users', '0002_auto_20180219_1409'), + ('openwisp_users', '0001_initial'), ] operations = [ From 5d28948358d1575d896e2a5db9aca13cc22a0b31 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 21:57:01 +0200 Subject: [PATCH 27/55] [connections] Added settings - OPENWISP_CONNECTORS - OPENWISP_UPDATE_STRATEGIES - OPENWISP_CONFIG_UPDATE_MAPPING --- README.rst | 67 ++++++++++++++++++- openwisp_controller/connection/models.py | 16 ++--- openwisp_controller/connection/settings.py | 13 ++++ .../connection/tests/test_models.py | 21 +++--- 4 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 openwisp_controller/connection/settings.py diff --git a/README.rst b/README.rst index 1a5baa474..90d64d8cf 100755 --- a/README.rst +++ b/README.rst @@ -188,6 +188,65 @@ Add the following settings to ``settings.py``: urlpatterns += staticfiles_urlpatterns() +Settings +-------- + +``OPENWISP_CONNECTORS`` +~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+--------------------------------------------------------------------+ +| **type**: | ``tuple`` | ++--------------+--------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | ( | +| | ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), | +| | ) | ++--------------+--------------------------------------------------------------------+ + +Available connector classes. Connectors are python classes that specify ways +in which OpenWISP can connect to devices in order to launch commands. + +``OPENWISP_UPDATE_STRATEGIES`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+----------------------------------------------------------------------------------------+ +| **type**: | ``tuple`` | ++--------------+----------------------------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | ( | +| | ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), | +| | ) | ++--------------+----------------------------------------------------------------------------------------+ + +Available update strategies. An update strategy is a subclass of a +connector class which defines an ``update_config`` method which is +in charge of updating the configuratio of the device. + +This operation is launched in a background worker when the configuration +of a device is changed. + +It's possible to write custom update strategies and add them to this +setting to make them available in OpenWISP. + +``OPENWISP_CONFIG_UPDATE_MAPPING`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++--------------+--------------------------------------------------------------------+ +| **type**: | ``dict`` | ++--------------+--------------------------------------------------------------------+ +| **default**: | .. code-block:: python | +| | | +| | { | +| | 'netjsonconfig.OpenWrt': OPENWISP_UPDATE_STRATEGIES[0][0], | +| | } | ++--------------+--------------------------------------------------------------------+ + +A dictionary that maps configuration backends to update strategies in order to +automatically determine the update strategy of a device connection if the +update strategy field is left blank by the user. + Installing for development -------------------------- @@ -209,12 +268,18 @@ Install your forked repo with `pipenv Create database: -.. code-block:: shell +.. code-block:: shell cd tests/ pipenv run ./manage.py migrate pipenv run ./manage.py createsuperuser +Launch celery worker (for background jobs): + +.. code-block:: shell + + celery -A openwisp2 worker -l info + Launch development server: .. code-block:: shell diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 82d4ac139..e77c3dcc7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -16,6 +16,7 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel +from . import settings as app_settings from .utils import get_interfaces logger = logging.getLogger(__name__) @@ -53,11 +54,8 @@ class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): """ Credentials for access """ - CONNECTOR_CHOICES = ( - ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), - ) connector = models.CharField(_('connection type'), - choices=CONNECTOR_CHOICES, + choices=app_settings.CONNECTORS, max_length=128, db_index=True) params = JSONField(_('parameters'), @@ -77,17 +75,11 @@ def __str__(self): @python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' - UPDATE_STRATEGY_CHOICES = ( - ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), - ) - CONFIG_BACKEND_MAPPING = { - 'netjsonconfig.OpenWrt': UPDATE_STRATEGY_CHOICES[0][0], - } device = models.ForeignKey('config.Device', on_delete=models.CASCADE) credentials = models.ForeignKey(Credentials, on_delete=models.CASCADE) update_strategy = models.CharField(_('update strategy'), help_text=_('leave blank to determine automatically'), - choices=UPDATE_STRATEGY_CHOICES, + choices=app_settings.UPDATE_STRATEGIES, max_length=128, blank=True, db_index=True) @@ -113,7 +105,7 @@ class Meta: def clean(self): if not self.update_strategy and hasattr(self.device, 'config'): try: - self.update_strategy = self.CONFIG_BACKEND_MAPPING[self.device.config.backend] + self.update_strategy = app_settings.CONFIG_UPDATE_MAPPING[self.device.config.backend] except KeyError as e: raise ValidationError({ 'update_stragy': _('could not determine update strategy ' diff --git a/openwisp_controller/connection/settings.py b/openwisp_controller/connection/settings.py new file mode 100644 index 000000000..0af20ada3 --- /dev/null +++ b/openwisp_controller/connection/settings.py @@ -0,0 +1,13 @@ +from django.conf import settings + +CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', ( + ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), +)) + +UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', ( + ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), +)) + +CONFIG_UPDATE_MAPPING = getattr(settings, 'OPENWISP_CONFIG_UPDATE_MAPPING', { + 'netjsonconfig.OpenWrt': UPDATE_STRATEGIES[0][0], +}) diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 42ae72145..4f8ec61c2 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -10,6 +10,7 @@ from openwisp_users.tests.utils import TestOrganizationMixin +from .. import settings as app_settings from ..models import Credentials, DeviceConnection, DeviceIp from ..utils import get_interfaces @@ -35,7 +36,7 @@ def tearDownClass(cls): def _create_credentials(self, **kwargs): opts = dict(name='Test credentials', - connector=Credentials.CONNECTOR_CHOICES[0][0], + connector=app_settings.CONNECTORS[0][0], params={'username': 'root', 'password': 'password', 'port': 22}) @@ -84,7 +85,7 @@ def _create_device_ip(self, **kwargs): class TestModels(TestConnectionMixin, TestCase): def test_connection_str(self): - c = Credentials(name='Dev Key', connector=Credentials.CONNECTOR_CHOICES[0][0]) + c = Credentials(name='Dev Key', connector=app_settings.CONNECTORS[0][0]) self.assertIn(c.name, str(c)) self.assertIn(c.get_connector_display(), str(c)) @@ -101,13 +102,13 @@ def test_device_connection_get_params(self): def test_device_connection_auto_update_strategy(self): dc = self._create_device_connection() - self.assertEqual(dc.update_strategy, dc.UPDATE_STRATEGY_CHOICES[0][0]) + self.assertEqual(dc.update_strategy, app_settings.UPDATE_STRATEGIES[0][0]) def test_device_connection_auto_update_strategy_key_error(self): - orig_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES - orig_mapping = DeviceConnection.CONFIG_BACKEND_MAPPING - DeviceConnection.UPDATE_STRATEGY_CHOICES = (('meddle', 'meddle'),) - DeviceConnection.CONFIG_BACKEND_MAPPING = {'wrong': 'wrong'} + orig_strategy = app_settings.UPDATE_STRATEGIES + orig_mapping = app_settings.CONFIG_UPDATE_MAPPING + app_settings.UPDATE_STRATEGIES = (('meddle', 'meddle'),) + app_settings.CONFIG_UPDATE_MAPPING = {'wrong': 'wrong'} try: self._create_device_connection() except ValidationError: @@ -115,8 +116,8 @@ def test_device_connection_auto_update_strategy_key_error(self): else: failed = True # restore - DeviceConnection.UPDATE_STRATEGY_CHOICES = orig_strategy - DeviceConnection.CONFIG_BACKEND_MAPPING = orig_mapping + app_settings.UPDATE_STRATEGIES = orig_strategy + app_settings.CONFIG_UPDATE_MAPPING = orig_mapping if failed: self.fail('ValidationError not raised') @@ -204,7 +205,7 @@ def test_device_connection_schema(self): self.fail('ValidationError not raised') def _prepare_address_list_test(self, addresses): - update_strategy = DeviceConnection.UPDATE_STRATEGY_CHOICES[0][0] + update_strategy = app_settings.UPDATE_STRATEGIES[0][0] device = self._create_device(organization=self._create_org()) dc = self._create_device_connection(device=device, update_strategy=update_strategy) From df755e330a92fd614a84ba519739db66c0f47fc9 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 8 May 2018 22:29:10 +0200 Subject: [PATCH 28/55] [connections] Updated runtests.py to use new setting file --- runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.py b/runtests.py index 6dd62fa93..84dd794f0 100755 --- a/runtests.py +++ b/runtests.py @@ -5,7 +5,7 @@ import sys sys.path.insert(0, "tests") -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp2.settings") if __name__ == "__main__": from django.core.management import execute_from_command_line From 5f1e755118324268545fb53ef68e618ea4dc7da9 Mon Sep 17 00:00:00 2001 From: Oliver Kraitschy Date: Wed, 9 May 2018 07:32:48 +0200 Subject: [PATCH 29/55] Fix markdown in README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 90d64d8cf..9bf11ce0c 100755 --- a/README.rst +++ b/README.rst @@ -268,8 +268,8 @@ Install your forked repo with `pipenv Create database: - .. code-block:: shell + cd tests/ pipenv run ./manage.py migrate pipenv run ./manage.py createsuperuser From 79ac0317c528b3c2c5ba5dfcc17ad0336af85599 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 14:40:28 +0200 Subject: [PATCH 30/55] [connection] Made config_modified_receiver more resilient to exceptions --- openwisp_controller/connection/apps.py | 15 +++++++++++---- openwisp_controller/connection/tasks.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 1f17d9b77..1a4e84c45 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -20,17 +20,24 @@ def ready(self): from .tasks import update_config def config_modified_receiver(**kwargs): - device_id = kwargs['device'].id - if not self._is_update_in_progress(device_id): - update_config.delay(device_id) + d = kwargs['device'] + conn_count = d.deviceconnection_set.count() + # if device has no connection specified + # or update is already in progress, stop here + if conn_count < 1 or self._is_update_in_progress(d.id): + return + update_config.delay(d.id) config_modified.connect(config_modified_receiver, dispatch_uid='connection.update_config', weak=False) def _is_update_in_progress(self, device_id): + active = inspect().active() + if not active: + return False # check if there's any other running task before adding it - for task_list in inspect().active().values(): + for task_list in active.values(): for task in task_list: if task['name'] == _TASK_NAME and str(device_id) in task['args']: return True diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py index d02727dd3..689e41570 100644 --- a/openwisp_controller/connection/tasks.py +++ b/openwisp_controller/connection/tasks.py @@ -21,6 +21,6 @@ def update_config(device_id): if device.config.status == 'running': return qs = device.deviceconnection_set.filter(device_id=device_id, enabled=True) - if qs.count() > 0: - conn = qs.first() + conn = qs.first() + if conn: conn.update_config() From 5745e2b8edc128974fcdecfb5aea900fcd2f1bc3 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 14:40:39 +0200 Subject: [PATCH 31/55] [connection] Updated MEDIA_ROOT --- tests/openwisp2/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 55c4cd099..e81727a00 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -90,7 +90,7 @@ USE_L10N = False STATIC_URL = '/static/' MEDIA_URL = '/media/' -MEDIA_ROOT = '{0}/media/'.format(BASE_DIR) +MEDIA_ROOT = '{0}/media/'.format(os.path.dirname(BASE_DIR)) TEMPLATES = [ { From 657ee64277f5d37bd6019bdf5ee9be82ec39ac99 Mon Sep 17 00:00:00 2001 From: Oliver Kraitschy Date: Wed, 9 May 2018 15:05:52 +0200 Subject: [PATCH 32/55] [connections] Update dependencies in README.rst and add setting for mod_spatialite --- README.rst | 7 ++++--- tests/openwisp2/settings.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 9bf11ce0c..3b281d007 100755 --- a/README.rst +++ b/README.rst @@ -250,12 +250,13 @@ update strategy field is left blank by the user. Installing for development -------------------------- -Install sqlite: +Install the dependencies: .. code-block:: shell - sudo apt-get install sqlite3 libsqlite3-dev libsqlite3-mod-spatialite openssl libssl-dev - sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev + sudo apt-get install sqlite3 libsqlite3-dev openssl libssl-dev + sudo apt-get install gdal-bin libproj-dev libgeos-dev libspatialite-dev libsqlite3-mod-spatialite + sudo apt-get install redis Install your forked repo with `pipenv `_: diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index e81727a00..3cae81556 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -129,6 +129,8 @@ CELERY_TASK_EAGER_PROPAGATES = True CELERY_BROKER_URL = 'memory://' +SPATIALITE_LIBRARY_PATH = 'mod_spatialite' + LOGGING = { 'version': 1, 'filters': { From afc8d223da94eae9dde3c8323a8b062af44eecf9 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 17:08:36 +0200 Subject: [PATCH 33/55] [connection] Fixed admin tests Added missing inline form parameters --- Pipfile | 1 + openwisp_controller/config/tests/test_admin.py | 9 +++++++++ openwisp_controller/connection/connectors/ssh.py | 5 +++-- openwisp_controller/connection/models.py | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index af36b47f4..59d8ca696 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ coverage = "*" coveralls = "*" isort = "*" flake8 = "*" +mock-ssh-server = {version = ">=0.5.0,<0.6.0"} [scripts] lint = "python -m flake8" diff --git a/openwisp_controller/config/tests/test_admin.py b/openwisp_controller/config/tests/test_admin.py index 8fa3a44f6..5f7cddf1e 100644 --- a/openwisp_controller/config/tests/test_admin.py +++ b/openwisp_controller/config/tests/test_admin.py @@ -41,6 +41,15 @@ class TestAdmin(CreateConfigTemplateMixin, TestAdminMixin, 'config-INITIAL_FORMS': 0, 'config-MIN_NUM_FORMS': 0, 'config-MAX_NUM_FORMS': 1, + # openwisp_controller.connection + 'deviceconnection_set-TOTAL_FORMS': 0, + 'deviceconnection_set-INITIAL_FORMS': 0, + 'deviceconnection_set-MIN_NUM_FORMS': 0, + 'deviceconnection_set-MAX_NUM_FORMS': 1000, + 'deviceip_set-TOTAL_FORMS': 0, + 'deviceip_set-INITIAL_FORMS': 0, + 'deviceip_set-MIN_NUM_FORMS': 0, + 'deviceip_set-MAX_NUM_FORMS': 1000, } # WARNING - WATCHOUT # this class attribute is changed dinamically diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 34ee18d2e..3174ee738 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,4 +1,5 @@ import logging +import sys import paramiko from django.utils.functional import cached_property @@ -6,9 +7,9 @@ from jsonschema.exceptions import ValidationError as SchemaError from scp import SCPClient -try: +if sys.version_info.major > 2: # pragma: nocover from io import StringIO -except ImportError: +else: # pragma: nocover from StringIO import StringIO diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index e77c3dcc7..c07ec3364 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -72,7 +72,6 @@ def __str__(self): return '{0} ({1})'.format(self.name, self.get_connector_display()) -@python_2_unicode_compatible class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' device = models.ForeignKey('config.Device', on_delete=models.CASCADE) From 06706939f4f04a74731192deab2045e792108e77 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 9 May 2018 17:52:25 +0200 Subject: [PATCH 34/55] [test] Removed `SPATIALITE_LIBRARY_PATH = 'mod_spatialite` Breaks the travis build. --- tests/openwisp2/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index 3cae81556..e81727a00 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -129,8 +129,6 @@ CELERY_TASK_EAGER_PROPAGATES = True CELERY_BROKER_URL = 'memory://' -SPATIALITE_LIBRARY_PATH = 'mod_spatialite' - LOGGING = { 'version': 1, 'filters': { From 4afab31581401f1ee500d1b7f2d7a6724c1859cc Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:32:20 +0200 Subject: [PATCH 35/55] [connection] Added exec_command method --- .../connection/connectors/openwrt/ssh.py | 2 +- .../connection/connectors/ssh.py | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/openwisp_controller/connection/connectors/openwrt/ssh.py b/openwisp_controller/connection/connectors/openwrt/ssh.py index ea646afde..1874e6f9e 100644 --- a/openwisp_controller/connection/connectors/openwrt/ssh.py +++ b/openwisp_controller/connection/connectors/openwrt/ssh.py @@ -3,4 +3,4 @@ class OpenWrt(Ssh): def update_config(self): - self.shell.exec_command('/etc/init.d/openwisp_config restart') + self.exec_command('/etc/init.d/openwisp_config restart') diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 3174ee738..c2f59d7c0 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,5 +1,7 @@ import logging +import socket import sys +from io import BytesIO import paramiko from django.utils.functional import cached_property @@ -16,6 +18,7 @@ logger = logging.getLogger(__name__) SSH_CONNECTION_TIMEOUT = 5 SSH_AUTH_TIMEOUT = 2 +SSH_COMMAND_TIMEOUT = 5 class Ssh(object): @@ -76,10 +79,54 @@ def connect(self): def disconnect(self): self.shell.close() + def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, + exit_codes=[0], raise_unexpected_exit=True): + """ + Executes a command and performs the following operations + - logs executed command + - logs standard output + - logs standard error + - aborts on exceptions + - raises socket.timeout exceptions + """ + print('$:> {0}'.format(command)) + # execute commmand + try: + stdin, stdout, stderr = self.shell.exec_command(command) + # re-raise socket.timeout to avoid being catched + # by the subsequent `except Exception as e` block + except socket.timeout: + raise socket.timeout() + # any other exception will abort the operation + except Exception as e: + logger.exception(e) + raise e + # store command exit status + exit_status = stdout.channel.recv_exit_status() + # log standard output + output = stdout.read().decode('utf8').strip() + if output: + print(output) + # log standard error + error = stderr.read().decode('utf8').strip() + if error: + print(error) + # abort the operation if any of the command + # returned with a non-zero exit status + if exit_status not in exit_codes and raise_unexpected_exit: + print('# Previus command failed, aborting upgrade...') + message = error if error else output + raise Exception(message) + return output, exit_status + def update_config(self): raise NotImplementedError() def upload(self, fl, remote_path): scp = SCPClient(self.shell.get_transport()) + if not hasattr(fl, 'getvalue'): + fl_memory = BytesIO(fl.read()) + fl.seek(0) + fl = fl_memory scp.putfo(fl, remote_path) scp.close() From e2c81f8df0a2a0aa640b0a87625f25ff03d73194 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:32:45 +0200 Subject: [PATCH 36/55] [connection] Close connection after update_config --- openwisp_controller/connection/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index c07ec3364..d665398ce 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -172,6 +172,7 @@ def update_config(self): logger.exception(e) else: self.device.config.set_status_running() + self.disconnect() @python_2_unicode_compatible From 445611fcd61e59d3b0cbfc9d483d74ce40bc76b7 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 10 May 2018 21:33:20 +0200 Subject: [PATCH 37/55] [connection] Added default settings --- openwisp_controller/connection/settings.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/openwisp_controller/connection/settings.py b/openwisp_controller/connection/settings.py index 0af20ada3..5b4f159b4 100644 --- a/openwisp_controller/connection/settings.py +++ b/openwisp_controller/connection/settings.py @@ -1,13 +1,17 @@ from django.conf import settings -CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', ( +DEFAULT_CONNECTORS = ( ('openwisp_controller.connection.connectors.ssh.Ssh', 'SSH'), -)) +) -UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', ( +CONNECTORS = getattr(settings, 'OPENWISP_CONNECTORS', DEFAULT_CONNECTORS) + +DEFAULT_UPDATE_STRATEGIES = ( ('openwisp_controller.connection.connectors.openwrt.ssh.OpenWrt', 'OpenWRT SSH'), -)) +) + +UPDATE_STRATEGIES = getattr(settings, 'OPENWISP_UPDATE_STRATEGIES', DEFAULT_UPDATE_STRATEGIES) CONFIG_UPDATE_MAPPING = getattr(settings, 'OPENWISP_CONFIG_UPDATE_MAPPING', { - 'netjsonconfig.OpenWrt': UPDATE_STRATEGIES[0][0], + 'netjsonconfig.OpenWrt': DEFAULT_UPDATE_STRATEGIES[0][0], }) From b055fc6afadb96c4cad4fda080e635fa5ea0e2ae Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 25 May 2018 12:55:43 +0300 Subject: [PATCH 38/55] [connection] Made test mixins reusable (for extensions) --- openwisp_controller/connection/tests/base.py | 83 +++++++++++++++++++ .../connection/tests/test_models.py | 83 +------------------ 2 files changed, 87 insertions(+), 79 deletions(-) create mode 100644 openwisp_controller/connection/tests/base.py diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py new file mode 100644 index 000000000..a57464e4f --- /dev/null +++ b/openwisp_controller/connection/tests/base.py @@ -0,0 +1,83 @@ +import os + +from django.conf import settings +from mockssh import Server as SshServer +from openwisp_controller.config.models import Config, Device +from openwisp_controller.config.tests import CreateConfigTemplateMixin + +from openwisp_users.tests.utils import TestOrganizationMixin + +from .. import settings as app_settings +from ..models import Credentials, DeviceConnection, DeviceIp + + +class CreateConnectionsMixin(CreateConfigTemplateMixin, TestOrganizationMixin): + device_model = Device + config_model = Config + + def _create_credentials(self, **kwargs): + opts = dict(name='Test credentials', + connector=app_settings.CONNECTORS[0][0], + params={'username': 'root', + 'password': 'password', + 'port': 22}) + opts.update(kwargs) + if 'organization' not in opts: + opts['organization'] = self._create_org() + c = Credentials(**opts) + c.full_clean() + c.save() + return c + + def _create_credentials_with_key(self, username='root', port=22, **kwargs): + opts = dict(name='Test SSH Key', + params={'username': username, + 'key': self._SSH_PRIVATE_KEY, + 'port': port}) + return self._create_credentials(**opts) + + def _create_device_connection(self, **kwargs): + opts = dict(enabled=True, + params={}) + opts.update(kwargs) + if 'credentials' not in opts: + opts['credentials'] = self._create_credentials() + org = opts['credentials'].organization + if 'device' not in opts: + opts['device'] = self._create_device(organization=org) + self._create_config(device=opts['device']) + dc = DeviceConnection(**opts) + dc.full_clean() + dc.save() + return dc + + def _create_device_ip(self, **kwargs): + opts = dict(address='10.40.0.1', + priority=1) + opts.update(kwargs) + if 'device' not in opts: + dc = self._create_device_connection() + opts['device'] = dc.device + ip = DeviceIp(**opts) + ip.full_clean() + ip.save() + return ip + + +class SshServerMixin(object): + _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') + _SSH_PRIVATE_KEY = None + + @classmethod + def setUpClass(cls): + with open(cls._TEST_RSA_KEY_PATH, 'r') as f: + cls._SSH_PRIVATE_KEY = f.read() + cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) + cls.ssh_server.__enter__() + + @classmethod + def tearDownClass(cls): + try: + cls.ssh_server.__exit__() + except OSError: + pass diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 4f8ec61c2..b9d0acc72 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -1,89 +1,14 @@ -import os - import paramiko -from django.conf import settings from django.core.exceptions import ValidationError from django.test import TestCase -from mockssh import Server as SshServer -from openwisp_controller.config.models import Config, Device -from openwisp_controller.config.tests import CreateConfigTemplateMixin - -from openwisp_users.tests.utils import TestOrganizationMixin from .. import settings as app_settings -from ..models import Credentials, DeviceConnection, DeviceIp +from ..models import Credentials, DeviceIp from ..utils import get_interfaces +from .base import CreateConnectionsMixin, SshServerMixin -class TestConnectionMixin(CreateConfigTemplateMixin, TestOrganizationMixin): - device_model = Device - config_model = Config - _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') - with open(_TEST_RSA_KEY_PATH, 'r') as f: - _SSH_PRIVATE_KEY = f.read() - - @classmethod - def setUpClass(cls): - cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) - cls.ssh_server.__enter__() - - @classmethod - def tearDownClass(cls): - try: - cls.ssh_server.__exit__() - except OSError: - pass - - def _create_credentials(self, **kwargs): - opts = dict(name='Test credentials', - connector=app_settings.CONNECTORS[0][0], - params={'username': 'root', - 'password': 'password', - 'port': 22}) - opts.update(kwargs) - if 'organization' not in opts: - opts['organization'] = self._create_org() - c = Credentials(**opts) - c.full_clean() - c.save() - return c - - def _create_credentials_with_key(self, username='root', port=22, **kwargs): - opts = dict(name='Test SSH Key', - params={'username': username, - 'key': self._SSH_PRIVATE_KEY, - 'port': port}) - return self._create_credentials(**opts) - - def _create_device_connection(self, **kwargs): - opts = dict(enabled=True, - params={}) - opts.update(kwargs) - if 'credentials' not in opts: - opts['credentials'] = self._create_credentials() - org = opts['credentials'].organization - if 'device' not in opts: - opts['device'] = self._create_device(organization=org) - self._create_config(device=opts['device']) - dc = DeviceConnection(**opts) - dc.full_clean() - dc.save() - return dc - - def _create_device_ip(self, **kwargs): - opts = dict(address='10.40.0.1', - priority=1) - opts.update(kwargs) - if 'device' not in opts: - dc = self._create_device_connection() - opts['device'] = dc.device - ip = DeviceIp(**opts) - ip.full_clean() - ip.save() - return ip - - -class TestModels(TestConnectionMixin, TestCase): +class TestModels(SshServerMixin, CreateConnectionsMixin, TestCase): def test_connection_str(self): c = Credentials(name='Dev Key', connector=app_settings.CONNECTORS[0][0]) self.assertIn(c.name, str(c)) @@ -127,7 +52,7 @@ def test_device_connection_auto_update_strategy_missing_config(self): try: self._create_device_connection(device=device) except ValidationError as e: - self.assertIn('inferred from', str(e)) + self.assertIn('inferred from', str(e)) else: self.fail('ValidationError not raised') From d32877c5e45c5a5e5832450623268187bafd4718 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 15 Jun 2018 08:40:53 +0200 Subject: [PATCH 39/55] [connection] Enforce multitenancy in DeviceConnection.credentials --- openwisp_controller/connection/admin.py | 10 +++++++++- openwisp_controller/connection/models.py | 6 ++++++ openwisp_controller/connection/tests/base.py | 5 ++++- .../connection/tests/test_models.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py index 9175fca03..b8c29f34a 100644 --- a/openwisp_controller/connection/admin.py +++ b/openwisp_controller/connection/admin.py @@ -25,11 +25,19 @@ def get_queryset(self, request): return qs.order_by('priority') -class DeviceConnectionInline(admin.StackedInline): +class DeviceConnectionInline(MultitenantAdminMixin, admin.StackedInline): model = DeviceConnection exclude = ['params', 'created', 'modified'] readonly_fields = ['is_working', 'failure_reason', 'last_attempt'] extra = 0 + multitenant_shared_relations = ('credentials',) + + def get_queryset(self, request): + """ + Override MultitenantAdminMixin.get_queryset() because it breaks + """ + return super(admin.StackedInline, self).get_queryset(request) + DeviceAdmin.inlines += [DeviceConnectionInline, DeviceIpInline] diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index d665398ce..39b4de6e7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -102,6 +102,12 @@ class Meta: verbose_name_plural = _('Device connections') def clean(self): + cred_org = self.credentials.organization + if cred_org and cred_org != self.device.organization: + raise ValidationError({ + 'credentials': _('The organization of these credentials doesn\'t ' + 'match the organization of the device') + }) if not self.update_strategy and hasattr(self.device, 'config'): try: self.update_strategy = app_settings.CONFIG_UPDATE_MAPPING[self.device.config.backend] diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py index a57464e4f..5b254a9e8 100644 --- a/openwisp_controller/connection/tests/base.py +++ b/openwisp_controller/connection/tests/base.py @@ -41,7 +41,10 @@ def _create_device_connection(self, **kwargs): params={}) opts.update(kwargs) if 'credentials' not in opts: - opts['credentials'] = self._create_credentials() + cred_opts = {} + if 'device' in opts: + cred_opts = {'organization': opts['device'].organization} + opts['credentials'] = self._create_credentials(**cred_opts) org = opts['credentials'].organization if 'device' not in opts: opts['device'] = self._create_device(organization=org) diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index b9d0acc72..f52b9060a 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -163,3 +163,21 @@ def test_address_list_link_local_ip(self): interfaces = get_interfaces() self.assertEqual(len(address_list), len(interfaces)) self.assertIn(ipv6_linklocal, address_list[0]) + + def test_device_connection_credential_org_validation(self): + dc = self._create_device_connection() + shared = self._create_credentials(name='cred-shared', + organization=None) + dc.credentials = shared + dc.full_clean() + # ensure credentials of other orgs aren't accepted + org2 = self._create_org(name='org2') + cred2 = self._create_credentials(name='cred2', + organization=org2) + try: + dc.credentials = cred2 + dc.full_clean() + except ValidationError as e: + self.assertIn('credentials', e.message_dict) + else: + self.fail('ValidationError not raised') From 7366f9149a9b50ab1d92cd107992035215a34ddb Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 2 Dec 2018 19:23:12 +0100 Subject: [PATCH 40/55] [connections] Added support for management interface --- openwisp_controller/config/views.py | 2 +- openwisp_controller/connection/models.py | 10 ++++----- .../connection/tests/test_models.py | 21 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/openwisp_controller/config/views.py b/openwisp_controller/config/views.py index a63b8cec9..9cabf590f 100644 --- a/openwisp_controller/config/views.py +++ b/openwisp_controller/config/views.py @@ -13,7 +13,7 @@ def get_default_templates(request, organization_id): """ user = request.user authenticated = user.is_authenticated - if callable(authenticated): + if callable(authenticated): # pragma: nocover authenticated = authenticated() if not authenticated and not user.is_staff: return HttpResponse(status=403) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 39b4de6e7..6041ca0b7 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -2,7 +2,7 @@ import ipaddress import logging -from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible @@ -142,10 +142,10 @@ def get_addresses(self): else: for interface in get_interfaces(): address_list.append('{0}%{1}'.format(address, interface)) - try: - address_list.append(self.device.config.last_ip) - except ObjectDoesNotExist: - pass + if self.device.management_ip: + address_list.append(self.device.management_ip) + if self.device.last_ip: + address_list.append(self.device.last_ip) return address_list def get_params(self): diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index f52b9060a..63c760b1a 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -129,9 +129,13 @@ def test_device_connection_schema(self): else: self.fail('ValidationError not raised') - def _prepare_address_list_test(self, addresses): + def _prepare_address_list_test(self, addresses, + last_ip=None, + management_ip=None): update_strategy = app_settings.UPDATE_STRATEGIES[0][0] - device = self._create_device(organization=self._create_org()) + device = self._create_device(organization=self._create_org(), + last_ip=last_ip, + management_ip=management_ip) dc = self._create_device_connection(device=device, update_strategy=update_strategy) for index, address in enumerate(addresses): @@ -147,13 +151,16 @@ def test_address_list(self): '192.168.40.1' ]) - def test_address_list_with_config_last_ip(self): - dc = self._prepare_address_list_test(['192.168.40.1']) - self._create_config(device=dc.device, - last_ip='10.40.0.2') + def test_address_list_with_device_ip(self): + dc = self._prepare_address_list_test( + ['192.168.40.1'], + management_ip='10.0.0.2', + last_ip='84.32.46.153', + ) self.assertEqual(dc.get_addresses(), [ '192.168.40.1', - '10.40.0.2', + '10.0.0.2', + '84.32.46.153' ]) def test_address_list_link_local_ip(self): From a5e89e8e460cd0e74c7e8f2ffe3ebb7550696113 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 2 Dec 2018 19:31:53 +0100 Subject: [PATCH 41/55] [connections] Updated deprecated set_status_running to set_status_applied --- openwisp_controller/connection/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 6041ca0b7..872524592 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -177,7 +177,7 @@ def update_config(self): except Exception as e: logger.exception(e) else: - self.device.config.set_status_running() + self.device.config.set_status_applied() self.disconnect() From 7477e7594c58371ba5bfc4e8fe71b86fb507645a Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 6 Dec 2018 14:33:54 +0100 Subject: [PATCH 42/55] [connection] Added auto_add feature to Credentials --- openwisp_controller/connection/admin.py | 10 +++- openwisp_controller/connection/apps.py | 34 +++++++---- .../migrations/0002_credentials_auto_add.py | 18 ++++++ openwisp_controller/connection/models.py | 60 +++++++++++++++++++ .../connection/tests/test_models.py | 60 +++++++++++++++++++ 5 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 openwisp_controller/connection/migrations/0002_credentials_auto_add.py diff --git a/openwisp_controller/connection/admin.py b/openwisp_controller/connection/admin.py index b8c29f34a..41fbd3254 100644 --- a/openwisp_controller/connection/admin.py +++ b/openwisp_controller/connection/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from openwisp_utils.admin import MultitenantOrgFilter, TimeReadonlyAdminMixin +from openwisp_users.multitenancy import MultitenantOrgFilter +from openwisp_utils.admin import TimeReadonlyAdminMixin from ..admin import MultitenantAdminMixin from ..config.admin import DeviceAdmin @@ -9,7 +10,12 @@ @admin.register(Credentials) class CredentialsAdmin(MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.ModelAdmin): - list_display = ('name', 'organization', 'connector', 'created', 'modified') + list_display = ('name', + 'organization', + 'connector', + 'auto_add', + 'created', + 'modified') list_filter = [('organization', MultitenantOrgFilter), 'connector'] list_select_related = ('organization',) diff --git a/openwisp_controller/connection/apps.py b/openwisp_controller/connection/apps.py index 1a4e84c45..6dc7b9366 100644 --- a/openwisp_controller/connection/apps.py +++ b/openwisp_controller/connection/apps.py @@ -1,5 +1,6 @@ from celery.task.control import inspect from django.apps import AppConfig +from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ from django_netjsonconfig.signals import config_modified @@ -17,22 +18,29 @@ def ready(self): to the ``update_config`` celery task which will be executed in the background """ - from .tasks import update_config + config_modified.connect(self.config_modified_receiver, + dispatch_uid='connection.update_config') + + from ..config.models import Config + from .models import Credentials - def config_modified_receiver(**kwargs): - d = kwargs['device'] - conn_count = d.deviceconnection_set.count() - # if device has no connection specified - # or update is already in progress, stop here - if conn_count < 1 or self._is_update_in_progress(d.id): - return - update_config.delay(d.id) + post_save.connect(Credentials.auto_add_credentials_to_device, + sender=Config, + dispatch_uid='connection.auto_add_credentials') - config_modified.connect(config_modified_receiver, - dispatch_uid='connection.update_config', - weak=False) + @classmethod + def config_modified_receiver(cls, **kwargs): + from .tasks import update_config + d = kwargs['device'] + conn_count = d.deviceconnection_set.count() + # if device has no connection specified + # or update is already in progress, stop here + if conn_count < 1 or cls._is_update_in_progress(d.id): + return + update_config.delay(d.id) - def _is_update_in_progress(self, device_id): + @classmethod + def _is_update_in_progress(cls, device_id): active = inspect().active() if not active: return False diff --git a/openwisp_controller/connection/migrations/0002_credentials_auto_add.py b/openwisp_controller/connection/migrations/0002_credentials_auto_add.py new file mode 100644 index 000000000..d7373f79e --- /dev/null +++ b/openwisp_controller/connection/migrations/0002_credentials_auto_add.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.3 on 2018-12-05 13:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('connection', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='credentials', + name='auto_add', + field=models.BooleanField(default=False, help_text='automatically add these credentials to the devices of this organization; if no organization is specified will be added to all the new devices', verbose_name='auto add'), + ), + ] diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 872524592..9f48fa062 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -17,6 +17,7 @@ from openwisp_utils.base import TimeStampedEditableModel from . import settings as app_settings +from ..config.models import Device from .utils import get_interfaces logger = logging.getLogger(__name__) @@ -63,6 +64,12 @@ class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): help_text=_('global connection parameters'), load_kwargs={'object_pairs_hook': collections.OrderedDict}, dump_kwargs={'indent': 4}) + auto_add = models.BooleanField(_('auto add'), + default=False, + help_text=_('automatically add these credentials ' + 'to the devices of this organization; ' + 'if no organization is specified will ' + 'be added to all the new devices')) class Meta: verbose_name = _('Access credentials') @@ -71,6 +78,59 @@ class Meta: def __str__(self): return '{0} ({1})'.format(self.name, self.get_connector_display()) + def save(self, *args, **kwargs): + super(Credentials, self).save(*args, **kwargs) + self.auto_add_to_devices() + + def auto_add_to_devices(self): + """ + When ``auto_add`` is ``True``, adds the credentials + to each relevant ``Device`` and ``DeviceConnection`` objects + """ + if not self.auto_add: + return + devices = Device.objects.all() + org = self.organization + if org: + devices = devices.filter(organization=org) + # exclude devices which have been already added + devices = devices.exclude(deviceconnection__credentials=self) + for device in devices: + conn = DeviceConnection(device=device, + credentials=self, + enabled=True) + conn.full_clean() + conn.save() + + @classmethod + def auto_add_credentials_to_device(cls, instance, created, **kwargs): + """ + Adds relevant credentials as ``DeviceConnection`` + when a device is created, this is called from a + post_save signal receiver hooked to the ``Config`` model + (why ``Config`` and not ``Device``? because at the moment + we can automatically create a DeviceConnection if we have + a ``Config`` object) + """ + if not created: + return + device = instance.device + # select credentials which + # - are flagged as auto_add + # - belong to the same organization of the device + # OR + # belong to no organization (hence are shared) + conditions = (models.Q(organization=device.organization) | + models.Q(organization=None)) + credentials = cls.objects.filter(conditions) \ + .filter(auto_add=True) + for cred in credentials: + conn = DeviceConnection(device=device, + credentials=cred, + enabled=True) + conn.full_clean() + conn.save() + class DeviceConnection(ConnectorMixin, TimeStampedEditableModel): _connector_field = 'update_strategy' diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 63c760b1a..e10d137ba 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -2,6 +2,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase +from openwisp_users.models import Organization + from .. import settings as app_settings from ..models import Credentials, DeviceIp from ..utils import get_interfaces @@ -188,3 +190,61 @@ def test_device_connection_credential_org_validation(self): self.assertIn('credentials', e.message_dict) else: self.fail('ValidationError not raised') + + def test_auto_add_to_new_device(self): + c = self._create_credentials(auto_add=True, + organization=None) + self._create_credentials(name='cred2', + auto_add=False, + organization=None) + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + + def test_auto_add_to_existing_device_on_creation(self): + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + self.assertEqual(d.deviceconnection_set.count(), 0) + c = self._create_credentials(auto_add=True, + organization=None) + org2 = Organization.objects.create(name='org2', slug='org2') + self._create_credentials(name='cred2', + auto_add=True, + organization=org2) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + self._create_credentials(name='cred3', + auto_add=False, + organization=None) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + + def test_auto_add_to_existing_device_on_edit(self): + d = self._create_device(organization=Organization.objects.first()) + self._create_config(device=d) + self.assertEqual(d.deviceconnection_set.count(), 0) + c = self._create_credentials(auto_add=False, + organization=None) + org2 = Organization.objects.create(name='org2', slug='org2') + self._create_credentials(name='cred2', + auto_add=True, + organization=org2) + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 0) + c.auto_add = True + c.full_clean() + c.save() + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) + # ensure further edits are idempotent + c.name = 'changed' + c.full_clean() + c.save() + d.refresh_from_db() + self.assertEqual(d.deviceconnection_set.count(), 1) + self.assertEqual(d.deviceconnection_set.first().credentials, c) From a8a2af742badc4444e79b19667c620e9ef35c708 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sat, 8 Dec 2018 18:13:10 +0100 Subject: [PATCH 43/55] [connection] Upgraded celery to 4.2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bee535c7b..c2dbe51bb 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ # TODO: make optional? "paramiko>=2.4.1,<2.5.0", "scp>=0.13.0,<0.14.0", - "celery>=4.1.0,<4.2.0" + "celery>=4.2.0,<4.3.0" ], classifiers=[ 'Development Status :: 3 - Alpha', From bb49a480e4139d8987865bd1b07b0d38b61096a3 Mon Sep 17 00:00:00 2001 From: Noumbissi valere Gille Geovan Date: Sun, 14 Apr 2019 04:03:54 +0100 Subject: [PATCH 44/55] [tests] Added admin tests --- openwisp_controller/connection/models.py | 2 +- openwisp_controller/connection/tests/base.py | 1 + .../connection/tests/test_admin.py | 118 ++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 openwisp_controller/connection/tests/test_admin.py diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 9f48fa062..486ad860c 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -16,8 +16,8 @@ from openwisp_users.mixins import ShareableOrgMixin from openwisp_utils.base import TimeStampedEditableModel -from . import settings as app_settings from ..config.models import Device +from . import settings as app_settings from .utils import get_interfaces logger = logging.getLogger(__name__) diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py index 5b254a9e8..cfcbd1fed 100644 --- a/openwisp_controller/connection/tests/base.py +++ b/openwisp_controller/connection/tests/base.py @@ -34,6 +34,7 @@ def _create_credentials_with_key(self, username='root', port=22, **kwargs): params={'username': username, 'key': self._SSH_PRIVATE_KEY, 'port': port}) + opts.update(kwargs) return self._create_credentials(**opts) def _create_device_connection(self, **kwargs): diff --git a/openwisp_controller/connection/tests/test_admin.py b/openwisp_controller/connection/tests/test_admin.py new file mode 100644 index 000000000..399e065fd --- /dev/null +++ b/openwisp_controller/connection/tests/test_admin.py @@ -0,0 +1,118 @@ +import json + +from django.test import TestCase +from django.urls import reverse + +from ...config.models import Template +from ...config.tests.test_admin import TestAdmin as TestConfigAdmin +from ...tests.utils import TestAdminMixin +from .. import settings as app_settings +from ..models import Credentials, DeviceConnection, DeviceIp +from .base import CreateConnectionsMixin, SshServerMixin + + +class TestAdmin(TestAdminMixin, CreateConnectionsMixin, + SshServerMixin, TestCase): + template_model = Template + credentials_model = Credentials + deviceip_model = DeviceIp + connection_model = DeviceConnection + operator_permission_filters = [ + {'codename__endswith': 'config'}, + {'codename__endswith': 'device'}, + {'codename__endswith': 'template'}, + {'codename__endswith': 'connection'}, + {'codename__endswith': 'credentials'}, + {'codename__endswith': 'device_connection'}, + {'codename__endswith': 'device_ip'}, + ] + _device_params = TestConfigAdmin._device_params.copy() + + def _get_device_params(self, org): + p = self._device_params.copy() + p['organization'] = org.pk + return p + + def test_device_config_update(self): + org1 = self._create_org(name='org1') + cred = self._create_credentials_with_key(organization=org1, port=self.ssh_server.port) + device = self._create_device(organization=org1) + update_strategy = app_settings.UPDATE_STRATEGIES[0][0] + c = self._create_config(device=device) + dc = self._create_device_connection(device=device, + credentials=cred, + update_strategy=update_strategy) + self._create_device_ip(device=device, + address=self.ssh_server.host) + uid = self.ssh_server.users + config = json.dumps({ + 'interfaces': [ + { + 'name': 'eth0', + 'type': 'ethernet', + 'addresses': [ + { + 'family': 'ipv4', + 'proto': 'dhcp' + } + ] + } + ] + }) + c.config = config + dc.update_config() + self.ssh_server.client(*uid).exec_command('/etc/init.d/openwisp_config restart') + self.assertEqual(device.status, 'applied') + + def _create_multitenancy_test_env(self): + org1 = self._create_org(name='test1org') + org2 = self._create_org(name='test2org') + inactive = self._create_org(name='inactive-org', is_active=False) + operator = self._create_operator(organizations=[org1, inactive]) + cred1 = self._create_credentials(organization=org1, name='test1cred') + cred2 = self._create_credentials(organization=org2, name='test2cred') + cred3 = self._create_credentials(organization=inactive, name='test3cred') + dc1 = self._create_device_connection(credentials=cred1) + dc2 = self._create_device_connection(credentials=cred2) + dc3 = self._create_device_connection(credentials=cred3) + data = dict(cred1=cred1, cred2=cred2, cred3_inactive=cred3, + dc1=dc1, dc2=dc2, dc3_inactive=dc3, + org1=org1, org2=org2, inactive=inactive, + operator=operator) + return data + + def test_credentials_queryset(self): + data = self._create_multitenancy_test_env() + self._test_multitenant_admin( + url=reverse('admin:connection_credentials_changelist'), + visible=[data['cred1'].name, data['org1'].name], + hidden=[data['cred2'].name, data['org2'].name, + data['cred3_inactive'].name] + ) + + def test_credentials_organization_fk_queryset(self): + data = self._create_multitenancy_test_env() + self._test_multitenant_admin( + url=reverse('admin:connection_credentials_add'), + visible=[data['org1'].name], + hidden=[data['org2'].name, data['inactive']], + select_widget=True + ) + + def test_connection_queryset(self): + data = self._create_multitenancy_test_env() + self._test_multitenant_admin( + url=reverse('admin:connection_credentials_changelist'), + visible=[data['dc1'].credentials.name, data['org1'].name], + hidden=[data['dc2'].credentials.name, data['org2'].name, + data['dc3_inactive'].credentials.name] + ) + + def test_connection_credentials_fk_queryset(self): + data = self._create_multitenancy_test_env() + self._test_multitenant_admin( + url=reverse('admin:config_device_add'), + visible=[data['cred1'].name + ' (SSH)'], + hidden=[data['cred2'].name + ' (SSH)', data['cred3_inactive']], + select_widget=True + ) From 3575048e05f141308cf2a14999fb83d0fbf8fd86 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 14 Apr 2019 17:52:12 -0400 Subject: [PATCH 45/55] [tests] Moved test_device_config_update --- .../connection/tests/test_admin.py | 31 ---------------- .../connection/tests/test_models.py | 37 +++++++++++++++++++ 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/openwisp_controller/connection/tests/test_admin.py b/openwisp_controller/connection/tests/test_admin.py index 399e065fd..70e2bce0c 100644 --- a/openwisp_controller/connection/tests/test_admin.py +++ b/openwisp_controller/connection/tests/test_admin.py @@ -33,37 +33,6 @@ def _get_device_params(self, org): p['organization'] = org.pk return p - def test_device_config_update(self): - org1 = self._create_org(name='org1') - cred = self._create_credentials_with_key(organization=org1, port=self.ssh_server.port) - device = self._create_device(organization=org1) - update_strategy = app_settings.UPDATE_STRATEGIES[0][0] - c = self._create_config(device=device) - dc = self._create_device_connection(device=device, - credentials=cred, - update_strategy=update_strategy) - self._create_device_ip(device=device, - address=self.ssh_server.host) - uid = self.ssh_server.users - config = json.dumps({ - 'interfaces': [ - { - 'name': 'eth0', - 'type': 'ethernet', - 'addresses': [ - { - 'family': 'ipv4', - 'proto': 'dhcp' - } - ] - } - ] - }) - c.config = config - dc.update_config() - self.ssh_server.client(*uid).exec_command('/etc/init.d/openwisp_config restart') - self.assertEqual(device.status, 'applied') - def _create_multitenancy_test_env(self): org1 = self._create_org(name='test1org') org2 = self._create_org(name='test2org') diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index e10d137ba..e1e950455 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -248,3 +248,40 @@ def test_auto_add_to_existing_device_on_edit(self): d.refresh_from_db() self.assertEqual(d.deviceconnection_set.count(), 1) self.assertEqual(d.deviceconnection_set.first().credentials, c) + + def test_device_config_update(self): + org1 = self._create_org(name='org1') + cred = self._create_credentials_with_key(organization=org1, port=self.ssh_server.port) + device = self._create_device(organization=org1) + update_strategy = app_settings.UPDATE_STRATEGIES[0][0] + c = self._create_config(device=device) + self._create_device_connection(device=device, + credentials=cred, + update_strategy=update_strategy) + self._create_device_ip(device=device, + address=self.ssh_server.host) + c.config = { + 'interfaces': [ + { + 'name': 'eth0', + 'type': 'ethernet', + 'addresses': [ + { + 'family': 'ipv4', + 'proto': 'dhcp' + } + ] + } + ] + } + # here you can start capturing standard output + # see how it's done here: https://github.com/ncouture/MockSSH/blob/master/tests/test_mock_cisco.py#L14-L20 + c.full_clean() + c.save() + # at this point the update_config() method will be triggered + # you need to create a command in the mocked SSH server that + # just prints something like: mock-ssh-server: openwisp-config restarted + # so you can then do: + # self.assertIn('mock-ssh-server: openwisp-config restarted', sdout) + # if update_config() finishes successfully then status will be set to "applied" + self.assertEqual(device.status, 'applied') From 21f1f4ad08159a2e13b3d780bb98c09b3583733d Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 14 Apr 2019 17:56:36 -0400 Subject: [PATCH 46/55] [connections] Commented out Ssh.upload This method is not used yet but it will be in the future. For the moment we can avoid adding tests for it and keep it commented out. --- .../connection/connectors/ssh.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index c2f59d7c0..24b52dfad 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -122,11 +122,13 @@ def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, def update_config(self): raise NotImplementedError() - def upload(self, fl, remote_path): - scp = SCPClient(self.shell.get_transport()) - if not hasattr(fl, 'getvalue'): - fl_memory = BytesIO(fl.read()) - fl.seek(0) - fl = fl_memory - scp.putfo(fl, remote_path) - scp.close() + # TODO: this method is not used yet + # but will be necessary in the future to support other OSes + # def upload(self, fl, remote_path): + # scp = SCPClient(self.shell.get_transport()) + # if not hasattr(fl, 'getvalue'): + # fl_memory = BytesIO(fl.read()) + # fl.seek(0) + # fl = fl_memory + # scp.putfo(fl, remote_path) + # scp.close() From c23336665907c2b114146a4a91c211aa8cfe574b Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 14 Apr 2019 17:57:05 -0400 Subject: [PATCH 47/55] [connections] Corrected "Previus command failed" message --- openwisp_controller/connection/connectors/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 24b52dfad..4c7669789 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -114,7 +114,7 @@ def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, # abort the operation if any of the command # returned with a non-zero exit status if exit_status not in exit_codes and raise_unexpected_exit: - print('# Previus command failed, aborting upgrade...') + print('# Previus command failed, aborting...') message = error if error else output raise Exception(message) return output, exit_status From 571999502f834915d142a9741e269214d8ce6fb7 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Sun, 14 Apr 2019 18:00:51 -0400 Subject: [PATCH 48/55] [connections] Excluded not implemented method from test coverage --- openwisp_controller/connection/connectors/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 4c7669789..e4304a717 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -119,7 +119,7 @@ def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, raise Exception(message) return output, exit_status - def update_config(self): + def update_config(self): # pragma: no cover raise NotImplementedError() # TODO: this method is not used yet From d760329698d5d0d962160514198c6a3c6b521e72 Mon Sep 17 00:00:00 2001 From: Noumbissi valere Gille Geovan Date: Fri, 19 Apr 2019 06:44:10 +0100 Subject: [PATCH 49/55] [tests] Fixed model tests --- .../connection/connectors/ssh.py | 2 - .../connection/tests/test_admin.py | 7 +-- .../connection/tests/test_models.py | 60 ++++++++++++++++--- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index e4304a717..5fbfb3f1a 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -1,13 +1,11 @@ import logging import socket import sys -from io import BytesIO import paramiko from django.utils.functional import cached_property from jsonschema import validate from jsonschema.exceptions import ValidationError as SchemaError -from scp import SCPClient if sys.version_info.major > 2: # pragma: nocover from io import StringIO diff --git a/openwisp_controller/connection/tests/test_admin.py b/openwisp_controller/connection/tests/test_admin.py index 70e2bce0c..3133a5c01 100644 --- a/openwisp_controller/connection/tests/test_admin.py +++ b/openwisp_controller/connection/tests/test_admin.py @@ -1,12 +1,9 @@ -import json - from django.test import TestCase from django.urls import reverse from ...config.models import Template from ...config.tests.test_admin import TestAdmin as TestConfigAdmin from ...tests.utils import TestAdminMixin -from .. import settings as app_settings from ..models import Credentials, DeviceConnection, DeviceIp from .base import CreateConnectionsMixin, SshServerMixin @@ -81,7 +78,7 @@ def test_connection_credentials_fk_queryset(self): data = self._create_multitenancy_test_env() self._test_multitenant_admin( url=reverse('admin:config_device_add'), - visible=[data['cred1'].name + ' (SSH)'], - hidden=[data['cred2'].name + ' (SSH)', data['cred3_inactive']], + visible=[str(data['cred1'].name) + str(" (SSH)")], + hidden=[str(data['cred2'].name) + str(" (SSH)"), data['cred3_inactive']], select_widget=True ) diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index e1e950455..48eabd5dd 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -1,6 +1,7 @@ import paramiko from django.core.exceptions import ValidationError from django.test import TestCase +from mock import Mock, patch from openwisp_users.models import Organization @@ -274,14 +275,55 @@ def test_device_config_update(self): } ] } - # here you can start capturing standard output - # see how it's done here: https://github.com/ncouture/MockSSH/blob/master/tests/test_mock_cisco.py#L14-L20 c.full_clean() c.save() - # at this point the update_config() method will be triggered - # you need to create a command in the mocked SSH server that - # just prints something like: mock-ssh-server: openwisp-config restarted - # so you can then do: - # self.assertIn('mock-ssh-server: openwisp-config restarted', sdout) - # if update_config() finishes successfully then status will be set to "applied" - self.assertEqual(device.status, 'applied') + device.refresh_from_db() + self.assertEqual(device.status, 'modified') + c.config = { + 'interfaces': [ + { + 'name': 'eth10', + 'type': 'ethernet', + 'addresses': [ + { + 'family': 'ipv4', + 'proto': 'dhcp' + } + ] + } + ] + } + stdin = Mock() + stdout = Mock() + stderr = Mock() + stdout.read().decode('utf8').strip.return_value = \ + 'Mock Object: executed command successfully' + stdout.channel.recv_exit_status.return_value = 0 + stderr.read().decode('utf8').strip.return_value = '' + with patch('paramiko.SSHClient.exec_command') as mock_object: + mock_object.return_value = [stdin, stdout, stderr] + c.full_clean() + c.save() + device.refresh_from_db() + self.assertEqual(paramiko.SSHClient.exec_command.call_count, 1) + actual = str(paramiko.SSHClient.exec_command.call_args) + expected = str("call('/etc/init.d/openwisp_config restart')") + self.assertEqual(expected, actual) + self.assertEqual(device.status, 'applied') + + def test_ssh_exec_exist_status(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connect() + _, exit_status = dc.connector_instance.exec_command('ls') + self.assertEqual(exit_status, 0) + + def test_ssh_exec_exception(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + with self.assertRaises(Exception): + dc.connector_instance.exec_command('ls') From 89009586dda5c7e1d0105c6049a45d2e1cd8770a Mon Sep 17 00:00:00 2001 From: Noumbissi valere Gille Geovan Date: Fri, 19 Apr 2019 06:44:43 +0100 Subject: [PATCH 50/55] [update] Added mock as a required package --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c2dbe51bb..78e3394eb 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ # TODO: make optional? "paramiko>=2.4.1,<2.5.0", "scp>=0.13.0,<0.14.0", - "celery>=4.2.0,<4.3.0" + "celery>=4.2.0,<4.3.0", + "mock==2.0.0" ], classifiers=[ 'Development Status :: 3 - Alpha', From c0d99f9a73d9e19fe4dc2d174c9c9faca5100bd0 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 22 Apr 2019 15:56:30 -0400 Subject: [PATCH 51/55] [connections] Improved tests for SSH connector --- openwisp_controller/connection/tests/base.py | 16 +++- .../connection/tests/test_models.py | 92 ++++++++++--------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/openwisp_controller/connection/tests/base.py b/openwisp_controller/connection/tests/base.py index cfcbd1fed..b434ed8ae 100644 --- a/openwisp_controller/connection/tests/base.py +++ b/openwisp_controller/connection/tests/base.py @@ -1,7 +1,7 @@ import os from django.conf import settings -from mockssh import Server as SshServer +from mockssh.server import Server as BaseSshServer from openwisp_controller.config.models import Config, Device from openwisp_controller.config.tests import CreateConfigTemplateMixin @@ -68,12 +68,25 @@ def _create_device_ip(self, **kwargs): return ip +class SshServer(BaseSshServer): + is_teardown = False + + def _run(self): + try: + super(SshServer, self)._run() + # do not raise exceptions during tear down + except Exception as e: + if not self.is_teardown: + raise e + + class SshServerMixin(object): _TEST_RSA_KEY_PATH = os.path.join(settings.BASE_DIR, 'test-key.rsa') _SSH_PRIVATE_KEY = None @classmethod def setUpClass(cls): + super(SshServerMixin, cls).setUpClass() with open(cls._TEST_RSA_KEY_PATH, 'r') as f: cls._SSH_PRIVATE_KEY = f.read() cls.ssh_server = SshServer({'root': cls._TEST_RSA_KEY_PATH}) @@ -81,6 +94,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + cls.ssh_server.is_teardown = True try: cls.ssh_server.__exit__() except OSError: diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 48eabd5dd..9e95a46bf 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -1,7 +1,9 @@ +import socket + +import mock import paramiko from django.core.exceptions import ValidationError from django.test import TestCase -from mock import Mock, patch from openwisp_users.models import Organization @@ -250,6 +252,19 @@ def test_auto_add_to_existing_device_on_edit(self): self.assertEqual(d.deviceconnection_set.count(), 1) self.assertEqual(d.deviceconnection_set.first().credentials, c) + _exec_command_path = 'paramiko.SSHClient.exec_command' + + def _exec_command_return_value(self, stdin='', stdout='mocked', + stderr='', exit_code=0): + stdin_ = mock.Mock() + stdout_ = mock.Mock() + stderr_ = mock.Mock() + stdin_.read().decode('utf8').strip.return_value = stdin + stdout_.read().decode('utf8').strip.return_value = stdout + stdout_.channel.recv_exit_status.return_value = exit_code + stderr_.read().decode('utf8').strip.return_value = stderr + return (stdin_, stdout_, stderr_) + def test_device_config_update(self): org1 = self._create_org(name='org1') cred = self._create_credentials_with_key(organization=org1, port=self.ssh_server.port) @@ -261,24 +276,6 @@ def test_device_config_update(self): update_strategy=update_strategy) self._create_device_ip(device=device, address=self.ssh_server.host) - c.config = { - 'interfaces': [ - { - 'name': 'eth0', - 'type': 'ethernet', - 'addresses': [ - { - 'family': 'ipv4', - 'proto': 'dhcp' - } - ] - } - ] - } - c.full_clean() - c.save() - device.refresh_from_db() - self.assertEqual(device.status, 'modified') c.config = { 'interfaces': [ { @@ -293,37 +290,50 @@ def test_device_config_update(self): } ] } - stdin = Mock() - stdout = Mock() - stderr = Mock() - stdout.read().decode('utf8').strip.return_value = \ - 'Mock Object: executed command successfully' - stdout.channel.recv_exit_status.return_value = 0 - stderr.read().decode('utf8').strip.return_value = '' - with patch('paramiko.SSHClient.exec_command') as mock_object: - mock_object.return_value = [stdin, stdout, stderr] - c.full_clean() + c.full_clean() + + with mock.patch(self._exec_command_path) as mocked: + mocked.return_value = self._exec_command_return_value() c.save() - device.refresh_from_db() - self.assertEqual(paramiko.SSHClient.exec_command.call_count, 1) - actual = str(paramiko.SSHClient.exec_command.call_args) - expected = str("call('/etc/init.d/openwisp_config restart')") - self.assertEqual(expected, actual) - self.assertEqual(device.status, 'applied') + mocked.assert_called_once() + device.refresh_from_db() + self.assertEqual(device.status, 'applied') - def test_ssh_exec_exist_status(self): + def test_ssh_exec_exit_code(self): ckey = self._create_credentials_with_key(port=self.ssh_server.port) dc = self._create_device_connection(credentials=ckey) self._create_device_ip(address=self.ssh_server.host, device=dc.device) - dc.connect() - _, exit_status = dc.connector_instance.exec_command('ls') - self.assertEqual(exit_status, 0) + dc.connector_instance.connect() + with mock.patch(self._exec_command_path) as mocked: + mocked.return_value = self._exec_command_return_value(exit_code=1) + with self.assertRaises(Exception): + dc.connector_instance.exec_command('trigger_command_not_found') + dc.connector_instance.disconnect() + mocked.assert_called_once() + + def test_ssh_exec_timeout(self): + ckey = self._create_credentials_with_key(port=self.ssh_server.port) + dc = self._create_device_connection(credentials=ckey) + self._create_device_ip(address=self.ssh_server.host, + device=dc.device) + dc.connector_instance.connect() + with mock.patch(self._exec_command_path) as mocked: + mocked.side_effect = socket.timeout() + with self.assertRaises(socket.timeout): + dc.connector_instance.exec_command('trigger_timeout') + dc.connector_instance.disconnect() + mocked.assert_called_once() def test_ssh_exec_exception(self): ckey = self._create_credentials_with_key(port=self.ssh_server.port) dc = self._create_device_connection(credentials=ckey) self._create_device_ip(address=self.ssh_server.host, device=dc.device) - with self.assertRaises(Exception): - dc.connector_instance.exec_command('ls') + dc.connector_instance.connect() + with mock.patch(self._exec_command_path) as mocked: + mocked.side_effect = RuntimeError('test') + with self.assertRaises(RuntimeError): + dc.connector_instance.exec_command('trigger_exception') + dc.connector_instance.disconnect() + mocked.assert_called_once() From 026e2f6c9c166272b009726b035124f44f3c5af1 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 22 Apr 2019 16:04:20 -0400 Subject: [PATCH 52/55] [connections] Update status (running > applied) --- openwisp_controller/config/tests/test_controller.py | 2 +- openwisp_controller/connection/tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openwisp_controller/config/tests/test_controller.py b/openwisp_controller/config/tests/test_controller.py index b6cf43583..6a3d1daa7 100644 --- a/openwisp_controller/config/tests/test_controller.py +++ b/openwisp_controller/config/tests/test_controller.py @@ -126,7 +126,7 @@ def test_report_status_404_disabled_org(self): org = self._create_org(is_active=False) c = self._create_config(organization=org) response = self.client.post(reverse('controller:report_status', args=[c.device.pk]), - {'key': c.device.key, 'status': 'running'}) + {'key': c.device.key, 'status': 'applied'}) self.assertEqual(response.status_code, 404) def test_checksum_200(self): diff --git a/openwisp_controller/connection/tasks.py b/openwisp_controller/connection/tasks.py index 689e41570..ccf3151b7 100644 --- a/openwisp_controller/connection/tasks.py +++ b/openwisp_controller/connection/tasks.py @@ -18,7 +18,7 @@ def update_config(device_id): sleep(4) # avoid repeating the operation multiple times device = Device.objects.select_related('config').get(pk=device_id) - if device.config.status == 'running': + if device.config.status == 'applied': return qs = device.deviceconnection_set.filter(device_id=device_id, enabled=True) conn = qs.first() From 043644501d02a057b91e7ce50fe6db71e4260182 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 22 Apr 2019 16:05:05 -0400 Subject: [PATCH 53/55] [connections] Honor timeout --- openwisp_controller/connection/connectors/ssh.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openwisp_controller/connection/connectors/ssh.py b/openwisp_controller/connection/connectors/ssh.py index 5fbfb3f1a..86899f7d5 100644 --- a/openwisp_controller/connection/connectors/ssh.py +++ b/openwisp_controller/connection/connectors/ssh.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) SSH_CONNECTION_TIMEOUT = 5 SSH_AUTH_TIMEOUT = 2 -SSH_COMMAND_TIMEOUT = 5 +SSH_COMMAND_TIMEOUT = 30 class Ssh(object): @@ -90,7 +90,8 @@ def exec_command(self, command, timeout=SSH_COMMAND_TIMEOUT, print('$:> {0}'.format(command)) # execute commmand try: - stdin, stdout, stderr = self.shell.exec_command(command) + stdin, stdout, stderr = self.shell.exec_command(command, + timeout=timeout) # re-raise socket.timeout to avoid being catched # by the subsequent `except Exception as e` block except socket.timeout: From c5f89c65843f5dd22e9415d19aa83cac3acb6f1c Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 22 Apr 2019 16:25:45 -0400 Subject: [PATCH 54/55] [connections] Fixed remaining failing tests --- openwisp_controller/connection/models.py | 1 + openwisp_controller/connection/tests/test_models.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openwisp_controller/connection/models.py b/openwisp_controller/connection/models.py index 486ad860c..dfbc63e4b 100644 --- a/openwisp_controller/connection/models.py +++ b/openwisp_controller/connection/models.py @@ -51,6 +51,7 @@ def connector_instance(self): addresses=self.get_addresses()) +@python_2_unicode_compatible class Credentials(ConnectorMixin, ShareableOrgMixin, BaseModel): """ Credentials for access diff --git a/openwisp_controller/connection/tests/test_models.py b/openwisp_controller/connection/tests/test_models.py index 9e95a46bf..55e0fbade 100644 --- a/openwisp_controller/connection/tests/test_models.py +++ b/openwisp_controller/connection/tests/test_models.py @@ -296,8 +296,8 @@ def test_device_config_update(self): mocked.return_value = self._exec_command_return_value() c.save() mocked.assert_called_once() - device.refresh_from_db() - self.assertEqual(device.status, 'applied') + c.refresh_from_db() + self.assertEqual(c.status, 'applied') def test_ssh_exec_exit_code(self): ckey = self._create_credentials_with_key(port=self.ssh_server.port) From 0b1fd0a8e0b34855f3021cd6dc5c5a25be5de0f6 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Mon, 22 Apr 2019 16:45:18 -0400 Subject: [PATCH 55/55] [connections] Move mock dependency to dev-packages --- Pipfile | 1 + setup.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 59d8ca696..852de7262 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ coverage = "*" coveralls = "*" isort = "*" flake8 = "*" +mock = "*" mock-ssh-server = {version = ">=0.5.0,<0.6.0"} [scripts] diff --git a/setup.py b/setup.py index 78e3394eb..310bcbd7b 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,9 @@ "openwisp-utils[users]<0.3.1", "django-loci>=0.1.1,<0.3.0", "djangorestframework-gis>=0.12.0,<0.14.0", - # connections feature - # TODO: make optional? "paramiko>=2.4.1,<2.5.0", "scp>=0.13.0,<0.14.0", "celery>=4.2.0,<4.3.0", - "mock==2.0.0" ], classifiers=[ 'Development Status :: 3 - Alpha',