Merge ~aluria/charm-openstack-service-checks/+git/charm-openstack-service-checks:bug/1821316-rewrite into ~canonical-bootstack/charm-openstack-service-checks:master

[Alvaro Uria]

Rewrite to integrate template-python-pytest code and approach

* separation between charm logic (reactive) and helper functions (lib)
* improvement on charm states and messages
* unit and functional tests added (using pytest framework)

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Unable to determine commit message from repository - please click "Set commit message" and enter the commit message manually.

[Alvaro Uria (aluria)]

Code has already been reviewed at [1][2][3].

I have also tested a merge to master, and as we see in this MP, 2 minor conflicts were raised:
a) config.yaml: swift_check_params didn't have a default value, and 'default: "/"' needs to be resolved (plus run "git add config.yaml" to resolve the conflict).
b) reactive/service_checks.py: script has been renamed (and refactored) into an already existing script (reactive/openstack_service_checks.py). We need to run "git rm reactive/service_checks.py"

Once both conflicts are resolved, run "git merge --continue" and that should be it.

1. https://code.launchpad.net/~aluria/charm-openstack-service-checks/+git/charm-openstack-service-checks/+merge/364971
2. https://code.launchpad.net/~aluria/charm-openstack-service-checks/+git/charm-openstack-service-checks/+merge/364970
3. https://code.launchpad.net/~aluria/charm-openstack-service-checks/+git/charm-openstack-service-checks/+merge/364969

[Alvaro Uria (aluria)]

The purpose of this MP is to:
1) move all code off "src/" folder, per latest template-python-pytest revision
2) update tox.ini and Makefile with new environment variables (PYTEST_KEEP_MODEL vs old TEST_PRESERVE_MODEL), as well as update functional/conftest.py, etc. per latest revision (note functional/test_deploy.py has been kept as is, for now).

[Stuart Bishop (stub)]

Yup.

The conflicts need to be resolved on this branch. Merge in master, resolve the conflicts, and push. Wait a few seconds, reload this page, set the MP status to 'Approved' and mergebot will be able to land.

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change has no commit message, setting status to needs review.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 1c0564e386ee87e22b0dfb00b1a3f9c05c09a63d

diff --git a/.gitignore b/.gitignore
index dccc0dc..6a23b4b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,22 @@
-*~
-/bin
-debian/files
-/pkg
-*.pyc
-__pycache__
-*.swp
-.tox
-.venv
-.idea
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Log files
+*.log
+
+.tox/
+.coverage
+
+# vi
+.*.swp
+
+# pycharm
+.idea/
+
+# version data
+repo-info
+
+# reports
+reports/*
diff --git a/Makefile b/Makefile
index 27eeff6..4dc77e9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,26 +1,55 @@
-#!/usr/bin/make
+PROJECTPATH = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+DIRNAME = $(notdir $(PROJECTPATH:%/=%))
 
-all: lint test
+ifndef CHARM_BUILD_DIR
+    CHARM_BUILD_DIR := /tmp/$(DIRNAME)-builds
+    $(warning Warning CHARM_BUILD_DIR was not set, defaulting to $(CHARM_BUILD_DIR))
+endif
 
+help:
+	@echo "This project supports the following targets"
+	@echo ""
+	@echo " make help - show this text"
+	@echo " make lint - run flake8"
+	@echo " make test - run the unittests and lint"
+	@echo " make unittest - run the tests defined in the unittest subdirectory"
+	@echo " make functional - run the tests defined in the functional subdirectory"
+	@echo " make release - build the charm"
+	@echo " make clean - remove unneeded files"
+	@echo ""
 
-.PHONY: clean
-clean:
-	@rm -rf .tox
+lint:
+	@echo "Running flake8"
+	@tox -e lint
 
-.PHONY: apt_prereqs
-apt_prereqs:
-	@# Need tox, but don't install the apt version unless we have to (don't want to conflict with pip)
-	@which tox >/dev/null || (sudo apt-get install -y python3-pip && sudo pip3 install tox)
+test: lint unittest functional
 
-.PHONY: lint
-lint: apt_prereqs
-	@tox -e pep8
-	@charm proof
+unittest:
+	@tox -e unit
 
-.PHONY: test
-unit_test: apt_prereqs
-	@echo Starting tests...
-	tox
+functional: build
+	@PYTEST_KEEP_MODEL=$(PYTEST_KEEP_MODEL) \
+	    PYTEST_CLOUD_NAME=$(PYTEST_CLOUD_NAME) \
+	    PYTEST_CLOUD_REGION=$(PYTEST_CLOUD_REGION) \
+	    CHARM_BUILD_DIR=$(CHARM_BUILD_DIR) \
+	    tox -e functional
 
 build:
-	charm build
+	@echo "Building charm to base directory $(CHARM_BUILD_DIR)"
+	@CHARM_LAYERS_DIR=./layers \
+	    CHARM_INTERFACES_DIR=./interfaces \
+			TERM=linux \
+			CHARM_BUILD_DIR=$(CHARM_BUILD_DIR) \
+			charm build . --force
+
+release: clean build
+	@echo "Charm is built at $(CHARM_BUILD_DIR)"
+
+clean:
+	@echo "Cleaning files"
+	@if [ -d $(CHARM_BUILD_DIR) ] ; then rm -r $(CHARM_BUILD_DIR) ; fi
+	@if [ -d $(PROJECTPATH)/.tox ] ; then rm -r $(PROJECTPATH)/.tox ; fi
+	@if [ -d $(PROJECTPATH)/.pytest_cache ] ; then rm -r $(PROJECTPATH)/.pytest_cache ; fi
+
+# The targets below don't depend on a file
+.PHONY: lint test unittest functional build release clean help
diff --git a/config.yaml b/config.yaml
index b153340..6f09823 100644
--- a/config.yaml
+++ b/config.yaml
@@ -75,6 +75,7 @@ options:
       If true, create NRPE checks matching all 'admin' URLs in the Keystone catalog.
   swift_check_params:
     type: string
+    default: "/"
     description: |
       URL to use with check_http if there is a Swift endpoint.  Default is '/', but it's possible to add extra params,
       e.g. '/v3 -e Unauthorized -d x-openstack-request-id' or a different url, e.g. '/healthcheck'.  Mitaka Swift
diff --git a/files/plugins/check_nova_services.py b/files/plugins/check_nova_services.py
index 6498542..8a3666d 100755
--- a/files/plugins/check_nova_services.py
+++ b/files/plugins/check_nova_services.py
@@ -75,8 +75,7 @@ def check_nova_services(args, nova):
                        if agg['critical']])
     status_warn = len([agg['warning'] for agg in status
                        if agg['warning']])
-    for x in status:
-        (msg.append(x['msg_text']) if x['msg_text'] is not '' else None)
+    msg.extend([x['msg_text'] for x in status if x['msg_text'] != ''])
     if status_crit:
         output = "CRITICAL: {}".format(", ".join(msg))
         raise nagios_plugin.CriticalError(output)
diff --git a/interfaces/.empty b/interfaces/.empty
new file mode 100644
index 0000000..792d600
--- /dev/null
+++ b/interfaces/.empty
@@ -0,0 +1 @@
+#
diff --git a/layer.yaml b/layer.yaml
index 92676ed..290b275 100644
--- a/layer.yaml
+++ b/layer.yaml
@@ -1,3 +1,8 @@
+# exclude the interfaces and layers folders we use for submodules
+exclude:
+  - interfaces
+  - layers
+# include required layers here
 includes:
   - 'layer:basic'
   - 'interface:keystone-credentials'
diff --git a/layers/.empty b/layers/.empty
new file mode 100644
index 0000000..792d600
--- /dev/null
+++ b/layers/.empty
@@ -0,0 +1 @@
+#
diff --git a/lib/lib_openstack_service_checks.py b/lib/lib_openstack_service_checks.py
new file mode 100644
index 0000000..563d09c
--- /dev/null
+++ b/lib/lib_openstack_service_checks.py
@@ -0,0 +1,247 @@
+import os
+from urllib.parse import urlparse
+
+from charmhelpers.core.templating import render
+from charmhelpers.contrib.openstack.utils import config_flags_parser
+from charmhelpers.core import hookenv, host, unitdata
+from charmhelpers.contrib.charmsupport.nrpe import NRPE
+import keystoneauth1
+from keystoneclient import session
+
+
+class OSCCredentialsError(Exception):
+    pass
+
+
+class OSCEndpointError(OSCCredentialsError):
+    pass
+
+
+class OSCHelper():
+    def __init__(self):
+        self.charm_config = hookenv.config()
+
+    def store_keystone_credentials(self, creds):
+        '''store keystone credentials'''
+        unitdata.kv().set('keystonecreds', creds)
+        return
+
+    @property
+    def novarc(self):
+        return '/var/lib/nagios/nagios.novarc'
+
+    @property
+    def plugins_dir(self):
+        return '/usr/local/lib/nagios/plugins/'
+
+    def get_os_credentials(self):
+        ident_creds = config_flags_parser(self.charm_config['os-credentials'])
+        if not ident_creds.get('auth_url'):
+            raise OSCCredentialsError('auth_url')
+        elif '/v3' in ident_creds.get('auth_url'):
+            extra_attrs = ['domain']
+            creds = {'auth_version': 3}
+        else:
+            extra_attrs = []
+            creds = {}
+
+        common_attrs = ('username password region_name auth_url'
+                        ' credentials_project').split()
+        all_attrs = common_attrs + extra_attrs
+        missing = [k for k in all_attrs if k not in ident_creds]
+        if missing:
+            raise OSCCredentialsError(', '.join(missing))
+
+        ident_creds['auth_url'] = ident_creds['auth_url'].strip('\"\'')
+        creds.update(dict([(k, ident_creds.get(k))
+                           for k in all_attrs
+                           if k not in ('credentials_project', 'domain')]))
+        if extra_attrs:
+            creds.update({'project_name': ident_creds['credentials_project'],
+                          'user_domain_name': ident_creds['domain'],
+                          'project_domain_name': ident_creds['domain'],
+                          })
+        else:
+            creds['tenant_name'] = ident_creds['credentials_project']
+
+        return creds
+
+    def get_keystone_credentials(self):
+        '''retrieve keystone credentials from either config or relation data
+
+        If config 'os-crendentials' is set, return that info otherwise look for a keystonecreds relation data'
+
+        :return: dict of credential information for keystone
+        '''
+        return unitdata.kv().get('keystonecreds')
+
+    @property
+    def nova_warn(self):
+        return self.charm_config.get('nova_warn')
+
+    @property
+    def nova_crit(self):
+        return self.charm_config.get('nova_crit')
+
+    @property
+    def skip_disabled(self):
+        if self.charm_config.get('skip-disabled'):
+            return '--skip-disabled'
+        else:
+            return ''
+
+    @property
+    def check_dns(self):
+        return self.charm_config.get('check-dns')
+
+    def render_checks(self, creds):
+        render(source='nagios.novarc', target=self.novarc, context=creds,
+               owner='nagios', group='nagios')
+
+        nrpe = NRPE()
+        if not os.path.exists(self.plugins_dir):
+            os.makedirs(self.plugins_dir)
+
+        charm_plugin_dir = os.path.join(hookenv.charm_dir(),
+                                        'files',
+                                        'plugins/')
+        host.rsync(charm_plugin_dir,
+                   self.plugins_dir,
+                   options=['--executability'])
+
+        nova_check_command = os.path.join(self.plugins_dir,
+                                          'check_nova_services.py')
+        check_command = '{} --warn {} --crit {} {}'.format(
+            nova_check_command, self.nova_warn, self.nova_crit,
+            self.skip_disabled).strip()
+        nrpe.add_check(shortname='nova_services',
+                       description='Check that enabled Nova services are up',
+                       check_cmd=check_command,
+                       )
+
+        nrpe.add_check(shortname='neutron_agents',
+                       description='Check that enabled Neutron agents are up',
+                       check_cmd=os.path.join(self.plugins_dir,
+                                              'check_neutron_agents.sh'),
+                       )
+
+        if len(self.check_dns):
+            nrpe.add_check(shortname='dns_multi',
+                           description='Check DNS names are resolvable',
+                           check_cmd='{} {}'.format(
+                               os.path.join(self.plugins_dir,
+                                            'check_dns_multi.sh'),
+                               ' '.join(self.check_dns.split())),
+                           )
+        else:
+            nrpe.remove_check(shortname='dns_multi')
+        nrpe.write()
+
+        self.create_endpoint_checks(creds)
+
+    def create_endpoint_checks(self, creds):
+        """
+        Create an NRPE check for each Keystone catalog endpoint.
+
+        Read the Keystone catalog, and create a check for each endpoint listed.
+        If there is a healthcheck endpoint for the API, use that URL, otherwise check
+        the url '/'.
+        If SSL, add a check for the cert.
+
+        v2 endpoint needs the 'interface' attribute:
+        <Endpoint {'id': 'XXXXX', 'region': 'RegionOne', 'publicurl': 'http://10.x.x.x:9696',
+        'service_id': 'YYY', 'internalurl': 'http://10.x.x.x:9696', 'enabled': True,
+        'adminurl': 'http://10.x.x.x:9696'}>
+        """
+        # provide URLs that can be used for healthcheck for some services
+        # This also provides a nasty hack-ish way to add switches if we need
+        # for some services.
+        health_check_params = {
+            'keystone': '/healthcheck',
+            's3': '/healthcheck',
+            'aodh': '/healthcheck',
+            'ceilometer': '/ -e Unauthorized -d x-openstack-request-id',
+            'cinderv3': '/v3 -e Unauthorized -d x-openstack-request-id',
+            'cinderv2': '/v2 -e Unauthorized -d x-openstack-request-id',
+            'cinderv1': '/v1 -e Unauthorized -d x-openstack-request-id',
+            'glance': '/healthcheck',
+            'nova': '/healthcheck',
+            'placement': '/healthcheck -e Unauthorized -d x-openstack-request-id',
+            'swift': self.charm_config.get('swift_check_params', '/'),
+            }
+        keystone_client = self.get_keystone_client(creds)
+        try:
+            endpoints = keystone_client.endpoints.list()
+        except keystoneauth1.exceptions.http.InternalServerError as error:
+            raise OSCEndpointError(
+                'Unable to list the keystone endpoints, yet: {}'.format(error))
+
+        services = [x for x in keystone_client.services.list() if x.enabled]
+        nrpe = NRPE()
+        skip_service = set()
+        for endpoint in endpoints:
+            endpoint.service_names = [x.name
+                                      for x in services
+                                      if x.id == endpoint.service_id]
+            service_name = endpoint.service_names[0]
+            endpoint.healthcheck_url = health_check_params.get(service_name, '/')
+            if not hasattr(endpoint, 'interface'):
+                if service_name == 'keystone':
+                    # Note(aluria): filter:healthcheck is not configured in v2
+                    # https://docs.openstack.org/keystone/pike/configuration.html#health-check-middleware
+                    continue
+                for interface in 'admin internal public'.split():
+                    old_interface_name = '{}url'.format(interface)
+                    if not hasattr(endpoint, old_interface_name):
+                        continue
+                    endpoint.interface = interface
+                    endpoint.url = getattr(endpoint, old_interface_name)
+                    skip_service.add(service_name)
+                    break
+
+            if self.charm_config.get('check_{}_urls'.format(endpoint.interface)):
+                cmd_params = ['/usr/lib/nagios/plugins/check_http']
+                check_url = urlparse(endpoint.url)
+                host, port = check_url.netloc.split(':')
+                cmd_params.append('-H {} -p {}'.format(host, port))
+                cmd_params.append('-u {}'.format(endpoint.healthcheck_url))
+                # if this is https, we want to add a check for cert expiry
+                # also need to tell check_http use use TLS
+                if check_url.scheme == 'https':
+                    cmd_params.append('-S')
+                    # Add an extra check for TLS cert expiry
+                    cmd_params.append('-C {},{}'.format(self.charm_config['tls_warn_days'] or 30,
+                                                        self.charm_config['tls_crit_days'] or 14))
+                    nrpe.add_check(shortname='{}_{}_cert'.format(service_name, endpoint.interface),
+                                   description='Certificate expiry check for {} {}'.format(service_name,
+                                                                                           endpoint.interface),
+                                   check_cmd=' '.join(cmd_params))
+
+                # Add the actual health check for the URL
+                nrpe.add_check(shortname='{}_{}'.format(service_name, endpoint.interface),
+                               description='Endpoint url check for {} {}'.format(service_name, endpoint.interface),
+                               check_cmd=' '.join(cmd_params))
+        nrpe.write()
+
+    def get_keystone_client(self, creds):
+        """Import the appropriate Keystone client depending on API version.
+
+        Use credential info to determine the Keystone API version, and make a
+        client session object that is to be used for authenticated
+        communication with Keystone.
+
+        :returns: a keystoneclient Client object
+        """
+        if int(creds.get('auth_version', 0)) >= 3:
+            from keystoneclient.v3 import client
+            from keystoneclient.auth.identity import v3 as kst_version
+            auth_fields = 'username password auth_url user_domain_name project_domain_name project_name'.split()
+        else:
+            from keystoneclient.v2_0 import client
+            from keystoneclient.auth.identity import v2 as kst_version
+            auth_fields = 'username password auth_url tenant_name'.split()
+
+        auth_creds = dict([(key, creds.get(key)) for key in auth_fields])
+        auth = kst_version.Password(**auth_creds)
+        sess = session.Session(auth=auth)
+        return client.Client(session=sess)
diff --git a/metadata.yaml b/metadata.yaml
index 96c1f7e..3ca8dc0 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -1,17 +1,16 @@
 name: openstack-service-checks
 summary: OpenStack Services NRPE Checks
-description: |
-  OpenStack Services NRPE Checks
-maintainer: Jill Rouleau <[e-mail]>
+description: OpenStack Services NRPE Checks
+maintainer: Nagios Charm Developers <[e-mail]>
 subordinate: false
 tags:
   - openstack
   - ops
   - monitoring
 series:
+- bionic
 - xenial
 - trusty
-- bionic
 provides:
   nrpe-external-master:
     interface: nrpe-external-master
diff --git a/reactive/openstack_service_checks.py b/reactive/openstack_service_checks.py
new file mode 100644
index 0000000..c122bf1
--- /dev/null
+++ b/reactive/openstack_service_checks.py
@@ -0,0 +1,248 @@
+# -*- coding: us-ascii -*-
+"""
+install_osc +------------------------> render_config+--------------> do_restart
+                                          ^       +                      ^
+conf_kst_user+---> save_creds +-----------+       v                      |
+                                                  create_endpoints +-----+
+
+Available states:
+    identity-credentials.available: kst creds available
+    identity-credentials.connected: kst relation joined
+    nrpe-external-master.available: nrpe relation joined
+    openstack-service-checks.configured: render_config allowed
+    openstack-service-checks.endpoints.configured: create_endpoints allowed
+    openstack-service-checks.installed: install_osc entrypoint
+    openstack-service-checks.started: if not set, restart nagios-nrpe-server
+    openstack-service-checks.stored-creds: kst creds available for the unit
+"""
+import base64
+import subprocess
+
+from charmhelpers.core import hookenv, host, unitdata
+from charms.reactive import clear_flag, set_flag, when, when_not
+
+from lib_openstack_service_checks import (
+    OSCHelper,
+    OSCCredentialsError,
+    OSCEndpointError
+)
+
+CERT_FILE = '/usr/local/share/ca-certificates/openstack-service-checks.crt'
+helper = OSCHelper()
+
+
+@when_not('openstack-service-checks.installed')
+@when('nrpe-external-master.available')
+def install_openstack_service_checks():
+    """Entry point to start configuring the unit
+
+    Triggered if related to the nrpe-external-master relation.
+    Some relation data can be initialized if the application is related to
+    keystone.
+    """
+    set_flag('openstack-service-checks.installed')
+    clear_flag('openstack-service-checks.configured')
+
+
+@when_not('identity-credentials.available')
+@when('identity-credentials.connected')
+def configure_ident_username(keystone):
+    """Requests a user to the Identity Service
+    """
+    username = 'nagios'
+    keystone.request_credentials(username)
+    clear_flag('openstack-service-checks.stored-creds')
+
+
+@when_not('openstack-service-checks.stored-creds')
+@when('identity-credentials.available')
+def save_creds(keystone):
+    """Collect and save credentials from Keystone relation.
+
+    Get credentials from the Keystone relation,
+    reformat them into something the Keystone client can use, and
+    save them into the unitdata.
+    """
+    creds = {'username': keystone.credentials_username(),
+             'password': keystone.credentials_password(),
+             'region': keystone.region(),
+             }
+    if keystone.api_version() == '3':
+        api_url = 'v3'
+        try:
+            domain = keystone.domain()
+        except AttributeError:
+            domain = 'service_domain'
+        # keystone relation sends info back with funny names, fix here
+        creds.update({'project_name': keystone.credentials_project(),
+                      'auth_version': '3',
+                      'user_domain_name': domain,
+                      'project_domain_name': domain})
+    else:
+        api_url = 'v2.0'
+        creds['tenant_name'] = keystone.credentials_project()
+
+    creds['auth_url'] = '{proto}://{host}:{port}/{api_url}'.format(
+        proto=keystone.auth_protocol(), host=keystone.auth_host(),
+        port=keystone.auth_port(), api_url=api_url)
+
+    helper.store_keystone_credentials(creds)
+    set_flag('openstack-service-checks.stored-creds')
+    clear_flag('openstack-service-checks.configured')
+
+
+@when_not('identity-credentials.connected')
+@when_not('identity-credentials.available')
+@when('openstack-service-checks.stored-creds')
+def allow_keystone_store_overwrite():
+    """Allow unitdata overwrite if keystone relation is recycled.
+    """
+    clear_flag('openstack-service-checks.stored-creds')
+
+
+def get_credentials():
+    """Get credential info from either config or relation data
+
+    If config 'os-credentials' is set, return it. Otherwise look for a
+    keystonecreds relation data.
+    """
+    try:
+        creds = helper.get_os_credentials()
+    except OSCCredentialsError as error:
+        creds = helper.get_keystone_credentials()
+        if not creds:
+            hookenv.log('render_config: No credentials yet, skipping')
+            hookenv.status_set('blocked',
+                               'Missing os-credentials vars: {}'.format(error))
+            return
+    return creds
+
+
+@when('openstack-service-checks.installed')
+@when_not('openstack-service-checks.configured')
+def render_config():
+    """Render nrpe checks from the templates
+
+    This code is only triggered after the nrpe relation is set. If a relation
+    with keystone is later set, it will be re-triggered. On the other hand,
+    if a keystone relation exists but not a nrpe relation, it won't be run.
+
+    Furthermore, juju config os-credentials take precedence over keystone
+    related data.
+    """
+    def block_tls_failure(error):
+        hookenv.log('update-ca-certificates failed: {}'.format(error),
+                    hookenv.ERROR)
+        hookenv.status_set('blocked',
+                           'update-ca-certificates error. check logs')
+        return
+
+    creds = get_credentials()
+    if not creds:
+        return
+
+    # Fix TLS
+    if helper.charm_config['trusted_ssl_ca'].strip():
+        trusted_ssl_ca = helper.charm_config['trusted_ssl_ca'].strip()
+        hookenv.log('Writing ssl ca cert:{}'.format(trusted_ssl_ca))
+        cert_content = base64.b64decode(trusted_ssl_ca).decode()
+        try:
+            with open(CERT_FILE, 'w') as fd:
+                fd.write(cert_content)
+            subprocess.call(['/usr/sbin/update-ca-certificates'])
+
+        except subprocess.CalledProcessError as error:
+            block_tls_failure(error)
+            return
+        except PermissionError as error:
+            block_tls_failure(error)
+            return
+
+    hookenv.log('render_config: Got credentials for'
+                ' username={}'.format(creds.get('username')))
+
+    try:
+        helper.render_checks(creds)
+        set_flag('openstack-service-checks.endpoints.configured')
+    except OSCEndpointError as error:
+        hookenv.log(error)
+
+    set_flag('openstack-service-checks.configured')
+    clear_flag('openstack-service-checks.started')
+
+
+@when('openstack-service-checks.installed')
+@when('openstack-service-checks.configured')
+@when_not('openstack-service-checks.endpoints.configured')
+def configure_nrpe_endpoints():
+    """Create an NRPE check for each Keystone catalog endpoint.
+
+    Read the Keystone catalog, and create a check for each endpoint listed.
+    If there is a healthcheck endpoint for the API, use that URL. Otherwise,
+    check the url '/'.
+
+    If TLS is enabled, add a check for the cert.
+    """
+    creds = get_credentials()
+    if not creds:
+        return
+
+    try:
+        helper.create_endpoint_checks(creds)
+        set_flag('openstack-service-checks.endpoints.configured')
+        clear_flag('openstack-service-checks.started')
+    except OSCEndpointError as error:
+        hookenv.log(error)
+
+
+@when('openstack-service-checks.configured')
+@when_not('openstack-service-checks.started')
+def do_restart():
+    hookenv.log('Reloading nagios-nrpe-server')
+    host.service_restart('nagios-nrpe-server')
+    hookenv.status_set('active', 'Unit is ready')
+    set_flag('openstack-service-checks.started')
+
+
+@when('config.changed.os-credentials')
+@when('nrpe-external-master.available')
+def do_reconfigure_nrpe():
+    clear_flag('openstack-service-checks.configured')
+    clear_flag('openstack-service-checks.endpoints.configured')
+
+
+@when_not('nrpe-external-master.available')
+def missing_nrpe():
+    """Avoid a user action to be missed or overwritten by another hook
+    """
+    if hookenv.hook_name() != 'update-status':
+        hookenv.status_set('blocked', 'Missing relations: nrpe')
+
+
+@when('openstack-service-checks.installed')
+@when('nrpe-external-master.available')
+def parse_hooks():
+    if hookenv.hook_name() == 'upgrade-charm':
+        # Check if creds storage needs to be migrated
+        # Old key: keystone-relation-creds
+        # New key: keystonecreds
+        kv = unitdata.kv()
+        creds = kv.get('keystonecreds')
+        old_creds = kv.get('keystone-relation-creds')
+        if old_creds and not creds:
+            # This set of creds needs an update to a newer format
+            creds = {
+                'username': old_creds['credentials_username'],
+                'password': old_creds['credentials_password'],
+                'project_name': old_creds['credentials_project'],
+                'tenant_name': old_creds['credentials_project'],
+                'user_domain_name': old_creds.get('credentials_user_domain'),
+                'project_domain_name': old_creds.get('credentials_project_domain'),
+                }
+            kv.set('keystonecreds', creds)
+
+        if old_creds:
+            kv.unset('keystone-relation-creds')
+
+        # render configs again
+        do_reconfigure_nrpe()
diff --git a/reactive/service_checks.py b/reactive/service_checks.py
deleted file mode 100644
index 1a6f5bb..0000000
--- a/reactive/service_checks.py
+++ /dev/null
@@ -1,326 +0,0 @@
-from __future__ import print_function
-import base64
-import keystoneauth1
-import os
-import subprocess
-from charms.reactive import (
-    when,
-    when_not,
-    set_state,
-    remove_state,
-)
-
-from charmhelpers.core.templating import render
-from charmhelpers.contrib.openstack.utils import config_flags_parser
-from charmhelpers.core import (
-    host,
-    hookenv,
-    unitdata,
-)
-
-from charmhelpers.contrib.charmsupport.nrpe import NRPE
-from urllib.parse import urlparse
-
-config = hookenv.config()
-NOVARC = '/var/lib/nagios/nagios.novarc'
-PLUGINS_DIR = '/usr/local/lib/nagios/plugins/'
-
-
-@when_not('os-service-checks.installed')
-def install_service_checks():
-    # Apt package installs are handled by the basic layer
-    set_state('os-service-checks.installed')
-    remove_state('os-service-checks.configured')
-    hookenv.status_set('active', 'Ready')
-
-
-@when('identity-credentials.connected')
-def configure_ident_username(keystone):
-    username = 'nagios'
-    keystone.request_credentials(username)
-
-
-@when('identity-credentials.available')
-def save_creds(keystone):
-    """Collect and save credentials from Keystone relation.
-
-    Get credentials from the Keystone relation,
-    reformat them into something the Keystone client
-    can use, and save them into the unitdata.
-    """
-    creds = {
-        'username': keystone.credentials_username(),
-        'password': keystone.credentials_password(),
-        'region': keystone.region(),
-    }
-    if keystone.api_version() == '3':
-        api_url = "v3"
-        try:
-            domain = keystone.domain()
-        except AttributeError:
-            domain = 'service_domain'
-        # keystone relation sends info back with funny names, fix here
-        creds.update({
-            'project_name': keystone.credentials_project(),
-            'auth_version': '3',
-            'user_domain_name': domain,
-            'project_domain_name': domain
-        })
-    else:
-        api_url = "v2.0"
-        creds['tenant_name'] = keystone.credentials_project()
-
-    auth_url = "%s://%s:%s/%s" % (keystone.auth_protocol(),
-                                  keystone.auth_host(), keystone.auth_port(),
-                                  api_url)
-    creds['auth_url'] = auth_url
-    unitdata.kv().set('keystonecreds', creds)
-    remove_state('os-service-checks.configured')
-
-
-# allow user to override credentials (and the need to be related to Keystone)
-# with 'os-credentials'
-def get_credentials():
-    """Get credential info from either config or relation data.
-
-    If config 'os-credentials' is set, return that info otherwise look for for a keystonecreds relation data.
-
-    :return: dictionary of credential information for Keystone.
-    """
-    ident_creds = config_flags_parser(config.get('os-credentials'))
-    if ident_creds:
-        creds = {
-            'username': ident_creds['username'],
-            'password': ident_creds['password'],
-            'region': ident_creds['region_name'],
-            'auth_url': ident_creds['auth_url'].strip('\"\''),
-        }
-        if '/v3' in ident_creds['auth_url']:
-            creds.update({
-                'project_name': ident_creds['credentials_project'],
-                'auth_version': '3',
-                'user_domain_name': ident_creds['domain'],
-                'project_domain_name': ident_creds['domain'],
-            })
-        else:
-            creds['tenant_name'] = ident_creds['credentials_project']
-    else:
-        kv = unitdata.kv()
-        creds = kv.get('keystonecreds')
-        old_creds = kv.get('keystone-relation-creds')
-        if old_creds and not creds:
-            # This set of creds needs an update to a newer format
-            creds['username'] = old_creds.pop('credentials_username')
-            creds['password'] = old_creds.pop('credentials_password')
-            creds['project_name'] = old_creds.pop('credentials_project')
-            creds['tenant_name'] = old_creds['project_name']
-            creds['user_domain_name'] = old_creds.pop('credentials_user_domain', None)
-            creds['project_domain_name'] = old_creds.pop('credentials_project_domain', None)
-            kv.set('keystonecreds', creds)
-            kv.update(creds, 'keystonecreds')
-    return creds
-
-
-def render_checks():
-    nrpe = NRPE()
-    if not os.path.exists(PLUGINS_DIR):
-        os.makedirs(PLUGINS_DIR)
-    charm_file_dir = os.path.join(hookenv.charm_dir(), 'files')
-    charm_plugin_dir = os.path.join(charm_file_dir, 'plugins')
-    host.rsync(
-        charm_plugin_dir,
-        '/usr/local/lib/nagios/',
-        options=['--executability']
-    )
-
-    warn = config.get("nova_warn")
-    crit = config.get("nova_crit")
-    skip_disabled = config.get("skip-disabled")
-    check_dns = config.get("check-dns")
-    nova_check_command = os.path.join(PLUGINS_DIR, 'check_nova_services.py')
-    check_command = '{} --warn {} --crit {}'.format(nova_check_command, warn, crit)
-
-    if skip_disabled:
-        check_command = check_command + ' --skip-disabled'
-
-    nrpe.add_check(shortname='nova_services',
-                   description='Check that enabled Nova services are up',
-                   check_cmd=check_command)
-
-    nrpe.add_check(shortname='neutron_agents',
-                   description='Check that enabled Neutron agents are up',
-                   check_cmd=os.path.join(PLUGINS_DIR, 'check_neutron_agents.sh'))
-
-    if len(check_dns):
-        nrpe.add_check(shortname='dns_multi',
-                       description='Check DNS names are resolvable',
-                       check_cmd=PLUGINS_DIR + 'check_dns_multi.sh ' + ' '.join(check_dns.split()))
-    else:
-        nrpe.remove_check(shortname='dns_multi')
-
-    endpoint_checks = create_endpoint_checks()
-
-    if endpoint_checks is None:
-        return False
-    else:
-        for check in endpoint_checks:
-            nrpe.add_check(**check)
-        nrpe.write()
-        return True
-
-
-@when('nrpe-external-master.available')
-def nrpe_connected(nem):
-    remove_state('os-service-checks.configured')
-
-
-@when('os-service-checks.installed')
-@when_not('os-service-checks.configured')
-def render_config():
-    if config.get('trusted_ssl_ca', None):
-        fix_ssl()
-    creds = get_credentials()
-    if not creds:
-        hookenv.log('render_config: No credentials yet, skipping')
-        return
-    hookenv.log('render_config: Got credentials for username={}'.format(
-        creds['username']))
-    render('nagios.novarc', NOVARC, creds,
-           owner='nagios', group='nagios')
-    if render_checks():
-        hookenv.status_set('active', 'Ready')
-        set_state('os-service-checks.configured')
-        remove_state('os-service-checks.started')
-    else:
-        hookenv.status_set('blocked', 'waiting for Keystone to be ready')
-
-
-@when('os-service-checks.configured')
-@when_not('os-service-checks.started')
-def do_restart():
-    hookenv.log('Reloading nagios-nrpe-server')
-    host.service_restart('nagios-nrpe-server')
-    hookenv.status_set('active', 'Ready')
-    set_state('os-service-checks.started')
-
-
-def fix_ssl():
-    cert_file = '/usr/local/share/ca-certificates/openstack-service-checks.crt'
-    trusted_ssl_ca = config.get('trusted_ssl_ca').strip()
-    hookenv.log("Writing ssl ca cert:{}".format(trusted_ssl_ca))
-    cert_content = base64.b64decode(trusted_ssl_ca).decode()
-    with open(cert_file, 'w') as f:
-        print(cert_content, file=f)
-    subprocess.call(["/usr/sbin/update-ca-certificates"])
-
-
-def create_endpoint_checks():
-    """Create an NRPE check for each Keystone catalog endpoint.
-
-    Read the Keystone catalog, and create a check for each endpoint listed.
-    If there is a healthcheck endpoint for the API, use that URL, otherwise check
-    the url '/'.
-    If SSL, add a check for the cert.
-    """
-    # provide URLs that can be used for healthcheck for some services
-    # This also provides a nasty hack-ish way to add switches if we need
-    # for some services.
-    health_check_params = {
-        'keystone': '/healthcheck',
-        's3': '/healthcheck',
-        'aodh': '/healthcheck',
-        'cinderv3': '/v3 -e Unauthorized -d x-openstack-request-id',
-        'cinderv2': '/v2 -e Unauthorized -d x-openstack-request-id',
-        'cinderv1': '/v1 -e Unauthorized -d x-openstack-request-id',
-        'ceilometer': '/ -e Unauthorized -d x-openstack-request-id',
-        'placement': '/healthcheck -e Unauthorized -d x-openstack-request-id',
-        'glance': '/healthcheck',
-        'nova': '/healthcheck',
-        'swift': config.get('swift_check_params', '/'),
-    }
-
-    creds = get_credentials()
-    keystone_client = get_keystone_client(creds)
-    try:
-        endpoints = keystone_client.endpoints.list()
-    except keystoneauth1.exceptions.http.InternalServerError:
-        return None
-
-    services = [x for x in keystone_client.services.list() if x.enabled]
-    nrpe_checks = []
-    for endpoint in endpoints:
-        endpoint.service_names = [x.name for x in services if x.id == endpoint.service_id]
-        service_name = endpoint.service_names[0]
-        endpoint.healthcheck_url = health_check_params.get(service_name, '/')
-        if config.get('check_{}_urls'.format(endpoint.interface)):
-            cmd_params = ['/usr/lib/nagios/plugins/check_http ']
-            check_url = urlparse(endpoint.url)
-            host_port = check_url.netloc.split(':')
-            # assume http port 80 if no port specified
-            endpoint_port = 80
-            if len(host_port) < 2:
-                if check_url.scheme == 'https':
-                    endpoint_port = 443
-            else:
-                endpoint_port = host_port[1]
-            cmd_params.append('-H {} -p {}'.format(host_port[0], endpoint_port))
-            cmd_params.append('-u {}'.format(endpoint.healthcheck_url))
-            # if this is https, we want to add a check for cert expiry
-            # also need to tell check_http use use TLS
-            if check_url.scheme == 'https':
-                cmd_params.append('-S')
-                # Add an extra check for TLS cert expiry
-                cert_check = ['-C {},{}'.format(
-                    config.get('tls_warn_days'),
-                    config.get('tls_crit_days'))]
-                nrpe_checks.append({
-                    'shortname': "{}_{}_cert".format(
-                        service_name,
-                        endpoint.interface),
-                    'description': 'Certificate expiry check for {} {}'.format(
-                        service_name,
-                        endpoint.interface),
-                    'check_cmd': ' '.join(cmd_params + cert_check)
-                })
-            # Add the actual health check for the URL
-            nrpe_checks.append({
-                'shortname': "{}_{}".format(
-                    service_name,
-                    endpoint.interface),
-                'description': 'Endpoint url check for {} {}'.format(
-                    service_name,
-                    endpoint.interface),
-                'check_cmd': (' '.join(cmd_params))})
-    return nrpe_checks
-
-
-def get_keystone_client(creds):
-    """Import the appropriate Keystone client depending on API version.
-
-    Use credential info to determine the Keystone API version, and make a client session object that is to be
-    used for authenticated communication with Keystone.
-
-    :returns: a keystoneclient Client object
-    """
-    from keystoneclient import session
-    if int(creds['auth_version']) >= 3:
-        from keystoneclient.v3 import client
-        from keystoneclient.auth.identity import v3
-        auth_fields = ['auth_url', 'password', 'username', 'user_domain_name',
-                       'project_domain_name', 'project_name']
-        auth_creds = {}
-        for key in auth_fields:
-            auth_creds[key] = creds[key]
-        auth = v3.Password(**auth_creds)
-
-    else:
-        from keystoneclient.v2_0 import client
-        from keystoneclient.auth.identity import v2
-        auth_fields = ['auth_url', 'password', 'username', 'tenant_name']
-        auth_creds = {}
-        for key in auth_fields:
-            auth_creds[key] = creds[key]
-        auth = v2.Password(**auth_creds)
-
-    sess = session.Session(auth=auth)
-    return client.Client(session=sess)
diff --git a/test-requirements.txt b/test-requirements.txt
deleted file mode 100644
index 77ec3f5..0000000
--- a/test-requirements.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Lint and unit test requirements
-flake8>=2.2.4,<=2.4.1
-os-testr>=0.4.1
-requests>=2.18.4
-charms.reactive
-mock>=1.2
-nose>=1.3.7
-coverage>=3.6
-netifaces
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
new file mode 100644
index 0000000..9f93ed0
--- /dev/null
+++ b/tests/functional/conftest.py
@@ -0,0 +1,179 @@
+'''
+Reusable pytest fixtures for functional testing
+
+Environment variables
+---------------------
+
+test_preserve_model:
+if set, the testing model won't be torn down at the end of the testing session
+'''
+
+import asyncio
+import json
+import os
+import uuid
+
+import pytest
+import subprocess
+import juju
+from juju.controller import Controller
+from juju.errors import JujuError
+
+STAT_CMD = '''python3 - <<EOF
+import json
+import os
+
+s = os.stat('%s')
+stat_hash = {
+    'uid': s.st_uid,
+    'gid': s.st_gid,
+    'mode': oct(s.st_mode),
+    'size': s.st_size
+}
+stat_json = json.dumps(stat_hash)
+print(stat_json)
+
+EOF
+'''
+
+
[e-mail](scope='module')
+def event_loop():
+    '''Override the default pytest event loop to allow for fixtures using a
+    broader scope'''
+    loop = asyncio.get_event_loop_policy().new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.set_debug(True)
+    yield loop
+    loop.close()
+    asyncio.set_event_loop(None)
+
+
[e-mail](scope='module')
+async def controller():
+    '''Connect to the current controller'''
+    _controller = Controller()
+    await _controller.connect_current()
+    yield _controller
+    await _controller.disconnect()
+
+
[e-mail](scope='module')
+async def model(controller):
+    '''This model lives only for the duration of the test'''
+    model_name = "functest-{}".format(str(uuid.uuid4())[-12:])
+    _model = await controller.add_model(model_name,
+                                        cloud_name=os.getenv('PYTEST_CLOUD_NAME'),
+                                        region=os.getenv('PYTEST_CLOUD_REGION'),
+                                        )
+    # https://github.com/juju/python-libjuju/issues/267
+    subprocess.check_call(['juju', 'models'])
+    while model_name not in await controller.list_models():
+        await asyncio.sleep(1)
+    yield _model
+    await _model.disconnect()
+    if not os.getenv('PYTEST_KEEP_MODEL'):
+        await controller.destroy_model(model_name)
+        while model_name in await controller.list_models():
+            await asyncio.sleep(1)
+
+
[e-mail]
+async def get_app(model):
+    '''Returns the application requested'''
+    async def _get_app(name):
+        try:
+            return model.applications[name]
+        except KeyError:
+            raise JujuError("Cannot find application {}".format(name))
+    return _get_app
+
+
[e-mail]
+async def get_unit(model):
+    '''Returns the requested <app_name>/<unit_number> unit'''
+    async def _get_unit(name):
+        try:
+            (app_name, unit_number) = name.split('/')
+            return model.applications[app_name].units[unit_number]
+        except (KeyError, ValueError):
+            raise JujuError("Cannot find unit {}".format(name))
+    return _get_unit
+
+
[e-mail]
+async def get_entity(get_unit, get_app):
+    '''Returns a unit or an application'''
+    async def _get_entity(name):
+        try:
+            return await get_unit(name)
+        except JujuError:
+            try:
+                return await get_app(name)
+            except JujuError:
+                raise JujuError("Cannot find entity {}".format(name))
+    return _get_entity
+
+
[e-mail]
+async def run_command(get_unit):
+    '''
+    Runs a command on a unit.
+
+    :param cmd: Command to be run
+    :param target: Unit object or unit name string
+    '''
+    async def _run_command(cmd, target):
+        unit = (
+            target
+            if isinstance(target, juju.unit.Unit)
+            else await get_unit(target)
+        )
+        action = await unit.run(cmd)
+        return action.results
+    return _run_command
+
+
[e-mail]
+async def file_stat(run_command):
+    '''
+    Runs stat on a file
+
+    :param path: File path
+    :param target: Unit object or unit name string
+    '''
+    async def _file_stat(path, target):
+        cmd = STAT_CMD % path
+        results = await run_command(cmd, target)
+        return json.loads(results['Stdout'])
+    return _file_stat
+
+
[e-mail]
+async def file_contents(run_command):
+    '''
+    Returns the contents of a file
+
+    :param path: File path
+    :param target: Unit object or unit name string
+    '''
+    async def _file_contents(path, target):
+        cmd = 'cat {}'.format(path)
+        results = await run_command(cmd, target)
+        return results['Stdout']
+    return _file_contents
+
+
[e-mail]
+async def reconfigure_app(get_app, model):
+    '''Applies a different config to the requested app'''
+    async def _reconfigure_app(cfg, target):
+        application = (
+            target
+            if isinstance(target, juju.application.Application)
+            else await get_app(target)
+        )
+        await application.set_config(cfg)
+        await application.get_config()
+        await model.block_until(lambda: application.status == 'active')
+    return _reconfigure_app
diff --git a/tests/functional/requirements.txt b/tests/functional/requirements.txt
new file mode 100644
index 0000000..f76bfbb
--- /dev/null
+++ b/tests/functional/requirements.txt
@@ -0,0 +1,6 @@
+flake8
+juju
+mock
+pytest
+pytest-asyncio
+requests
diff --git a/tests/functional/test_deploy.py b/tests/functional/test_deploy.py
new file mode 100644
index 0000000..add1c63
--- /dev/null
+++ b/tests/functional/test_deploy.py
@@ -0,0 +1,92 @@
+import collections
+import os
+
+import pytest
+
+# Treat all tests as coroutines
+pytestmark = pytest.mark.asyncio
+
+SERIES = ['xenial', 'bionic']
+CHARM_BUILD_DIR = os.getenv('CHARM_BUILD_DIR', '.').rstrip('/')
+
+
+# Custom fixtures
+
[e-mail](scope='module', params=SERIES)
+async def osc_apps(request, model):
+    series = request.param
+    app = model.applications.get('openstack-service-checks-{}'.format(series))
+    return app
+
+
[e-mail](scope='module', params=SERIES)
+async def units(request, apps):
+    return apps.units
+
+
+def app_names(series):
+    apps_list = ('keystone neutron-api nova-cloud-controller percona-cluster rabbitmq-server'
+                 ' openstack-service-checks nagios nrpe').split()
+    apps = dict([(app, '{}-{}'.format(app, series)) for app in apps_list])
+    return apps
+
+
[e-mail](scope='module', params=SERIES)
+async def deploy_openstack(request, model):
+    series = request.param
+    nunits = collections.defaultdict(lambda: 1)
+    apps = app_names(series)
+    active_apps = []
+    for app in apps:
+        if app == 'openstack-service-checks':
+            continue
+        if app == 'nrpe':
+            nunits[app] = 0
+        app_deploy = await model.deploy('cs:{}'.format(app), series=series,
+                                        application_name=apps[app], num_units=nunits[app])
+        if app not in ('nova-cloud-controller', 'nrpe'):
+            active_apps.append(app_deploy)
+
+    await model.add_relation('{}:shared-db'.format(apps['keystone']),
+                             '{}:shared-db'.format(apps['percona-cluster']))
+    await model.add_relation('{}:monitors'.format(apps['nrpe']),
+                             '{}:monitors'.format(apps['nagios']))
+
+    for app in 'neutron-api nova-cloud-controller'.split():
+        if app == 'openstack-service-checks':
+            continue
+
+        await model.add_relation('{}:amqp'.format(apps[app]),
+                                 '{}:amqp'.format(apps['rabbitmq-server']))
+        await model.add_relation('{}:shared-db'.format(apps[app]),
+                                 '{}:shared-db'.format(apps['percona-cluster']))
+        await model.add_relation('{}:identity-service'.format(apps[app]),
+                                 '{}:identity-service'.format(apps['keystone']))
+
+    yield active_apps
+
+
[e-mail](scope='module', params=SERIES)
+async def deploy_app(request, model):
+    series = request.param
+    apps = app_names(series)
+
+    # Starts a deploy for each series
+    osc_app = await model.deploy(os.path.join(CHARM_BUILD_DIR, 'openstack-service-checks'),
+                                 series=series, application_name=apps['openstack-service-checks'])
+
+    for app in 'keystone nrpe'.split():
+        await model.add_relation(apps[app], apps['openstack-service-checks'])
+
+    yield osc_app
+
+
+# Tests
+
+async def test_openstackservicechecks_deploy_openstack(deploy_openstack, model):
+    await model.block_until(lambda: all([app.status == 'active' for app in deploy_openstack]),
+                            timeout=900)
+
+
+async def test_openstackservicechecks_deploy(deploy_app, model):
+    await model.block_until(lambda: deploy_app.status == 'active', timeout=600)
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
new file mode 100644
index 0000000..32a4af9
--- /dev/null
+++ b/tests/unit/conftest.py
@@ -0,0 +1,77 @@
+import unittest.mock as mock
+
+import pytest
+
+
+# If layer options are used, add this to openstackservicechecks
+# and import layer in lib_openstack_service_checks
[e-mail]
+def mock_layers(monkeypatch):
+    import sys
+    sys.modules['charms.layer'] = mock.Mock()
+    sys.modules['reactive'] = mock.Mock()
+    # Mock any functions in layers that need to be mocked here
+
+    def options(layer):
+        # mock options for layers here
+        if layer == 'example-layer':
+            options = {'port': 9999}
+            return options
+        else:
+            return None
+
+    monkeypatch.setattr('lib_openstack_service_checks.layer.options', options)
+
+
[e-mail]
+def mock_hookenv_config(monkeypatch):
+    import yaml
+
+    def mock_config():
+        cfg = {}
+        yml = yaml.safe_load(open('./config.yaml'))
+
+        # Load all defaults
+        for key, value in yml['options'].items():
+            cfg[key] = value['default']
+
+        # Manually add cfg from other layers
+        # cfg['my-other-layer'] = 'mock'
+        return cfg
+
+    monkeypatch.setattr('lib_openstack_service_checks.hookenv.config', mock_config)
+
+
[e-mail]
+def mock_remote_unit(monkeypatch):
+    monkeypatch.setattr('lib_openstack_service_checks.hookenv.remote_unit', lambda: 'unit-mock/0')
+
+
[e-mail]
+def mock_charm_dir(monkeypatch):
+    monkeypatch.setattr('lib_openstack_service_checks.hookenv.charm_dir', lambda: '/mock/charm/dir')
+
+
[e-mail]
+def mock_unitdata_keystonecreds(monkeypatch):
+    creds = {'keystonecreds': {'username': 'nagios',
+                               'password': 'password',
+                               'project_name': 'services',
+                               'tenant_name': 'services',
+                               'user_domain_name': 'service_domain',
+                               'project_domain_name': 'service_domain',
+                               }
+             }
+    monkeypatch.setattr('lib_openstack_service_checks.unitdata.kv', lambda: creds)
+
+
[e-mail]
+def openstackservicechecks(tmpdir, mock_hookenv_config, mock_charm_dir, monkeypatch):
+    from lib_openstack_service_checks import OSCHelper
+    helper = OSCHelper()
+
+    # Any other functions that load helper will get this version
+    monkeypatch.setattr('lib_openstack_service_checks.hookenv.log', lambda msg, level='INFO': None)
+    monkeypatch.setattr('lib_openstack_service_checks.OSCHelper', lambda: helper)
+
+    return helper
diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt
new file mode 100644
index 0000000..1c828b8
--- /dev/null
+++ b/tests/unit/requirements.txt
@@ -0,0 +1,10 @@
+charmhelpers
+charms.reactive
+mock
+pytest
+pytest-cov
+os-testr>=0.4.1
+requests>=2.18.4
+netifaces
+keystoneauth1
+python-keystoneclient
diff --git a/tests/unit/test_lib.py b/tests/unit/test_lib.py
new file mode 100644
index 0000000..d10d44a
--- /dev/null
+++ b/tests/unit/test_lib.py
@@ -0,0 +1,44 @@
+from pytest import mark
+
+
+def test_openstackservicechecks_common_properties(openstackservicechecks):
+    '''Verify the most common properties from the class or default config.yaml'''
+    assert isinstance(openstackservicechecks.charm_config, dict)
+    assert openstackservicechecks.novarc == '/var/lib/nagios/nagios.novarc'
+    assert openstackservicechecks.plugins_dir == '/usr/local/lib/nagios/plugins/'
+    assert openstackservicechecks.nova_warn == 2
+    assert openstackservicechecks.nova_crit == 1
+    assert openstackservicechecks.skip_disabled == ''
+    assert openstackservicechecks.check_dns == ''
+
+
+def test_openstackservicechecks_get_keystone_credentials_unitdata(
+        openstackservicechecks, mock_unitdata_keystonecreds):
+    """Checks the expected behavior when 'os-credentials' are not shared, but the application is related to keystone.
+    """
+    assert openstackservicechecks.get_keystone_credentials() == {
+        'username': 'nagios', 'password': 'password', 'project_name': 'services',
+        'tenant_name': 'services', 'user_domain_name': 'service_domain',
+        'project_domain_name': 'service_domain'
+        }
+
+
[e-mail]('os_credentials,expected', [
+    (('username=nagios, password=password, region_name=RegionOne, auth_url="http://XX.XX.XX.XX:5000/v3",'
+     'credentials_project=services, domain=service_domain'),
+     {'username': 'nagios', 'password': 'password', 'project_name': 'services', 'auth_version': 3,
+      'user_domain_name': 'service_domain', 'project_domain_name': 'service_domain', 'region_name': 'RegionOne',
+      'auth_url': 'http://XX.XX.XX.XX:5000/v3'},
+     ),
+    (('username=nagios, password=password, region_name=RegionOne, auth_url="http://XX.XX.XX.XX:5000/v2.0",'
+      'credentials_project=services'),
+     {'username': 'nagios', 'password': 'password', 'tenant_name': 'services', 'region_name': 'RegionOne',
+      'auth_url': 'http://XX.XX.XX.XX:5000/v2.0'},
+     )
+])
+def test_openstackservicechecks_get_keystone_credentials_oscredentials(
+        os_credentials, expected, openstackservicechecks, mock_unitdata_keystonecreds):
+    """Checks the expected behavior when keystone v2 and v3 data is shared via the 'os-credentials' config parameter.
+    """
+    openstackservicechecks.charm_config['os-credentials'] = os_credentials
+    assert openstackservicechecks.get_os_credentials() == expected
diff --git a/tox.ini b/tox.ini
index f4560b5..3a54b37 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,40 +1,45 @@
 [tox]
 skipsdist=True
-envlist = pep8
+envlist = unit, functional
 skip_missing_interpreters = True
 
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
-         PYTHONHASHSEED=0
-         TERM=linux
-         LAYER_PATH={toxinidir}/layers
-         INTERFACE_PATH={toxinidir}/interfaces
-         JUJU_REPOSITORY={toxinidir}/build
-passenv = http_proxy https_proxy
-install_command =
-  pip install {opts} {packages}
-deps =
-    -r{toxinidir}/requirements.txt
-
-[testenv:build]
 basepython = python3
-commands =
-    charm-build --log-level DEBUG -o {toxinidir}/build src {posargs}
+setenv =
+  PYTHONPATH = .
 
-[testenv:py3]
-basepython = python3
-deps = -r{toxinidir}/test-requirements.txt
-commands = ostestr {posargs}
+[testenv:unit]
+commands = pytest -v --ignore {toxinidir}/tests/functional \
+	   --cov=lib \
+	   --cov=reactive \
+	   --cov=actions \
+	   --cov-report=term \
+	   --cov-report=annotate:reports/annotated \
+	   --cov-report=html:reports/html
+deps = -r{toxinidir}/tests/unit/requirements.txt
+       -r{toxinidir}/requirements.txt
+setenv = PYTHONPATH={toxinidir}/lib
 
-[testenv:pep8]
-basepython = python3
-deps = -r{toxinidir}/test-requirements.txt
-commands = flake8 {posargs} --max-complexity=10 --max-line-length=120 reactive files unit_tests
+[testenv:functional]
+passenv =
+  HOME
+  CHARM_BUILD_DIR
+  PATH
+  PYTEST_KEEP_MODEL
+  PYTEST_CLOUD_NAME
+  PYTEST_CLOUD_REGION
+commands = pytest -v --ignore {toxinidir}/tests/unit
+deps = -r{toxinidir}/tests/functional/requirements.txt
+       -r{toxinidir}/requirements.txt
 
-[testenv:venv]
-basepython = python3
-commands = {posargs}
+[testenv:lint]
+commands = flake8
+deps = flake8
 
 [flake8]
-# E402 ignore necessary for path append before sys module import in actions
-ignore = E402
+exclude =
+    .git,
+    __pycache__,
+    .tox,
+max-line-length = 120
+max-complexity = 10