Merge ~peppepetra/charm-graylog:lp1805043 into ~graylog-charmers/charm-graylog:master

[Giuseppe Petralia]

Add ignore-indexer-failures action

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Zachary Zehring (zzehring)]

Added comments.

83b85b0... by Giuseppe Petralia on 2020-02-18

Address review comments:

- use with open(file) instead of f = open()
- improve action descritpion and comments
- refactor variable names to be more explicit

[Xav Paice (xavpaice)]

This change fails lint with ./actions/actions.py:20:1: E402 module level import not at top of file

Unfortunately the coverage for unit tests doesn't include a lot of the files touched in this change, but fwiw, they pass OK.

review: Needs Fixing

c2f2554... by Giuseppe Petralia on 2020-02-20

Fix linting

[Xav Paice (xavpaice)]

Tests work OK, LGTM

review: Approve

diff --git a/actions.yaml b/actions.yaml
index 7528e95..c8d6f78 100644
--- a/actions.yaml
+++ b/actions.yaml
@@ -9,3 +9,7 @@ set-admin-password:
       type: string
   required:
     - password
+ignore-indexer-failures:
+  description: |
+    From NRPE check viewpoint, ignore all indexer failures that have occurred before the action is run.
+    Future indexer failure occurrences will still be checked.
diff --git a/actions/actions.py b/actions/actions.py
index eed1233..5b9808d 100755
--- a/actions/actions.py
+++ b/actions/actions.py
@@ -1,5 +1,6 @@
 #!/usr/local/sbin/charm-env python3
 
+import datetime
 import os
 import sys
 import traceback
@@ -16,6 +17,8 @@ if libs_dir not in sys.path:
 from charms import reactive  # noqa E402
 from charmhelpers.core import hookenv, host, unitdata  # noqa E402
 
+from charms.layer.graylog.api import get_ignore_indexer_failures_file   # noqa E402
+
 
 def reactive_remove_state(state):
     reactive.remove_state(state)
@@ -49,6 +52,11 @@ def set_admin_passwd(admin_password=None):
     reactive_remove_state('graylog.configured')
 
 
+def ignore_indexer_failures():
+    with open(get_ignore_indexer_failures_file(), 'w') as f:
+        f.write(datetime.datetime.utcnow().isoformat())
+
+
 def main(argv):
     action = os.path.basename(argv[0])
     params = hookenv.action_get()
@@ -57,6 +65,8 @@ def main(argv):
             get_admin_passwd()
         elif action == 'set-admin-password':
             set_admin_passwd(params['password'])
+        elif action == 'ignore-indexer-failures':
+            ignore_indexer_failures()
         else:
             hookenv.action_fail('Action {} not implemented'.format(action))
     except Exception:
diff --git a/actions/ignore-indexer-failures b/actions/ignore-indexer-failures
new file mode 120000
index 0000000..405a394
--- /dev/null
+++ b/actions/ignore-indexer-failures
@@ -0,0 +1 @@
+actions.py
\ No newline at end of file
diff --git a/files/check_graylog_health.py b/files/check_graylog_health.py
index 7fde420..2cbb597 100755
--- a/files/check_graylog_health.py
+++ b/files/check_graylog_health.py
@@ -13,10 +13,10 @@ if libs_dir not in sys.path:
 
 try:
     # api will be under $CHARM_SOURCE_DIR/lib during unit tests
-    from charms.layer.graylog.api import GraylogApi
+    from charms.layer.graylog.api import GraylogApi, get_ignore_indexer_failures_file
 except ImportError:
     # api will be under $NAGIOS_PLUGIN_DIR at runtime
-    from api import GraylogApi
+    from api import GraylogApi, get_ignore_indexer_failures_file
 
 STATE_OK = 0
 STATE_WARNING = 1
@@ -87,6 +87,20 @@ def outstanding_notifications(gapi_con, args):
 
 def indexer_failures(gapi_con, args):
     since = datetime.datetime.utcnow() - datetime.timedelta(days=1)
+
+    # the variable "since" is equal to now() - 1 day or to ignore_indexer_failures action timestamp
+    # if ignore_indexer_failures action timestamp is more recent than 1 day.
+    try:
+        with open(get_ignore_indexer_failures_file(), 'r') as f:
+            file_content = f.read()
+            ignore_indexer_failures_since = datetime.datetime.strptime(file_content.strip(), "%Y-%m-%dT%H:%M:%S.%f")
+
+            if ignore_indexer_failures_since > since:
+                since = ignore_indexer_failures_since
+    except (FileNotFoundError, ValueError):
+        # Ignore actioned_since if file not exists or not contains valid ISO date
+        pass
+
     failures = gapi_con.indexer_failures_count(since.isoformat())['count']
     msg = 'Indexer failures: {}'.format(failures)
     if failures == 0:
diff --git a/lib/charms/layer/graylog/api.py b/lib/charms/layer/graylog/api.py
index 87ded7f..b38c2cc 100644
--- a/lib/charms/layer/graylog/api.py
+++ b/lib/charms/layer/graylog/api.py
@@ -20,6 +20,10 @@ else:
     charm = False
 
 
+def get_ignore_indexer_failures_file():
+    return "/usr/local/lib/nagios/plugins/ignore_indexer_failures.timestamp"
+
+
 class GraylogApi:
 
     def __init__(self, base_url, username, password, token_name='graylog-api'):