Merge ~iconstantin/usn-tool:master into usn-tool:master

[Ian Constantin]

Initial upload of the usn-term-check.py and woke.yaml which are part of the non-inclusive and inconsistent term check process being added to our USN publication process.

[Steve Beattie (sbeattie)]

Approve with a couple of chnages:

1) if we can not reimplement load_database() and instead is the one in usn.py, that would preferred
2) some minor cleanups of spelling and trailing spaces in the code.

Other than those quibbles, it all looks good to me. Thanks!

review: Approve

diff --git a/usn-term-check.py b/usn-term-check.py
 new file mode 100755
index 0000000..ca18c13
--- /dev/null
+++ b/usn-term-check.py
@@ -0,0 +1,198 @@
+#!/usr/bin/python3
+import pickle, yaml, json, os, sys
+from usn import load_database
+from argparse import ArgumentParser
+
+def parse_options():
+    parser = ArgumentParser()
+    parser.add_argument('--db', action="store", required=True, help="Use specified database file")
+    parser.add_argument('--usn', action="store", required=True, help="Current USN")
+    parser.add_argument('--path', action="store", required=True, help="Path + usn-tool")
+    options = parser.parse_args()
+    return options
+
+def load_yaml(filename):
+    ''' Opens provided yaml file to access our non-inclusive and consistent terms '''
+
+    if not os.path.isfile(filename):
+        print("YAML file not found:", filename)
+        return {}, {}
+
+    with open(filename, 'r') as yaml_file:
+        yaml_dict = yaml.safe_load(yaml_file)
+
+    return yaml_dict["non-inclusive-terms"], yaml_dict["consistent-terms"]
+
+def load_issues(filename):
+    ''' Opens/creates a local json file to track term issues '''
+
+    issues = {}
+
+    # If issues file does not exist, create it
+    if not os.path.isfile(filename):
+        open(filename, 'w').close()
+        return issues
+
+    with open(filename, 'r') as issues_file:
+        # To handle incorrectly formatted json files which would cause an exception on json.load
+        try:
+            issues = json.load(issues_file)
+        except:
+            pass # issues will be returned as {}
+
+    return issues
+
+def record_issues(filename, issues):
+    ''' Write out any identified isses to a local json file for temporary persistence '''
+
+    with open(filename, 'w') as issues_file:
+        json.dump(issues, issues_file)
+
+def issues_exist(issues):
+    ''' Returns True if needed is as a states for any text_type+term '''
+
+    for text_type in issues:
+        if "needed" in issues[text_type].values():
+            return True
+
+    return False
+
+def check_terms(text_type, text, check_type, term_list, issues):
+    ''' Checks for and tracks non-inclusive or inconsistent terms in USN text '''
+
+    text = text.lower().strip().replace('\n', ' ') # ignoring case for text check
+
+    '''
+    To handle phrases that may span two lines, we replace all new line characters with
+    spaces, effectivley making the text we are checking one long line.
+
+    It is possible this introduces double spaces - we could replace '\n' with an empty
+    string, however, we will keep with the above approach, and afterwards use a while
+    loop to replace all occurrences of double spaces with single spaces to ensure we are
+    working with text in which all words are separated with single spaces (or other punctuation).
+    '''
+
+    # Handle any double spaces
+    while "  " in text:
+        text = text.replace("  ", " ")
+
+    for term_dict in term_list:
+        terms = term_dict['terms']
+
+        for term in terms:
+            term = term.lower().strip() # ignoring case for text check
+
+            # Perform some checks if we have previously recorded issues/warnings for this text_type and term
+            if (text_type in issues) and (term in issues[text_type]):
+
+                # If we previously ignored this term for the current text_type, skip the check
+                if issues[text_type][term] == "ignored":
+                    continue
+
+                # Updates state for terms previously in text, now changed/removed (meaning it was addressed)
+                elif (term not in text) and (issues[text_type][term] != "addressed"):
+                    issues[text_type][term] = "addressed"
+                    continue
+
+            if term in text:
+
+                # If text_type not in issues dict we will add/initialize (term will be added further down)
+                if text_type not in issues:
+                    issues[text_type] = {}
+
+                # Logic to handle non-inclusive terms
+                if check_type == "non-inclusive":
+                    print("\n---\n\nISSUE: non-inclusive term <" + term + "> found in: <" + text_type + ">")
+                    print("\tPlease consider using one of the following alternatives:")
+
+                    alternatives = term_dict["alternatives"]
+
+                    # If a "security_preferred" alternative exists, highlight it as the first presented alternative
+                    if "security_preferred" in term_dict:
+                        security_preferred = term_dict["security_preferred"][0]
+
+                        # Remove from alternatives list in order to not print it more than once
+                        if security_preferred in alternatives:
+                            alternatives.remove(security_preferred)
+
+                        print("\t\t--", security_preferred, "** preferred option **")
+
+                    # Print out the rest of the alternative term options
+                    for alternative in alternatives:
+                        print("\t\t--", alternative)
+
+                    # Providing the user the ability to ignore this term as being an issue
+                    user_choice = input("\nI) to ignore this ISSUE (any to continue): ")
+
+                    if user_choice.lower() == 'i':
+                        issues[text_type][term] = "ignored"
+                    else:
+                        issues[text_type][term] = "needed"
+
+                # Logic to handle inconsistent terms
+                elif check_type == "consistent":
+                    alternative = term_dict["alternative"][0].lower().strip()
+
+                    print("\n---\n\nWARNING: inconsistent term <" + term + "> found in: <" + text_type + ">")
+                    print("\tRecommended alternative: <" + alternative + ">")
+
+                    user_choice = input("\nI) to ignore this WARNING (any to continue): ")
+
+                    if user_choice.lower() == 'i':
+                        issues[text_type][term] = "ignored"
+                    else:
+                        issues[text_type][term] = "needed"
+
+                ''' Optional break, by default commented/not used.
+		    Avoids potential redundant/false positive reportings on phrases
+                    for the same term, e.g. "white list" and "white listed" '''
+                #break
+def main():
+
+    # Grabbing args and defining paths
+    options = parse_options()
+
+    pickle_path = options.db   # /tmp/pending-usn/$USN.pickle
+    path = options.path        # $PATH:/home/USER/DIR/usn-tool
+    usn = options.usn          # $USN
+
+    pending_usn_dir = pickle_path[:pickle_path.index(usn + ".pickle")]   # /tmp/pending-usn
+    usn_tool_dir = path.split(':')[-1]                                   # /home/USER/DIR/usn-tool
+
+    yaml_path = usn_tool_dir + "/woke.yaml"                              # /home/USER/DIR/usn-tool/woke.yaml
+    issues_file_path = pending_usn_dir + "/term_issues_" + usn           # /tmp/pending-usn/term_issues_$USN
+
+    # Extracting terms, USN text, and issues dicts
+    non_inclusive_terms, consistent_terms = load_yaml(yaml_path)
+    usn_db = load_database(pickle_path)
+    issues = load_issues(issues_file_path)
+
+    # Parsing USN text fields
+    text_fields = {"title":usn_db[usn]["title"],
+                   "summary":usn_db[usn]["summary"],
+                   "isummary":usn_db[usn]["isummary"],
+                   "description":usn_db[usn]["description"]}
+
+    # Adding source package description(s) to text_fields
+    releases = usn_db[usn]["releases"]
+
+    for release in releases:
+        for src_pkg in releases[release]["sources"]:
+            text_fields[release + " src-pkg description"] = releases[release]["sources"][src_pkg]["description"]
+
+    # Running checks for non-inlcusive and inconsistent term use
+    for text_type in text_fields:
+        check_terms(text_type, text_fields[text_type], "non-inclusive", non_inclusive_terms, issues)
+        check_terms(text_type, text_fields[text_type], "consistent", consistent_terms, issues)
+
+    # Write out ISSUES and WARNINGS to local file
+    record_issues(issues_file_path, issues)
+
+
+    if issues_exist(issues):
+        # Returning exit code of 7 to indicate that issues exist and that the new-usn bash script should exit
+        sys.exit(7)
+
+if __name__ == "__main__":
+    main()
+
diff --git a/woke.yaml b/woke.yaml
 new file mode 100644
index 0000000..c8c8b42
--- /dev/null
+++ b/woke.yaml
@@ -0,0 +1,284 @@
+####################################################################################################################
+#
+# Based on Canonical's Guidelines:
+# https://docs.google.com/document/d/1mJUa1VJHOMWa723dmKmNjSKGW-nlBv9xktzGZZwacVo/edit#heading=h.5efudb237qdb
+# ---
+# Original yaml file available here:
+# https://git.launchpad.net/lp-inclusive-naming/plain/.woke.yaml
+# ---
+#
+# non-inclusive-terms:
+# 	security_preferred
+#		- optional
+#		- specified if one alternative is identified as the goto replacement for security USNs
+#
+# ---
+#
+# consistent-terms:
+#	alternative
+#		- should be a single option to which the inconsistent term should be changed to
+#
+####################################################################################################################
+
+non-inclusive-terms:
+  - name: whitelist
+    terms:
+      - whitelist
+      - white-list
+      - whitelisted
+      - white-listed
+      - white list
+      - white listed
+    alternatives:
+      - allowlist
+      - inclusion list
+    security_preferred:
+      - allowlist
+
+  - name: blacklist
+    terms:
+      - blacklist
+      - black-list
+      - blacklisted
+      - black-listed
+      - black list
+      - black listed
+    alternatives:
+      - revoke (use for certificates)
+      - filtered
+      - denylist
+      - blocklist
+      - exclusion list
+    security_preferred:
+      - blocklist
+
+  - name: master-slave
+    terms:
+      - master-slave
+      - master/slave
+      - master slave
+      - master and slave
+    alternatives:
+      - leader/follower
+      - main/replica
+      - primary/replica
+      - primary/standby
+      - primary/secondary
+
+  - name: slave
+    terms:
+      - slave
+    alternatives:
+      - follower
+      - replica
+      - standby
+
+  - name: grandfathered
+    terms:
+      - grandfathered
+      - grand fathered
+      - grandfather
+      - grand father
+    alternatives:
+      - legacy status
+      - legacied
+      - exempted
+      - carried
+      - brought forward
+      - rolled over
+
+  - name: man-hours
+    terms:
+      - man hours
+      - man-hours
+    alternatives:
+      - person hours
+      - engineer hours
+
+  - name: sanity
+    terms:
+      - sanity
+    alternatives:
+      - consistency check
+      - validate
+      - confidence
+      - clarity check
+      - quick check
+      - coherence check
+
+  - name: dummy
+    terms:
+      - dummy
+    alternatives:
+      - placeholder
+      - sample
+
+  - name: guys
+    terms:
+      - guys
+    alternatives:
+      - folks
+      - people
+      - you all
+      - y'all
+      - yinz
+
+  - name: whitebox
+    terms:
+      - white-box
+      - whitebox
+      - white box
+    alternatives:
+      - open-box
+
+  - name: blackbox
+    terms:
+      - black-box
+      - blackbox
+      - black box
+    alternatives:
+      - closed-box
+
+  - name: blackhat
+    terms:
+      - blackhat
+      - black-hat
+      - black hat
+    alternatives:
+      - attacker
+      - malicious actor
+
+  - name: whitehat
+    terms:
+      - whitehat
+      - white-hat
+      - white hat
+    alternatives:
+      - researcher
+      - security specialist
+
+  - name: illegal characters
+    terms:
+      - illegal characters
+    alternatives:
+      - invalid characters
+      - unsupported characters
+      - special characters
+    security_preferred:
+      - invalid characters
+
+  - name: native feature
+    terms:
+      - native feature
+    alternatives:
+      - core feature
+      - built-in feature
+    security_preferred:
+      - built-in feature
+
+  - name: chairman/foreman
+    terms:
+      - chairman
+      - chair man
+      - foreman
+      - fore man
+    alternatives:
+      - chair
+      - foreperson
+
+  - name: man in the middle
+    terms:
+      - man in the middle
+      - man-in-the-middle
+      - mitm
+    alternatives:
+      - machine-in-the-middle
+      - person-in-the-middle
+      - system-in-the-middle
+      - intermediary attack
+    security_preferred:
+      - machine-in-the-middle
+
+  - name: middleman
+    terms:
+      - middleman
+      - middle man
+    alternatives:
+      - middleperson
+      - intermediary
+    security_preferred:
+      - intermediary
+
+  - name: manned
+    terms:
+      - manned
+    alternatives:
+      - crewed
+      - staffed
+      - monitored
+      - human operated
+
+  - name: mom test / girlfriend test
+    terms:
+      - mom test
+      - mom-test
+      - girlfriend test
+      - girlfriend-test
+    alternatives:
+      - user test
+      - user friendly
+    security_preferred:
+      - user test
+
+  - name: crazy
+    terms:
+      - crazy
+    alternatives:
+      - baffling
+      - unexplained
+      - errant
+
+  - name: cripples
+    terms:
+      - cripples
+    alternatives:
+      - slows down
+      - hinders
+      - obstructs
+
+  - name: crippling
+    terms:
+      - crippling
+    alternatives:
+      - attenuating
+      - incapacitating
+
+  - name: stonith/stomith
+    terms:
+      - stonith
+      - stomith
+    alternatives:
+      - fence failed nodes
+      - machines
+
+  - name: demilitarized zone
+    terms:
+      - demilitarized zone
+      - dmz
+    alternatives:
+      - perimeter network
+      - passthrough network
+
+consistent-terms:
+  - name: file system
+    terms:
+      - file system
+      - file-system
+    alternative:
+      - filesystem
+
+  - name: machine in the middle
+    terms:
+      - machine in the middle
+    alternative:
+      - machine-in-the-middle