Merge ~vicamo/+git/hwe-tools:for-upstream/support-upstream-statements into ~vicamo/+git/hwe-tools:master

[You-Sheng Yang]

* remove debug prints
* detect `[ Upstream commit ... ]` and `commit ... upstream.`.

TODO: figure out why there are sometimes:

  fatal: bad object ...
  fatal: Not a valid commit name ...

diff --git a/find-upstream-fixes b/find-upstream-fixes
index 358df33..f1d6874 100755
--- a/find-upstream-fixes
+++ b/find-upstream-fixes
@@ -1,10 +1,40 @@
 #!/bin/bash
 
+BOT=0
+BOT_HEADER=""
+BOT_COMMITS=""
+QUIET=0
+
+function println() {
+  if [ ${QUIET} -ne 1 ] || [ ! -z $2 ]; then
+    if [ ${BOT} -ne 1 ]; then
+      echo "$1"
+    fi
+  fi
+}
+
 if [ $# -lt 2 ]; then
-  echo "Usage: ${BASH_SOURCE[0]} <ref-range> <find1>[ <find2>...]" >&2
+  echo "Usage: ${BASH_SOURCE[0]} [-b -q] <ref-range> <find1>[ <find2>...]" >&2
   exit 1
 fi
 
+OPTIND=1
+while getopts ":bqh?" opt; do
+    case ${opt} in
+        b)
+            QUIET=1
+            BOT=1
+            ;;
+        q)
+            QUIET=1
+            ;;
+        *)
+            ;;
+    esac
+done
+
+shift $((OPTIND-1))
+[ "${1:-}" = "--" ] && shift
 COMMIT_RANGE=$1
 shift;
 TARGET_TREES=("$@")
@@ -14,22 +44,31 @@ declare -A cherry_picked;
 
 header_printed=
 for commit in $(git log --reverse --pretty=format:%h "${COMMIT_RANGE}"); do
-  corb=( $(git log -1 --pretty=format:%b "${commit}" | \
+  msg="$(git log -1 --pretty=format:%b "${commit}")"
+  todo=( $(echo "${msg}" | \
            grep -E '^\((cherry picked|backported) from commit ' | \
            sed -E 's,^\((cherry picked|backported) from commit ([a-f0-9]+).*,\2,') )
-  if [ ${#corb} -ne 0 ]; then
-    cherry_picked["${commit}"]="${corb[*]}"
+  todo+=( $(echo "${msg}" | \
+            grep -E '^\[ Upstream commit [a-f0-9]+ \]$' | \
+            sed -E 's,^\[ Upstream commit ([a-f0-9]+) \]$,\1,') )
+  todo+=( $(echo "${msg}" | \
+            grep -E '^commit [a-f0-9]+ upstream.$' | \
+            sed -E 's,^commit ([a-f0-9]+) upstream.$,\1,') )
+  if [ ${#todo} -ne 0 ]; then
+    cherry_picked["${commit}"]="${todo[*]}"
     continue
   fi
 
   if [ -z "${header_printed}" ]; then
-    echo "* No cherry-pick/backport statement found:"
+    println "* No cherry-pick/backport/upstream statement found:"
     header_printed=yes
   fi
 
-  echo "  * commit ${commit} (\"$(git log -1 --pretty=format:%s "${commit}")\")"
-  git log -1 --pretty=format:%b "${commit}"
-  echo
+  println "  * commit ${commit} (\"$(git log -1 --pretty=format:%s "${commit}")\")"
+  if [ ${QUIET} -ne 1 ]; then
+    git log -1 --pretty=format:%b "${commit}"
+  fi
+  println ""
 done
 
 for checking in "${!cherry_picked[@]}"; do
@@ -38,31 +77,55 @@ for checking in "${!cherry_picked[@]}"; do
 
   for needle in ${origins}; do
     needle_printed=
-    needle_short=$(git log -1 --pretty=format:%h "${needle}")
+    if [ ${QUIET} -eq 1 ]; then
+      needle_short=$(git log -1 --pretty=format:%h "${needle}" 2> /dev/null)
+    else
+      needle_short=$(git log -1 --pretty=format:%h "${needle}")
+    fi
 
+    BOT_HEADER=""
     for haystack in "${TARGET_TREES[@]}"; do
-      git merge-base --is-ancestor "${needle}" "${haystack}" || continue
+      if [ ${QUIET} -eq 1 ]; then
+        git merge-base --is-ancestor "${needle}" "${haystack}" 2> /dev/null || continue
+      else
+        git merge-base --is-ancestor "${needle}" "${haystack}" || continue
+      fi
 
       found=( $(git log -P --grep="^\s*Fixes:.*\s+${needle_short}" --pretty="format:%h" "${needle_short}..${haystack}") )
       [ ${#found} -ne 0 ] || continue
 
       if [ -z "${header_printed}" ]; then
-        echo "###### Checking commit ${checking} (\"$(git log -1 --pretty=format:%s "${checking}")\") ######"
+        println "###### Checking commit ${checking} (\"$(git log -1 --pretty=format:%s "${checking}")\") ######" 1
         header_printed=yes
+        if [ ${BOT} -eq 1 ]; then
+          branch=`git branch --show-current`
+          BOT_HEADER="###### Branch ${branch} commit ${checking} (\"$(git log -1 --pretty=format:%s "${checking}")\") ######"
+          BOT_COMMITS=""
+	fi
       fi
       if [ -z "${needle_printed}" ]; then
-        echo "  * Checking for origin ${needle_short} ..."
+        println "  * Checking for origin ${needle_short} ..." 1
         needle_printed=yes
       fi
 
-      echo "      fixes found in ${haystack}: ${found[*]}"
+      println "      fixes found in ${haystack}: ${found[*]}" 1
+      if [ ${BOT} -eq 1 ]; then
+        BOT_COMMITS+="      fixes found in ${haystack}: ${found[*]}\n"
+	break
+      fi
       for fix in "${found[@]}"; do
         case " ${cherry_picked[@]} " in
         *" ${fix}"*) format="(applied) %h %s" ;;
         *)           format="(missing) %h %s" ;;
         esac
-        git log -1 --pretty="format:        ${format}" "${fix}"
+        if [ ${QUIET} -ne 1 ]; then
+          git log -1 --pretty="format:        ${format}" "${fix}"
+	fi
       done
     done
+    if [ ${BOT} -eq 1 ] && [ ! -z "${BOT_HEADER}" ] ; then
+      echo -e "${BOT_HEADER}"
+      echo -e "${BOT_COMMITS}"
+    fi
   done
 done
diff --git a/get-oem-delta.sh b/get-oem-delta.sh
index 89f617c..540e7c3 100755
--- a/get-oem-delta.sh
+++ b/get-oem-delta.sh
@@ -11,7 +11,7 @@
 # Where your linux-oem tree lives
 OEM_KERNEL_TREE=
 OEM_KERNEL_BRANCH="oem"
-GIT_PULL=0
+GIT_PULL=1
 WIDE_MODE=0
 
 bye()       { cd "$pwd"; exit $1; }
@@ -24,9 +24,9 @@ usage()
     echo -e "  -b \e[3m<oem_kernel_branch>\e[0m\tBranch name to checkout in OEM kernel git repo, default is 'oem'";
     echo -e "  -t \e[3m<target_git_repo>\e[0m\t\tGit repo to check whether commits have landed";
     echo -e "  -B \e[3m<target_repo_branch>\e[0m\tBranch to switch to in the target kernel git repo (optional)";
-    echo -e "  -p \t\t\t\tRuns git pull --rebase in the source and target git repos, default is not to git pull";
     echo -e "  -w \t\t\t\tWide mode, suitable for file output, not suitable for screen display";
     echo -e "  -T \e[3m<hash>\e[0m\t\t\tIf \e[3m<hash>\e[0m is provided then will only test this hash";
+    echo -e "  -P \t\t\t\tDO NOT run 'git pull --rebase' in the source and target git repos";
     echo -e "  -d \t\t\t\tDebug mode";
     echo -e "  -h \t\t\t\tPrint this help";
     echo
@@ -39,7 +39,7 @@ escape()
     echo $1 | sed -E 's/([][*])/\\\1/g' 
 }
 
-while getopts ":hps:b:t:B:wdT:" opt; do
+while getopts ":hPs:b:t:B:wdT:" opt; do
     case ${opt} in
         s)
             if [ -d $OPTARG ]; then
@@ -61,8 +61,8 @@ while getopts ":hps:b:t:B:wdT:" opt; do
         B)
             TARGET_GIT_BRANCH=$OPTARG
             ;;
-        p)
-            GIT_PULL=1
+        P)
+            GIT_PULL=0
             ;;
         w)
             WIDE_MODE=1
@@ -87,27 +87,35 @@ shift $((OPTIND -1))
 
 pwd="$PWD"
 
-cd "$OEM_KERNEL_TREE"
+cd "$target_repo"
 (
-git remote | grep -q mainline || (print_err "mainline remote missing in OEM kernel tree"; bye 1)
-git checkout $OEM_KERNEL_BRANCH
+[ -n $TARGET_GIT_BRANCH ] && git checkout $TARGET_GIT_BRANCH
 [ $GIT_PULL = 1 ] && git pull --rebase
 ) 1>&2
+target_origin=`git remote get-url origin`
+target_head_hash=`git rev-parse --short HEAD`
+target_branch=`git branch --show-current`
 
+cd "$OEM_KERNEL_TREE"
 (
-cd "$target_repo"
-[ -n $TARGET_GIT_BRANCH ] && git checkout $TARGET_GIT_BRANCH
+git remote | grep -q mainline || (print_err "mainline remote missing in OEM kernel tree"; bye 1)
+git checkout $OEM_KERNEL_BRANCH
+git fetch mainline
 [ $GIT_PULL = 1 ] && git pull --rebase
 ) 1>&2
-
+source_origin=`git remote get-url origin`
+source_head_hash=`git rev-parse --short HEAD`
+source_branch=`git branch --show-current`
 
 FIRST_AND_LAST_HASH=( `git log --oneline -E --grep "UBUNTU: Ubuntu-[1-9]+" | sort -rV -k3 | awk 'NR==1 { print $1 }; END { print $1 }'` )
 LATEST_UBUNTU_HASH=${FIRST_AND_LAST_HASH[0]}
 EARLIEST_UBUNTU_HASH_DATE=`git log -1 --format=%aI ${FIRST_AND_LAST_HASH[1]}`
 
 printf "Report generated at "; date
+printf "Source: $source_origin $source_head_hash ($source_branch)\n"
+printf "Target: $target_origin $target_head_hash ($target_branch)\n"
 printf "T=Type: C=Cherry-pick, B=Backport, S=SAUCE patch, R=Revert, F=conFig\n"
-printf "L=Landed in %s?\n" "${target_repo}"
+printf "L=Landed?\n"
 printf " + Y=Landed, N=Not landed, M=Log messages differ but w/ identical patch contents\n"
 printf " + x%%=Commit with same log message found but patch contents differ\n"
 perl -e 'print "-" x 180; print "\n"'
diff --git a/hwe-bug-checker.py b/hwe-bug-checker.py
index 89fdbc5..ec8f51a 100755
--- a/hwe-bug-checker.py
+++ b/hwe-bug-checker.py
@@ -12,7 +12,7 @@ from launchpadlib.uris                  import lookup_service_root
 
 APP_NAME = 'hwe-bug-checker'
 SERVICE_ROOT = 'production'
-HWE_OEM_PROJECTS = ['somerville/hwe-somerville', 'sutton/hwe-sutton', 'stella/hwe-stella', 'busan/hwe-busan', 'nashville/hwe-nashville', 'amaro/hwe-amaro', 'timbuktu/hwe-timbuktu', 'denver/hwe-denver']
+HWE_OEM_PROJECTS = ['somerville/hwe-somerville', 'sutton/hwe-sutton', 'stella/hwe-stella', 'busan/hwe-busan', 'nashville/hwe-nashville', 'amaro/hwe-amaro', 'timbuktu/hwe-timbuktu', 'denver/hwe-denver', 'anle/hwe-anle', 'pygmy-possum/hwe-pygmy-possum']
 
 logger = getLogger(APP_NAME)
 
diff --git a/hwe-bug-monitor.py b/hwe-bug-monitor.py
index 2aff1cf..519776b 100755
--- a/hwe-bug-monitor.py
+++ b/hwe-bug-monitor.py
@@ -1,27 +1,38 @@
 #!/usr/bin/python3
 
+import re
+
 from os                                 import path
 from subprocess                         import getoutput, getstatusoutput
 
 HWE_BUG_CHECKER = '~/hwe-tools/hwe-bug-checker.py'
-OEM_KERNELS = ['bionic git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oem/+git/bionic oem-next',
-               'focal git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oem/+git/focal oem-5.6-next',
-               'focal git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oem/+git/focal oem-5.10-next']
+OEM_KERNELS = ['focal git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oem/+git/focal oem-5.14-next Ubuntu-oem-5.14',
+               'jammy git://git.launchpad.net/~canonical-kernel/ubuntu/+source/linux-oem/+git/jammy oem-5.17-next Ubuntu-oem-5.17']
 
 for kernel in OEM_KERNELS:
     local = kernel.split(' ')[0]
     repo = kernel.split(' ')[1]
     branch = kernel.split(' ')[2]
+    tag_prefix = kernel.split(' ')[3]
     print("%s %s %s" % (local, repo, branch))
     if not path.exists(local):
         getoutput("git clone --depth 1 --single-branch --branch %s %s %s" % (branch, repo, local))
-    else:
-        status, result = getstatusoutput("cd %s; git rev-parse --verify %s" % (local, branch))
-        if status != 0:
-            print("hwe-bug-checker: adding new branch - %s" % branch)
-            getoutput("cd %s; git remote set-branches --add origin %s" % (local, branch))
+    status, result = getstatusoutput("cd %s; git rev-parse --verify %s" % (local, branch))
+    if status != 0:
+        print("hwe-bug-checker: adding new branch - %s" % branch)
+        getoutput("cd %s; git remote set-branches --add origin %s" % (local, branch))
+    getoutput("cd %s; git remote set-branches origin %s" % (local, branch))
     result = getoutput("(cd %s; git fetch origin)" % local)
-    if 'new tag' in result:
+    for line in result.splitlines():
+        tag_pattern = re.compile(r" \* \[new tag\] +(.*) -> (.*)")
+        tag = tag_pattern.match(line)
+        if tag != None:
+            if tag_prefix not in tag[1]:
+                getoutput("cd %s; git tag -d %s" % (local, tag[1]))
+                continue
+        else:
+            continue
+
         getoutput("(cd %s; git checkout -B %s origin/%s)" % (local, branch, branch))
         result = getoutput("(cd %s; %s)" % (local, HWE_BUG_CHECKER))
         print("hwe-bug-checker: %s" % result)