repo created

scripts/analysis/analysis-wrapper.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+BRANCH=$1
+LOG_USERNAME=$2
+LOG_PASSWORD=$3
+BUILD_NUMBER=$4
+PR_NUMBER=$5
+
+
+stableBranch="master"
+repository="talk-android"
+
+ruby scripts/analysis/lint-up.rb
+lintValue=$?
+
+curl "https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.xml" -o "/tmp/$stableBranch.xml"
+ruby scripts/analysis/spotbugs-up.rb "$stableBranch"
+spotbugsValue=$?
+
+# exit codes:
+# 0: count was reduced
+# 1: count was increased
+# 2: count stayed the same
+
+source scripts/lib.sh
+
+echo "Branch: $BRANCH"
+
+if [ "$BRANCH" = $stableBranch ]; then
+    echo "New spotbugs result for $stableBranch at: https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html"
+    curl -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT https://nextcloud.kaminsky.me/remote.php/dav/files/${LOG_USERNAME}/$repository-findbugs/$stableBranch.html --upload-file app/build/reports/spotbugs/spotbugs.html
+    curl 2>/dev/null -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT "https://nextcloud.kaminsky.me/remote.php/dav/files/${LOG_USERNAME}/$repository-findbugs/$stableBranch.xml" --upload-file app/build/reports/spotbugs/gplayDebug.xml
+
+    if [ $lintValue -ne 1 ]; then
+        echo "New lint result for $stableBranch at: https://www.kaminsky.me/nc-dev/$repository-lint/$stableBranch.html"
+        curl -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT https://nextcloud.kaminsky.me/remote.php/dav/files/${LOG_USERNAME}/$repository-lint/$stableBranch.html --upload-file app/build/reports/lint/lint.html
+        exit 0
+    fi
+else
+    if [ -e "${BUILD_NUMBER}" ]; then
+        6=$stableBranch"-"$(date +%F)
+    fi
+    echo "New lint results at https://www.kaminsky.me/nc-dev/$repository-lint/${BUILD_NUMBER}.html"
+    curl 2>/dev/null -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT "https://nextcloud.kaminsky.me/remote.php/dav/files/${LOG_USERNAME}/$repository-lint/${BUILD_NUMBER}.html" --upload-file app/build/reports/lint/lint.html
+
+    echo "New spotbugs results at https://www.kaminsky.me/nc-dev/$repository-findbugs/${BUILD_NUMBER}.html"
+    curl 2>/dev/null -u "${LOG_USERNAME}:${LOG_PASSWORD}" -X PUT "https://nextcloud.kaminsky.me/remote.php/dav/files/${LOG_USERNAME}/$repository-findbugs/${BUILD_NUMBER}.html" --upload-file app/build/reports/spotbugs/spotbugs.html
+
+    # delete all old comments, starting with Codacy
+    oldComments=$(curl_gh -X GET "https://api.github.com/repos/nextcloud/$repository/issues/${PR_NUMBER}/comments" | jq '.[] | select((.user.login | contains("github-actions")) and  (.body | test("<h1>Codacy.*"))) | .id')
+
+    echo "$oldComments" | while read -r comment ; do
+        curl_gh -X DELETE "https://api.github.com/repos/nextcloud/$repository/issues/comments/$comment"
+    done
+
+    # lint and spotbugs file must exist
+    if [ ! -s app/build/reports/lint/lint.html ] ; then
+        echo "lint.html file is missing!"
+        exit 1
+    fi
+
+    if [ ! -s app/build/reports/spotbugs/spotbugs.html ] ; then
+        echo "spotbugs.html file is missing!"
+        exit 1
+    fi
+
+    # add comment with results
+    lintResultNew=$(grep "Lint Report.* [0-9]* warning" app/build/reports/lint/lint.html | cut -f2 -d':' |cut -f1 -d'<')
+
+    lintErrorNew=$(echo $lintResultNew | grep "[0-9]* error" -o | cut -f1 -d" ")
+    if ( [ -z $lintErrorNew ] ); then
+        lintErrorNew=0
+    fi
+
+    lintWarningNew=$(echo $lintResultNew | grep "[0-9]* warning" -o | cut -f1 -d" ")
+    if ( [ -z $lintWarningNew ] ); then
+        lintWarningNew=0
+    fi
+
+    lintResultOld=$(curl 2>/dev/null "https://raw.githubusercontent.com/nextcloud/$repository/$stableBranch/scripts/analysis/lint-results.txt")
+    lintErrorOld=$(echo $lintResultOld | grep "[0-9]* error" -o | cut -f1 -d" ")
+    if ( [ -z $lintErrorOld ] ); then
+        lintErrorOld=0
+    fi
+
+    lintWarningOld=$(echo $lintResultOld | grep "[0-9]* warning" -o | cut -f1 -d" ")
+    if ( [ -z $lintWarningOld ] ); then
+        lintWarningOld=0
+    fi
+
+    if [ $stableBranch = "master" ] ; then
+        codacyValue=$(curl 2>/dev/null https://app.codacy.com/gh/nextcloud/$repository/dashboard | grep "total issues" | cut -d">" -f3 | cut -d"<" -f1)
+        codacyResult="<h1>Codacy</h1>$codacyValue"
+    else
+        codacyResult=""
+    fi
+
+    lintResult="<h1>Lint</h1><table width='500' cellpadding='5' cellspacing='2'><tr class='tablerow0'><td>Type</td><td><a href='https://www.kaminsky.me/nc-dev/$repository-lint/$stableBranch.html'>$stableBranch</a></td><td><a href='https://www.kaminsky.me/nc-dev/$repository-lint/${BUILD_NUMBER}.html'>PR</a></td></tr><tr class='tablerow1'><td>Warnings</td><td>$lintWarningOld</td><td>$lintWarningNew</td></tr><tr class='tablerow0'><td>Errors</td><td>$lintErrorOld</td><td>$lintErrorNew</td></tr></table>"
+
+    spotbugsResult="<h1>SpotBugs</h1>$(scripts/analysis/spotbugsComparison.py "/tmp/$stableBranch.xml" app/build/reports/spotbugs/gplayDebug.xml --link-new "https://www.kaminsky.me/nc-dev/$repository-findbugs/${BUILD_NUMBER}.html" --link-base "https://www.kaminsky.me/nc-dev/$repository-findbugs/$stableBranch.html")"
+
+    if ( [ $lintValue -eq 1 ] ) ; then
+        lintMessage="<h1>Lint increased!</h1>"
+    fi
+
+    if ( [ $spotbugsValue -eq 1 ] ) ; then
+        spotbugsMessage="<h1>SpotBugs increased!</h1>"
+    fi
+
+    # check gplay limitation: all changelog files must only have 500 chars
+    gplayLimitation=$(scripts/checkGplayLimitation.sh)
+
+    if [ ! -z "$gplayLimitation" ]; then
+        gplayLimitation="<h1>Following files are beyond 500 char limit:</h1><br><br>"$gplayLimitation
+    fi
+
+    # check for NotNull
+    if [[ $(grep org.jetbrains.annotations app/src/main/* -irl | wc -l) -gt 0 ]] ; then
+        notNull="org.jetbrains.annotations.NotNull is used. Please use androidx.annotation.NonNull instead.<br><br>"
+    fi
+
+    bodyContent="$codacyResult $lintResult $spotbugsResult $lintMessage $spotbugsMessage $gplayLimitation $notNull"
+    echo "$bodyContent" >> "$GITHUB_STEP_SUMMARY"
+    payload="{ \"body\" : \"$bodyContent\" }"
+    curl_gh -X POST "https://api.github.com/repos/nextcloud/$repository/issues/${PR_NUMBER}/comments" -d "$payload"
+
+    if [ ! -z "$gplayLimitation" ]; then
+        exit 1
+    fi
+
+    if [ ! $lintValue -eq 2 ]; then
+        exit $lintValue
+    fi
+
+    if [ -n "$notNull" ]; then
+        exit 1
+    fi
+
+    if [ $spotbugsValue -eq 2 ]; then
+        exit 0
+    else
+        exit $spotbugsValue
+    fi
+fi

scripts/analysis/getBranchName.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# $1: username, $2: password/token, $3: pull request number
+
+if [ -z "$3" ] ; then
+    git branch | grep '\*' | cut -d' ' -f2
+else
+    curl 2>/dev/null -u "$1":"$2" "https://api.github.com/repos/nextcloud/talk-android/pulls/$3" | jq .head.ref
+fi

scripts/analysis/lint-results.txt

@@ -0,0 +1,2 @@
+DO NOT TOUCH; GENERATED BY DRONE
+      <span class="mdl-layout-title">Lint Report: 99 warnings</span>

scripts/analysis/lint-results.txt.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+SPDX-License-Identifier: GPL-3.0-or-later

scripts/analysis/lint-up.rb

@@ -0,0 +1,191 @@
+## Script from https://github.com/tir38/android-lint-entropy-reducer at 07.05.2017
+# adapts to drone, use git username / token as parameter
+
+# TODO cleanup this script, it has a lot of unused stuff
+
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Jason Atwood 
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+puts "=================== starting Android Lint Entropy Reducer ===================="
+
+# ========================  SETUP ============================
+
+# User name for git commits made by this script.
+TRAVIS_GIT_USERNAME = String.new("Drone CI server")
+
+# File name and relative path of generated Lint report. Must match build.gradle file:
+#   lintOptions {
+#       htmlOutput file("[FILE_NAME].html")
+#   }
+LINT_REPORT_FILE = String.new("app/build/reports/lint/lint.html")
+
+# File name and relative path of previous results of this script.
+PREVIOUS_LINT_RESULTS_FILE=String.new("scripts/analysis/lint-results.txt")
+
+# Flag to evaluate warnings. true = check warnings; false = ignore warnings
+CHECK_WARNINGS = true
+
+# File name and relative path to custom lint rules; Can be null or "".
+CUSTOM_LINT_FILE = String.new("")
+
+# ================ SETUP DONE; DON'T TOUCH ANYTHING BELOW  ================
+
+require 'fileutils'
+require 'pathname'
+require 'open3'
+
+# since we need the xml-simple gem, and we want this script self-contained, let's grab it just when we need it
+begin
+    gem "xml-simple"
+    rescue LoadError
+    system("gem install --user-install xml-simple")
+    Gem.clear_paths
+end
+
+require 'xmlsimple'
+
+# add custom Lint jar
+if !CUSTOM_LINT_FILE.nil? &&
+    CUSTOM_LINT_FILE.length > 0
+
+    ENV["ANDROID_LINT_JARS"] = Dir.pwd + "/" + CUSTOM_LINT_FILE
+    puts "adding custom lint rules to default set: "
+    puts ENV["ANDROID_LINT_JARS"]
+end
+
+# run Lint
+puts "running Lint..."
+system './gradlew clean lintGplayDebug 1>/dev/null'
+
+# confirm that Lint ran w/out error
+result = $?.to_i
+if result != 0
+    puts "FAIL: failed to run ./gradlew clean lintGplayDebug"
+    exit 1
+end
+
+# find Lint report file
+lint_reports = Dir.glob(LINT_REPORT_FILE)
+if lint_reports.length == 0
+    puts "Lint HTML report not found."
+    exit 1
+end
+lint_report = String.new(lint_reports[0])
+
+# find error/warning count string in HTML report
+error_warning_string = ""
+File.open lint_report do |file|
+  error_warning_string = file.find { |line| line =~ /([0-9]* error[s]? and )?[0-9]* warning[s]?/ }
+end
+
+# find number of errors
+error_string = error_warning_string.match(/[0-9]* error[s]?/)
+
+if (error_string.nil?)
+    current_error_count = 0
+else
+    current_error_count = error_string[0].match(/[0-9]*/)[0].to_i
+end
+
+puts "found errors: " + current_error_count.to_s
+
+# find number of warnings
+if CHECK_WARNINGS == true
+    warning_string = error_warning_string.match(/[0-9]* warning[s]?/)[0]
+    current_warning_count = warning_string.match(/[0-9]*/)[0].to_i
+    puts "found warnings: " + current_warning_count.to_s
+end
+
+# get previous error and warning counts from last successful build
+
+previous_results = false
+
+previous_lint_reports = Dir.glob(PREVIOUS_LINT_RESULTS_FILE)
+if previous_lint_reports.nil? ||
+    previous_lint_reports.length == 0
+
+    previous_lint_report = File.new(PREVIOUS_LINT_RESULTS_FILE, "w") # create for writing to later
+else
+    previous_lint_report = String.new(previous_lint_reports[0])
+
+    previous_error_warning_string = ""
+    File.open previous_lint_report do |file|
+      previous_error_warning_string = file.find { |line| line =~ /([0-9]* error[s]? and )?[0-9]* warning[s]?/ }
+    end
+
+    unless previous_error_warning_string.nil?
+        previous_results = true
+
+        previous_error_string = previous_error_warning_string.match(/[0-9]* error[s]?/)
+        if previous_error_string.nil?
+            previous_error_string = "0 errors"
+        else
+            previous_error_string = previous_error_string[0]
+        end
+        previous_error_count = previous_error_string.match(/[0-9]*/)[0].to_i
+        puts "previous errors: " + previous_error_count.to_s
+
+        if CHECK_WARNINGS == true
+            previous_warning_string = previous_error_warning_string.match(/[0-9]* warning[s]?/)
+            if previous_warning_string.nil?
+                previous_warning_string = "0 warnings"
+            else
+                previous_warning_string = previous_warning_string[0]
+            end
+            previous_warning_count = previous_warning_string.match(/[0-9]*/)[0].to_i
+            puts "previous warnings: " + previous_warning_count.to_s
+        end
+    end
+end
+
+# compare previous error count with current error count
+if previous_results == true  &&
+    current_error_count > previous_error_count
+    puts "FAIL: error count increased"
+    exit 1
+end
+
+# compare previous warning count with current warning count
+if CHECK_WARNINGS  == true &&
+    previous_results == true &&
+    current_warning_count > previous_warning_count
+
+    puts "FAIL: warning count increased"
+    exit 1
+end
+
+# check if warning and error count stayed the same
+if  previous_results == true &&
+    current_error_count == previous_error_count &&
+    current_warning_count == previous_warning_count
+
+    puts "SUCCESS: count stayed the same"
+    exit 2
+end
+
+# either error count or warning count DECREASED
+
+# write new results to file (will overwrite existing, or create new)
+File.write(previous_lint_report, "DO NOT TOUCH; GENERATED BY DRONE\n" + error_warning_string)
+
+# update git user name and email for this script
+system ("git config --local user.name 'github-actions'")
+system ("git config --local user.email 'github-actions@github.com'")
+
+# add previous Lint result file to git
+system ('git add ' + PREVIOUS_LINT_RESULTS_FILE)
+
+# commit changes
+system('git commit -sm "Analysis: update lint results to reflect reduced error/warning count"')
+
+# push to origin
+system ('git push')
+
+puts "SUCCESS: count was reduced"
+exit 0 # success

scripts/analysis/spotbugs-up.rb

@@ -0,0 +1,53 @@
+## Script originally from https://github.com/tir38/android-lint-entropy-reducer at 07.05.2017
+# heavily modified since then
+
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Jason Atwood 
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+puts "=================== starting Android Spotbugs Entropy Reducer ===================="
+
+# get args
+base_branch = ARGV[0]
+
+require 'fileutils'
+require 'pathname'
+require 'open3'
+
+# run Spotbugs
+puts "running Spotbugs..."
+system './gradlew spotbugsGplayDebug'
+
+# find number of warnings
+current_warning_count = `./scripts/analysis/spotbugsSummary.py --total`.to_i
+puts "found warnings: " + current_warning_count.to_s
+
+# get warning counts from target branch
+previous_xml = "/tmp/#{base_branch}.xml"
+previous_results = File.file?(previous_xml)
+
+if previous_results == true
+    previous_warning_count = `./scripts/analysis/spotbugsSummary.py --total --file #{previous_xml}`.to_i
+    puts "previous warnings: " + previous_warning_count.to_s
+end
+
+# compare previous warning count with current warning count
+if previous_results == true && current_warning_count > previous_warning_count
+    puts "FAIL: warning count increased"
+    exit 1
+end
+
+# check if warning and error count stayed the same
+if  previous_results == true && current_warning_count == previous_warning_count
+    puts "SUCCESS: count stayed the same"
+    exit 0
+end
+
+# warning count DECREASED
+if previous_results == true && current_warning_count < previous_warning_count
+    puts "SUCCESS: count decreased from " + previous_warning_count.to_s + " to " + current_warning_count.to_s
+end

scripts/analysis/spotbugsComparison.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+import argparse
+import defusedxml.ElementTree as ET
+
+import spotbugsSummary
+
+
+def print_comparison(old: dict, new: dict, link_base: str, link_new: str):
+    all_keys = sorted(set(list(old.keys()) + list(new.keys())))
+
+    output = "<table><tr><th>Category</th>"
+    old_header = f"<a href='{link_base}'>Base</a>" if link_base is not None else "Base"
+    output += f"<th>{old_header}</th>"
+    new_header = f"<a href='{link_new}'>New</a>" if link_new is not None else "New"
+    output += f"<th>{new_header}</th>"
+    output += "</tr>"
+
+    for category in all_keys:
+        category_count_old = old[category] if category in old else 0
+        category_count_new = new[category] if category in new else 0
+        new_str = f"<b>{category_count_new}</b>" if category_count_new != category_count_old else str(category_count_new)
+        output += "<tr>"
+        output += f"<td>{category}</td>"
+        output += f"<td>{category_count_old}</td>"
+        output += f"<td>{new_str}</td>"
+        output += "</tr>"
+
+    output += "<tr>"
+    output += "<td><b>Total</b></td>"
+    output += f"<td><b>{sum(old.values())}</b></td>"
+    output += f"<td><b>{sum(new.values())}</b></td>"
+    output += "</tr>"
+
+    output += "</table>"
+
+    print(output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base_file", help="base file for comparison")
+    parser.add_argument("new_file", help="new file for comparison")
+    parser.add_argument("--link-base", help="http link to base html report")
+    parser.add_argument("--link-new", help="http link to new html report")
+    args = parser.parse_args()
+
+    base_tree = ET.parse(args.base_file)
+    base_summary = spotbugsSummary.get_counts(base_tree)
+
+    new_tree = ET.parse(args.new_file)
+    new_summary = spotbugsSummary.get_counts(new_tree)
+
+    print_comparison(base_summary, new_summary, args.link_base, args.link_new)

scripts/analysis/spotbugsSummary.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
+# SPDX-FileCopyrightText: 2017 Tobias Kaminsky <tobias@kaminsky.me>
+# SPDX-License-Identifier: GPL-3.0-or-later
+import argparse
+import defusedxml.ElementTree as ET
+
+
+def get_counts(tree):
+    category_counts = {}
+    category_names = {}
+    for child in tree.getroot():
+        if child.tag == "BugInstance":
+            category = child.attrib['category']
+            if category in category_counts:
+                category_counts[category] = category_counts[category] + 1
+            else:
+                category_counts[category] = 1
+        elif child.tag == "BugCategory":
+            category = child.attrib['category']
+            category_names[category] = child[0].text
+
+    summary = {}
+    for category in category_counts.keys():
+        summary[category_names[category]] = category_counts[category]
+    return summary
+
+
+def print_html(summary):
+    output = "<table><tr><th>Category</th><th>Count</th></tr>"
+
+    categories = sorted(summary.keys())
+    for category in categories:
+        output += "<tr>"
+        output += f"<td>{category}</td>"
+        output += f"<td>{summary[category]}</td>"
+        output += "</tr>"
+
+    output += "<tr>"
+    output += "<td><b>Total</b></td>"
+    output += f"<td><b>{sum(summary.values())}</b></td>"
+    output += "</tr>"
+
+    output += "</table>"
+
+    print(output)
+
+
+def print_total(summary):
+    print(sum(summary.values()))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--total", help="print total count instead of summary HTML",
+                        action="store_true")
+    parser.add_argument("--file", help="file to parse", default="app/build/reports/spotbugs/gplayDebug.xml")
+    args = parser.parse_args()
+    tree = ET.parse(args.file)
+    summary = get_counts(tree)
+    if args.total:
+        print_total(summary)
+    else:
+        print_html(summary)