Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/actions/setup-java-with-cache/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ runs:
id: cache-key
shell: bash
run: |
echo "value=maven-repo-${{ hashFiles('**/pom.xml', '.mvn/extensions.xml') }}" >> "$GITHUB_OUTPUT"
echo "value=maven-repo-${{ hashFiles('**/pom.xml', '.mvn/extensions.xml', '.mvn/wrapper/maven-wrapper.properties') }}" >> "$GITHUB_OUTPUT"

- name: Remove any leftover Maven repo
id: cleanup
Expand All @@ -73,6 +73,7 @@ runs:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
~/.m2/wrapper
key: ${{ steps.cache-key.outputs.value }}

- name: Exact hit?
Expand All @@ -99,6 +100,7 @@ runs:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
~/.m2/wrapper
key: maven-repo-

- name: Any hit?
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ env:
HADOOP_IMAGE: ghcr.io/apache/hadoop
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
MVNW_VERBOSE: true
OZONE_IMAGE: ghcr.io/apache/ozone
OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
OZONE_VOLUME_OWNER: 1000
Expand Down Expand Up @@ -230,9 +231,10 @@ jobs:
# TODO remove hard-code in HDDS-17509
if: ${{ inputs.script == 'acceptance' }}
run: |
HADOOP_VERSION=$(mvn help:evaluate -N -Dexpression=hadoop.version -q -DforceStdout -Dscan=false)
source hadoop-ozone/dev-support/checks/_lib.sh
HADOOP_VERSION=$(get_version hadoop)
echo "HADOOP_VERSION=$HADOOP_VERSION" >> $GITHUB_ENV
RANGER_VERSION=$(mvn help:evaluate -N -Dexpression=ranger.version -q -DforceStdout -Dscan=false)
RANGER_VERSION=$(get_version ranger)
echo "RANGER_VERSION=$RANGER_VERSION" >> $GITHUB_ENV

- name: Restore Hadoop
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ env:
# MAVEN_ARGS and MAVEN_OPTS are duplicated in several workflows, please keep in sync
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
MVNW_VERBOSE: true
OZONE_WITH_COVERAGE: ${{ github.event_name != 'pull_request' }}

permissions: { }
Expand Down
22 changes: 13 additions & 9 deletions .github/workflows/populate-cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ on:
- '.github/actions/setup-java-with-cache/action.yml'
- '.github/workflows/populate-cache.yml'
- '.mvn/extensions.xml'
- '.mvn/wrapper/maven-wrapper.properties'
workflow_call:
workflow_dispatch:

Expand All @@ -44,6 +45,7 @@ env:
TEST_JAVA_VERSION: 25 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml -Ddevelocity.user.config=${{ github.workspace }}/.mvn/develocity-asf.xml
MAVEN_OPTS: -Daether.connector.http.connectionMaxTtl=25 -Daether.connector.http.retryHandler.count=5 -Daether.connector.http.retryHandler.intervalMax=30000
MVNW_VERBOSE: true
SKIP_APACHE_VERIFY_DOWNLOAD: false

jobs:
Expand All @@ -62,14 +64,19 @@ jobs:
java-version: ${{ env.TEST_JAVA_VERSION }}
only-exact-hit: true

- name: Fetch plugin dependencies
if: steps.restore-cache.outputs.cache-hit != 'true'
run: ./mvnw $MAVEN_ARGS --no-transfer-progress --show-version dependency:resolve-plugins

- name: Get versions
id: versions
run: |
NODEJS_VERSION=$(mvn help:evaluate -N -Dexpression=nodejs.version -q -DforceStdout -Dscan=false)
source hadoop-ozone/dev-support/checks/_lib.sh
NODEJS_VERSION=$(get_version nodejs)
echo "NODEJS_VERSION=$NODEJS_VERSION" >> $GITHUB_ENV
HADOOP_VERSION=$(mvn help:evaluate -N -Dexpression=hadoop.version -q -DforceStdout -Dscan=false)
HADOOP_VERSION=$(get_version hadoop)
echo "HADOOP_VERSION=$HADOOP_VERSION" >> $GITHUB_ENV
RANGER_VERSION=$(mvn help:evaluate -N -Dexpression=ranger.version -q -DforceStdout -Dscan=false)
RANGER_VERSION=$(get_version ranger)
echo "RANGER_VERSION=$RANGER_VERSION" >> $GITHUB_ENV

- name: Restore Hadoop
Expand Down Expand Up @@ -133,21 +140,17 @@ jobs:
if: steps.restore-cache.outputs.cache-hit != 'true' && steps.restore-nodejs.outputs.cache-hit != 'true'
run: dev-support/ci/download-nodejs.sh

- name: Fetch plugin dependencies
if: steps.restore-cache.outputs.cache-hit != 'true'
run: mvn $MAVEN_ARGS --no-transfer-progress --show-version dependency:resolve-plugins

- name: Fetch dependencies
if: steps.restore-cache.outputs.cache-hit != 'true'
run: mvn $BUILD_ARGS $MAVEN_ARGS --no-transfer-progress --show-version -Pgo-offline clean verify
run: ./mvnw $BUILD_ARGS $MAVEN_ARGS --no-transfer-progress --show-version -Pgo-offline clean verify

- name: Delete Ozone jars from repo
if: steps.restore-cache.outputs.cache-hit != 'true'
run: rm -fr ~/.m2/repository/org/apache/ozone

- name: List repo contents
if: always()
run: find ~/.m2/repository -type f | sort | xargs ls -lh
run: find ~/.m2 -type f | sort | xargs ls -lh

- name: Save cache for Maven dependencies
if: steps.restore-cache.outputs.cache-hit != 'true'
Expand All @@ -156,4 +159,5 @@ jobs:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
~/.m2/wrapper
key: ${{ steps.restore-cache.outputs.cache-key }}
3 changes: 3 additions & 0 deletions .mvn/wrapper/maven-wrapper.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
wrapperVersion=3.3.4
distributionType=only-script
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.10.1/apache-maven-3.10.1-bin.zip
3 changes: 2 additions & 1 deletion dev-support/ci/download-nodejs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ mkdir -p "${REPO_DIR}"
declare -i rc=1

if [[ -z "${NODEJS_VERSION:-}" ]]; then
NODEJS_VERSION=$(mvn help:evaluate -Dexpression=nodejs.version -q -DforceStdout -Dscan=false)
source hadoop-ozone/dev-support/checks/_lib.sh
NODEJS_VERSION=$(get_version nodejs)
fi

if [[ -n "${NODEJS_VERSION}" ]]; then
Expand Down
104 changes: 42 additions & 62 deletions dev-support/ci/errorprone.bats
Original file line number Diff line number Diff line change
Expand Up @@ -15,88 +15,68 @@
# limitations under the License.

setup() {
TEST_TMPDIR=$(mktemp -d)
export TEST_TMPDIR
export OUTPUT_DIR="${TEST_TMPDIR}/report"
mkdir -p "${TEST_TMPDIR}/bin" "${OUTPUT_DIR}"
echo 9 > "${OUTPUT_DIR}/failures"
export MAVEN_EXIT_CODE=1
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
#!/usr/bin/env bash
rm -rf "${OUTPUT_DIR}"
printf '%s\n' "$*" > "${TEST_TMPDIR}/mvn-args"
printf '%s\n' \
"[WARNING] The following options were not recognized by any processor: '[artifactId]'" \
"[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding" \
"[WARNING] /src/Custom.java:[3,1] [Custom-Rule_1] custom finding" \
"[ERROR] /src/Format.java:[2,1] [FormatString] error finding" \
"[ERROR] /src/Format.java:[2,1] [FormatString] error finding" \
"/src/Raw.java:4: warning: [RawWarning] raw warning" \
"[ERROR] Failed to execute goal example:example: failed"
printf '%s\n' "/src/Raw.java:5: error: [RawError] raw error" >&2
exit "${MAVEN_EXIT_CODE}"
EOF
chmod +x "${TEST_TMPDIR}/bin/mvn"
export PATH="${TEST_TMPDIR}/bin:${PATH}"
REPORT_DIR=$(mktemp -d)
export REPORT_DIR
}

teardown() {
rm -rf "${TEST_TMPDIR}"
rm -rf "${REPORT_DIR}"
}

@test "Error Prone reports Maven and javac findings and fails for errors" {
run hadoop-ozone/dev-support/checks/errorprone.sh
export rc=1
cat > "${REPORT_DIR}/output.log" <<'EOF'
[WARNING] The following options were not recognized by any processor: '[artifactId]'
[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding
[WARNING] /src/Custom.java:[3,1] [Custom-Rule_1] custom finding
[ERROR] /src/Format.java:[2,1] [FormatString] error finding
[ERROR] /src/Format.java:[2,1] [FormatString] error finding
/src/Raw.java:4: warning: [RawWarning] raw warning
/src/Raw.java:5: error: [RawError] raw error
[ERROR] Failed to execute goal example:example: failed
EOF

run hadoop-ozone/dev-support/checks/_errorprone_report.sh

[ "$status" -eq 1 ]
grep -q -- '-DskipTests' "${TEST_TMPDIR}/mvn-args"
grep -q -- '-DskipShade' "${TEST_TMPDIR}/mvn-args"
grep -q -- 'clean package' "${TEST_TMPDIR}/mvn-args"
[ "$(wc -l < "${OUTPUT_DIR}/diagnostics.txt")" -eq 5 ]
grep -q '^\[WARNING\].*\[JdkObsolete\]' "${OUTPUT_DIR}/diagnostics.txt"
grep -q '^\[ERROR\].*\[FormatString\]' "${OUTPUT_DIR}/diagnostics.txt"
grep -q '^\[WARNING\].*\[Custom-Rule_1\]' "${OUTPUT_DIR}/diagnostics.txt"
grep -q '^/src/Raw.java:4: warning: \[RawWarning\]' "${OUTPUT_DIR}/diagnostics.txt"
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${OUTPUT_DIR}/diagnostics.txt"
! grep -q 'Failed to execute goal' "${OUTPUT_DIR}/diagnostics.txt"
[ "$(wc -l < "${OUTPUT_DIR}/summary.txt")" -eq 2 ]
grep -q '^\[ERROR\].*\[FormatString\]' "${OUTPUT_DIR}/summary.txt"
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${OUTPUT_DIR}/summary.txt"
[ "$(< "${OUTPUT_DIR}/failures")" -eq 2 ]
[ "$(wc -l < "${REPORT_DIR}/diagnostics.txt")" -eq 5 ]
grep -q '^\[WARNING\].*\[JdkObsolete\]' "${REPORT_DIR}/diagnostics.txt"
grep -q '^\[ERROR\].*\[FormatString\]' "${REPORT_DIR}/diagnostics.txt"
grep -q '^\[WARNING\].*\[Custom-Rule_1\]' "${REPORT_DIR}/diagnostics.txt"
grep -q '^/src/Raw.java:4: warning: \[RawWarning\]' "${REPORT_DIR}/diagnostics.txt"
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${REPORT_DIR}/diagnostics.txt"
! grep -q 'Failed to execute goal' "${REPORT_DIR}/diagnostics.txt"
[ "$(wc -l < "${REPORT_DIR}/summary.txt")" -eq 2 ]
grep -q '^\[ERROR\].*\[FormatString\]' "${REPORT_DIR}/summary.txt"
grep -q '^/src/Raw.java:5: error: \[RawError\]' "${REPORT_DIR}/summary.txt"
[ "$(< "${REPORT_DIR}/failures")" -eq 2 ]
}

@test "Error Prone warnings do not fail the check" {
export MAVEN_EXIT_CODE=0
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
#!/usr/bin/env bash
rm -rf "${OUTPUT_DIR}"
printf '%s\n' \
"[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding" \
"/src/Raw.java:4: warning: [RawWarning] raw warning"
exit "${MAVEN_EXIT_CODE}"
export rc=0
cat > "${REPORT_DIR}/output.log" <<'EOF'
[WARNING] /src/Legacy.java:[1,1] [JdkObsolete] legacy finding
/src/Raw.java:4: warning: [RawWarning] raw warning
EOF
chmod +x "${TEST_TMPDIR}/bin/mvn"

run hadoop-ozone/dev-support/checks/errorprone.sh
run hadoop-ozone/dev-support/checks/_errorprone_report.sh

[ "$status" -eq 0 ]
[ "$(wc -l < "${OUTPUT_DIR}/diagnostics.txt")" -eq 2 ]
[ "$(< "${OUTPUT_DIR}/failures")" -eq 0 ]
[ ! -s "${OUTPUT_DIR}/summary.txt" ]
[ "$(wc -l < "${REPORT_DIR}/diagnostics.txt")" -eq 2 ]
[ "$(< "${REPORT_DIR}/failures")" -eq 0 ]
[ ! -s "${REPORT_DIR}/summary.txt" ]
}

@test "Error Prone reports an unknown Maven failure" {
cat > "${TEST_TMPDIR}/bin/mvn" <<'EOF'
#!/usr/bin/env bash
rm -rf "${OUTPUT_DIR}"
echo "[ERROR] Failed to execute goal example:example: failed"
exit "${MAVEN_EXIT_CODE}"
export rc=1
cat > "${REPORT_DIR}/output.log" <<'EOF'
[ERROR] Failed to execute goal example:example: failed
EOF
chmod +x "${TEST_TMPDIR}/bin/mvn"

run hadoop-ozone/dev-support/checks/errorprone.sh
run hadoop-ozone/dev-support/checks/_errorprone_report.sh

[ "$status" -eq 1 ]
[ ! -s "${OUTPUT_DIR}/diagnostics.txt" ]
[ "$(< "${OUTPUT_DIR}/summary.txt")" = "Unknown failure, check output.log" ]
[ "$(< "${OUTPUT_DIR}/failures")" -eq 1 ]
[ ! -s "${REPORT_DIR}/diagnostics.txt" ]
[ "$(< "${REPORT_DIR}/summary.txt")" = "Unknown failure, check output.log" ]
[ "$(< "${REPORT_DIR}/failures")" -eq 1 ]
}
2 changes: 1 addition & 1 deletion hadoop-ozone/dev-support/checks/_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ else
fi

export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
mvn ${MAVEN_OPTIONS} clean "$@" | tee output.log
./mvnw ${MAVEN_OPTIONS} clean "$@" | tee output.log
rc=$?

mkdir -p "$REPORT_DIR" # after `mvn clean`
Expand Down
36 changes: 36 additions & 0 deletions hadoop-ozone/dev-support/checks/_errorprone_report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -u -o pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

DIAGNOSTIC_FILE="$REPORT_DIR/diagnostics.txt"
REPORT_FILE="$REPORT_DIR/summary.txt"

MAVEN_DIAGNOSTIC_PATTERN='^\[(ERROR|WARNING)\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
JAVAC_DIAGNOSTIC_PATTERN='^.*:[0-9]+: (error|warning): \[[^]]+\]'
MAVEN_ERROR_PATTERN='^\[ERROR\] .*:\[[0-9]+,[0-9]+\] \[[^]]+\]'
JAVAC_ERROR_PATTERN='^.*:[0-9]+: error: \[[^]]+\]'
ERROR_DIAGNOSTIC_PATTERN="${MAVEN_ERROR_PATTERN}|${JAVAC_ERROR_PATTERN}"

grep -E "${MAVEN_DIAGNOSTIC_PATTERN}|${JAVAC_DIAGNOSTIC_PATTERN}" "${REPORT_DIR}/output.log" \
| awk '!seen[$0]++' > "$DIAGNOSTIC_FILE"

grep -E "$ERROR_DIAGNOSTIC_PATTERN" "$DIAGNOSTIC_FILE" > "$REPORT_FILE" || true

source "${DIR}/_post_process.sh"
6 changes: 6 additions & 0 deletions hadoop-ozone/dev-support/checks/_lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,9 @@ create_aws_dir() {
popd
fi
}

get_version() {
local dep="$1"
shift
MVNW_VERBOSE=false ./mvnw --non-recursive --quiet help:evaluate -Dexpression="${dep}.version" -DforceStdout -Dscan=false "$@"
}
8 changes: 4 additions & 4 deletions hadoop-ozone/dev-support/checks/acceptance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,21 @@ OZONE_ROOT=$(pwd -P)
: ${OZONE_ACCEPTANCE_TEST_TYPE:="robot"}
: ${OZONE_WITH_COVERAGE:="false"}

source "${DIR}/_lib.sh"

if [[ "${OZONE_TEST_SELECTOR}" == "xcompat/test-old.sh" ]]; then
: ${OZONE_OLD_VERSIONS:="${2:-}"}
export OZONE_OLD_VERSIONS

# use JDK8 image for Rebot in compat-old, to avoid pulling JDK25 image just for generating reports
# TODO after HDDS-17509 this can be moved to env: of compat-old job
export OZONE_RUNNER_VERSION=$(mvn help:evaluate -pl :ozone-dist -Dexpression="docker.ozone-runner.client.version" -q -DforceStdout -Dscan=false)
export OZONE_RUNNER_VERSION=$(get_version "docker.ozone-runner.client" -f hadoop-ozone/dist)
fi

source "${DIR}/_lib.sh"

REPORT_DIR=${OUTPUT_DIR:-"${OZONE_ROOT}/target/acceptance"}
REPORT_FILE="$REPORT_DIR/summary.txt"

OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout -Dscan=false)
OZONE_VERSION=$(get_version ozone)
DIST_DIR="${OZONE_ROOT}/hadoop-ozone/dist/target/ozone-$OZONE_VERSION"

# workaround attempt for https://github.com/docker/compose/issues/12747
Expand Down
4 changes: 2 additions & 2 deletions hadoop-ozone/dev-support/checks/checkstyle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ MAVEN_OPTIONS='-B -fae -DskipRecon -Dcheckstyle.failOnViolation=false --no-trans
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dscan.value.check=${CHECK}"

declare -i rc
mvn ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log"
./mvnw ${MAVEN_OPTIONS} checkstyle:check > "${REPORT_DIR}/output.log"
rc=$?
if [[ ${rc} -ne 0 ]]; then
mvn ${MAVEN_OPTIONS} clean test-compile checkstyle:check > output.log
./mvnw ${MAVEN_OPTIONS} clean test-compile checkstyle:check > output.log
rc=$?
mkdir -p "$REPORT_DIR" # removed by mvn clean
mv output.log "${REPORT_DIR}"/
Expand Down
5 changes: 3 additions & 2 deletions hadoop-ozone/dev-support/checks/coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ REPORT_DIR="$DIR/../../../target/coverage"

mkdir -p "$REPORT_DIR"

JACOCO_VERSION=$(mvn help:evaluate -Dexpression=jacoco.version -q -DforceStdout -Dscan=false)
source "${DIR}/_lib.sh"
JACOCO_VERSION=$(get_version jacoco)

#Install jacoco cli
mvn --non-recursive --no-transfer-progress -Dscan=false \
./mvnw --non-recursive --no-transfer-progress -Dscan=false \
org.apache.maven.plugins:maven-dependency-plugin:copy \
-Dartifact=org.jacoco:org.jacoco.cli:${JACOCO_VERSION}:jar:nodeps

Expand Down
Loading