diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index c84889ce2..265d414fd 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -52,6 +52,44 @@ while getopts ":t:p:sh" o; do esac done + +upload_package_test_logs() { + local retry_count=0 + local package_folder="" + + retry_count=${BUILDKITE_RETRY_COUNT:-"0"} + package_folder="${PACKAGE}" + + if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-""}" == "false" ]]; then + package_folder="${package_folder}-stack_agent" + fi + + if [[ "${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD:-""}" != "" ]]; then + package_folder="${package_folder}-${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD}" + fi + + if [[ "${retry_count}" -ne 0 ]]; then + package_folder="${package_folder}_retry_${retry_count}" + fi + + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ + "insecure-logs/${package_folder}/elastic-agent-logs/" + + # required for <8.6.0 + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \ + "insecure-logs/${package_folder}/elastic-agent-logs/default/" + + upload_safe_logs \ + "${JOB_GCS_BUCKET_INTERNAL}" \ + "build/container-logs/*.log" \ + "insecure-logs/${package_folder}/container-logs/" + +} + if [[ "${TARGET}" == "" ]]; then echo "Missing target" usage @@ -63,15 +101,15 @@ add_bin_path if [[ "$SERVERLESS" == "false" ]]; then # If packages are tested with Serverless, these action are already performed # here: .buildkite/scripts/test_packages_with_serverless.sh - echo "--- install go" + echo "--- Install go" with_go if [[ "${TARGET}" != "${TEST_BUILD_ZIP_TARGET}" ]]; then # Not supported in Macos ARM - echo "--- install docker" + echo "--- Install docker" with_docker - echo "--- install docker-compose plugin" + echo "--- Install docker-compose plugin" with_docker_compose_plugin fi fi @@ -80,7 +118,7 @@ echo "--- install yq" with_yq if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then - echo "--- install kubectl & kind" + echo "--- Install kubectl & kind" with_kubernetes fi @@ -88,56 +126,31 @@ label="${TARGET}" if [ -n "${PACKAGE}" ]; then label="${label} - ${PACKAGE}" fi + +echo "--- Install elastic-package" +make install + echo "--- Run integration test ${label}" if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then - make install - # allow to fail this command, to be able to upload safe logs set +e make SERVERLESS="${SERVERLESS}" PACKAGE_UNDER_TEST="${PACKAGE}" "${TARGET}" testReturnCode=$? set -e - retry_count=${BUILDKITE_RETRY_COUNT:-"0"} if [[ "${UPLOAD_SAFE_LOGS}" -eq 1 ]] ; then - package_folder="${PACKAGE}" - if [[ "${ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT:-""}" == "false" ]]; then - package_folder="${package_folder}-stack_agent" - fi - - if [[ "${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD:-""}" != "" ]]; then - package_folder="${package_folder}-${ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD}" - fi - - if [[ "${retry_count}" -ne 0 ]]; then - package_folder="${package_folder}_retry_${retry_count}" - fi - - upload_safe_logs \ - "${JOB_GCS_BUCKET_INTERNAL}" \ - "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/*.*" \ - "insecure-logs/${package_folder}/elastic-agent-logs/" - - # required for <8.6.0 - upload_safe_logs \ - "${JOB_GCS_BUCKET_INTERNAL}" \ - "build/elastic-stack-dump/check-${PACKAGE}/logs/elastic-agent-internal/default/*" \ - "insecure-logs/${package_folder}/elastic-agent-logs/default/" - - upload_safe_logs \ - "${JOB_GCS_BUCKET_INTERNAL}" \ - "build/container-logs/*.log" \ - "insecure-logs/${package_folder}/container-logs/" + upload_package_test_logs fi if [ $testReturnCode != 0 ]; then echo "make SERVERLESS=${SERVERLESS} PACKAGE_UNDER_TEST=${PACKAGE} ${TARGET} failed with ${testReturnCode}" exit ${testReturnCode} fi - - make check-git-clean - exit 0 +else + make "${TARGET}" fi -make install "${TARGET}" check-git-clean +echo "--- Check git clean" +make check-git-clean +exit 0 diff --git a/.buildkite/scripts/release.sh b/.buildkite/scripts/release.sh index 3aa20825c..3491d7cd6 100755 --- a/.buildkite/scripts/release.sh +++ b/.buildkite/scripts/release.sh @@ -17,11 +17,11 @@ VERSION="" add_bin_path with_go -echo "--- fetching tags" +echo "--- Fetching tags" # Ensure that tags are present so goreleaser can build the changelog from the last release. git rev-parse --is-shallow-repository git fetch origin --tags -echo "--- running goreleaser" +echo "--- Running goreleaser" # Run latest version of goreleaser curl -sL https://git.io/goreleaser | bash diff --git a/.buildkite/scripts/test-with-integrations.sh b/.buildkite/scripts/test-with-integrations.sh index 2410e3efe..51b687f3a 100755 --- a/.buildkite/scripts/test-with-integrations.sh +++ b/.buildkite/scripts/test-with-integrations.sh @@ -5,10 +5,10 @@ set -euo pipefail add_bin_path -echo "--- install gh cli" +echo "--- Install gh cli" with_github_cli -echo "--- install jq" +echo "--- Install jq" with_jq @@ -73,7 +73,7 @@ create_integrations_pull_request() { update_dependency() { # it needs to set the Golang version from the integrations repository (.go-version file) - echo "--- install go for integrations repository :go:" + echo "--- Install go for integrations repository :go:" with_go echo "--- Updating go.mod and go.sum with ${GITHUB_PR_HEAD_SHA} :hammer_and_wrench:" @@ -154,7 +154,7 @@ create_or_update_pull_request() { rm -rf "${temp_path}" - echo "--- adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:" + echo "--- Adding comment into ${GITHUB_PR_BASE_REPO} pull request :memo:" add_pr_comment "${BUILDKITE_PULL_REQUEST}" "$(get_integrations_pr_link "${integrations_pr_number}")" } @@ -170,5 +170,5 @@ add_pr_comment() { } -echo "--- creating or updating integrations pull request" +echo "--- Creating or updating integrations pull request" create_or_update_pull_request diff --git a/.buildkite/scripts/test_packages_with_serverless.sh b/.buildkite/scripts/test_packages_with_serverless.sh index cafe58e68..fe1f6c935 100755 --- a/.buildkite/scripts/test_packages_with_serverless.sh +++ b/.buildkite/scripts/test_packages_with_serverless.sh @@ -32,7 +32,7 @@ fi add_bin_path -echo "--- install go" +echo "--- Install go" with_go echo "--- Install docker" @@ -42,7 +42,7 @@ echo "--- Install docker-compose" with_docker_compose_plugin if [[ "${BUILDKITE_PULL_REQUEST}" != "false" ]]; then - echo "--- install gh cli" + echo "--- Install gh cli" with_github_cli add_pr_comment "${BUILDKITE_PULL_REQUEST}" "${BUILDKITE_BUILD_URL}" diff --git a/.buildkite/scripts/unit_tests_macos_arm.sh b/.buildkite/scripts/unit_tests_macos_arm.sh index d8814d207..5d8ae4a5f 100755 --- a/.buildkite/scripts/unit_tests_macos_arm.sh +++ b/.buildkite/scripts/unit_tests_macos_arm.sh @@ -6,7 +6,7 @@ set -euo pipefail add_bin_path -echo "--- install go" +echo "--- Install go" with_go echo "--- Running unit tests" diff --git a/scripts/test-build-install-zip-file.sh b/scripts/test-build-install-zip-file.sh index a7597e396..8ed265178 100755 --- a/scripts/test-build-install-zip-file.sh +++ b/scripts/test-build-install-zip-file.sh @@ -1,9 +1,14 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail cleanup() { local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" local output_path="build/elastic-stack-dump/install-zip" if [ ${USE_SHELLINIT} -eq 1 ]; then @@ -11,7 +16,7 @@ cleanup() { fi # Dump stack logs - elastic-package stack dump -v --output ${output_path} + elastic-package stack dump -v --output ${output_path} || true # Take down the stack elastic-package stack down -v diff --git a/scripts/test-build-install-zip.sh b/scripts/test-build-install-zip.sh index 1c46076b0..bdafa39a5 100755 --- a/scripts/test-build-install-zip.sh +++ b/scripts/test-build-install-zip.sh @@ -1,12 +1,17 @@ #!/bin/bash -set -euxo pipefail +set -euo pipefail cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump/build-zip + elastic-package stack dump -v --output build/elastic-stack-dump/build-zip || true # Take down the stack elastic-package stack down -v @@ -37,15 +42,17 @@ go run ./scripts/gpgkey for d in test/packages/*/*/; do # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "$(testype "$d")" == "false_positives" ]; then continue fi + echo "--- Building package: ${d}" elastic-package build -C "$d" --zip --sign -v done # Remove unzipped built packages, leave .zip files rm -r build/packages/*/ +echo "--- Prepare Elastic stack" # Boot up the stack elastic-package stack up -d -v @@ -54,12 +61,13 @@ eval "$(elastic-package stack shellinit)" # Install packages from working copy for d in test/packages/*/*/; do # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "$(testype "$d")" == "false_positives" ]; then continue fi package_name=$(yq -r '.name' "${d}/manifest.yml") package_version=$(yq -r '.version' "${d}/manifest.yml") + echo "--- Installing package: ${package_name} (${package_version})" elastic-package install -C "$d" -v # check that the package is installed diff --git a/scripts/test-build-zip.sh b/scripts/test-build-zip.sh index bdd382cbb..66982053e 100755 --- a/scripts/test-build-zip.sh +++ b/scripts/test-build-zip.sh @@ -1,9 +1,13 @@ #!/bin/bash - -set -euxo pipefail +set -euo pipefail cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Clean used resources for d in test/packages/*/*/; do @@ -31,9 +35,10 @@ go run ./scripts/gpgkey for d in test/packages/*/*/; do # Packages in false_positives can have issues. - if [ "$(testype $d)" == "false_positives" ]; then + if [ "$(testype "$d")" == "false_positives" ]; then continue fi + echo "--- Building zip package: ${d}" elastic-package build -C "$d" --zip --sign -v done diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh index f8f0bf070..941ae666c 100755 --- a/scripts/test-check-false-positives.sh +++ b/scripts/test-check-false-positives.sh @@ -8,9 +8,14 @@ set -euxo pipefail function cleanup() { r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Dump stack logs - elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" + elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" || true # Take down the stack elastic-package stack down -v @@ -94,6 +99,7 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH stack_args=$(stack_version_args) # --version +echo "--- Prepare Elastic stack" # Update the stack elastic-package stack update -v ${stack_args} @@ -108,6 +114,8 @@ elastic-package stack status # Run package tests for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + echo "--- Check build output: ${d}" check_build_output "$d" + echo "--- Check expected errors: ${d}" check_expected_errors "$d" done diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index b0fe92968..a0bbf5b58 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -8,10 +8,15 @@ set -euxo pipefail cleanup() { r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Dump stack logs elastic-package stack dump -v \ - --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" + --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-any}}" || true if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Dump kubectl details @@ -54,10 +59,13 @@ export SERVERLESS=${SERVERLESS:-"false"} # Build/check packages for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do + echo "--- Checking package ${d}" elastic-package check -C "$d" -v done if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then + echo "--- Create logstash profile" + # Create a logstash profile and use it elastic-package profiles create logstash -v elastic-package profiles use logstash @@ -73,6 +81,7 @@ fi # started to test all packages. In our CI, this Elastic serverless stack is started # at the beginning of the pipeline and must be running for all packages. if [[ "${SERVERLESS}" != "true" ]]; then + echo "--- Prepare Elastic stack" stack_args=$(stack_version_args) # --version # Update the stack @@ -90,6 +99,7 @@ fi if [ "${PACKAGE_TEST_TYPE:-other}" == "with-kind" ]; then # Boot up the kind cluster + echo "--- Create kind cluster" kind create cluster --config "$PWD/scripts/kind-config.yaml" --image "kindest/node:${K8S_VERSION}" fi @@ -100,6 +110,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop if [ "${package_to_test}" == "pipeline_benchmark" ]; then + echo "--- Run pipeline benchmarks and report for package ${package_to_test}" + rm -rf "${PWD}/build/benchmark-results" elastic-package benchmark pipeline -C "$d" -v --report-format xUnit --report-output file --fail-on-missing @@ -114,12 +126,16 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do --threshold 1 --report-output-path="${PWD}/build/benchreport" fi if [ "${package_to_test}" == "system_benchmark" ]; then + echo "--- Run system benchmarks for package ${package_to_test}" + elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s fi elif [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ] && [ "${package_to_test}" == "system_benchmark" ]; then + echo "--- Run system benchmarks for package ${package_to_test}" elastic-package benchmark system -C "$d" --benchmark logs-benchmark -v --defer-cleanup 1s else if [[ "${SERVERLESS}" == "true" ]]; then + echo "--- Run tests for package ${d} in Serverless mode" # skip system tests elastic-package test asset -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic elastic-package test static -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic @@ -129,6 +145,8 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do continue fi + + echo "--- Run tests for package ${d}" # Run all tests # defer-cleanup is set to a short period to verify that the option is available elastic-package test -C "$d" -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage --coverage-format=generic diff --git a/scripts/test-profiles-command.sh b/scripts/test-profiles-command.sh index 23eb2e90e..7b3988e53 100755 --- a/scripts/test-profiles-command.sh +++ b/scripts/test-profiles-command.sh @@ -3,7 +3,7 @@ set -euxo pipefail cleanup() { - r=$? + local r=$? # Delete extra profiles elastic-package profiles delete test_default diff --git a/scripts/test-stack-command.sh b/scripts/test-stack-command.sh index 03f5cbeb8..315913124 100755 --- a/scripts/test-stack-command.sh +++ b/scripts/test-stack-command.sh @@ -8,10 +8,15 @@ SELF_MONITOR_ENABLED=${SELF_MONITOR_ENABLED:-false} ELASTIC_SUBSCRIPTION=${ELASTIC_SUBSCRIPTION:-""} cleanup() { - r=$? + local r=$? + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + echo "~~~ elastic-package cleanup" # Dump stack logs - elastic-package stack dump -v --output "build/elastic-stack-dump/stack/${VERSION}" + elastic-package stack dump -v --output "build/elastic-stack-dump/stack/${VERSION}" || true # Take down the stack elastic-package stack down -v diff --git a/scripts/test-system-test-flags.sh b/scripts/test-system-test-flags.sh index b8665b7f6..bcd04bffe 100755 --- a/scripts/test-system-test-flags.sh +++ b/scripts/test-system-test-flags.sh @@ -5,10 +5,16 @@ set -euxo pipefail cleanup() { local r=$? local container_id="" - local agent_ids + local agent_ids="" + if [ "${r}" -ne 0 ]; then + # Ensure that the group where the failure happened is opened. + echo "^^^ +++" + fi + + echo "~~~ elastic-package cleanup" # Dump stack logs - elastic-package stack dump -v --output build/elastic-stack-dump/system-test-flags + elastic-package stack dump -v --output build/elastic-stack-dump/system-test-flags ||true if is_service_container_running "${DEFAULT_AGENT_CONTAINER_NAME}" ; then container_id=$(container_ids "${DEFAULT_AGENT_CONTAINER_NAME}")