From e312d1fdbbb3a53ee30ab84203344588154129f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Fri, 27 May 2022 16:36:01 +0100 Subject: [PATCH] gitlab: convert build/container jobs to .base_job_template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This converts the main build and container jobs to use the base job rules, defining the following new variables - QEMU_JOB_SKIPPED - jobs that are known to be currently broken and should not be run. Can still be manually launched if desired. - QEMU_JOB_AVOCADO - jobs that run the Avocado integration test harness. - QEMU_JOB_PUBLISH - jobs that publish content after the branch is merged upstream As build-tools-and-docs runs on master we declare the requirement of building amd64-debian-container optional as it should already exits once we merge. Signed-off-by: Daniel P. Berrangé Message-Id: <20220526110705.59952-5-berrange@redhat.com> [AJB: fix upstream typo, mention optional container req] Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220527153603.887929-32-alex.bennee@linaro.org> --- .gitlab-ci.d/base.yml | 22 ++++++++++++++++++++++ .gitlab-ci.d/buildtest-template.yml | 16 ++++------------ .gitlab-ci.d/buildtest.yml | 28 +++++++++++++--------------- .gitlab-ci.d/container-cross.yml | 6 ++---- .gitlab-ci.d/container-template.yml | 1 + .gitlab-ci.d/crossbuild-template.yml | 3 +++ .gitlab-ci.d/windows.yml | 1 + docs/devel/ci-jobs.rst.inc | 19 +++++++++++++++++++ 8 files changed, 65 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index e6953c77ae..4f091d5aad 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -16,10 +16,22 @@ - if: '$QEMU_JOB_CIRRUS && ($CIRRUS_GITHUB_REPO == "" || $CIRRUS_API_TOKEN == "")' when: never + # Publishing jobs should only run on the default branch in upstream + - if: '$QEMU_JOB_PUBLISH == "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' + when: never + + # Non-publishing jobs should only run on staging branches in upstream + - if: '$QEMU_JOB_PUBLISH != "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH !~ /staging/' + when: never + # Jobs only intended for forks should always be skipped on upstream - if: '$QEMU_JOB_ONLY_FORKS == "1" && $CI_PROJECT_NAMESPACE == "qemu-project"' when: never + # Avocado jobs don't run in forks unless $QEMU_CI_AVOCADO_TESTING is set + - if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != "qemu-project"' + when: never + ############################################################# # Stage 2: fine tune execution of jobs in specific scenarios @@ -31,6 +43,16 @@ when: manual allow_failure: true + # Skipped jobs should not be run unless manually triggered + - if: '$QEMU_JOB_SKIPPED' + when: manual + allow_failure: true + + # Avocado jobs can be manually start in forks if $QEMU_CI_AVOCADO_TESTING is unset + - if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != "qemu-project"' + when: manual + allow_failure: true + ############################################################# # Stage 3: catch all logic applying to any job not matching diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index b381345dbc..73ecfabb8d 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -1,4 +1,5 @@ .native_build_job_template: + extends: .base_job_template stage: build image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest before_script: @@ -27,6 +28,7 @@ fi .common_test_job_template: + extends: .base_job_template stage: test image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest script: @@ -77,15 +79,5 @@ after_script: - cd build - du -chs ${CI_PROJECT_DIR}/avocado-cache - rules: - # Only run these jobs if running on the mainstream namespace, - # or if the user set the QEMU_CI_AVOCADO_TESTING variable (either - # in its namespace setting or via git-push option, see documentation - # in /.gitlab-ci.yml of this repository). - - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' - when: on_success - - if: '$QEMU_CI_AVOCADO_TESTING' - when: on_success - # Otherwise, set to manual (the jobs are created but not run). - - when: manual - allow_failure: true + variables: + QEMU_JOB_AVOCADO: 1 diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index e9620c3074..ecac3ec50c 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -360,12 +360,11 @@ build-cfi-aarch64: expire_in: 2 days paths: - build - rules: + variables: # FIXME: This job is often failing, likely due to out-of-memory problems in # the constrained containers of the shared runners. Thus this is marked as - # manual until the situation has been solved. - - when: manual - allow_failure: true + # skipped until the situation has been solved. + QEMU_JOB_SKIPPED: 1 check-cfi-aarch64: extends: .native_test_job_template @@ -402,12 +401,11 @@ build-cfi-ppc64-s390x: expire_in: 2 days paths: - build - rules: + variables: # FIXME: This job is often failing, likely due to out-of-memory problems in # the constrained containers of the shared runners. Thus this is marked as - # manual until the situation has been solved. - - when: manual - allow_failure: true + # skipped until the situation has been solved. + QEMU_JOB_SKIPPED: 1 check-cfi-ppc64-s390x: extends: .native_test_job_template @@ -579,6 +577,7 @@ build-without-default-features: MAKE_CHECK_ARGS: check-unit check-qtest SPEED=slow build-libvhost-user: + extends: .base_job_template stage: build image: $CI_REGISTRY_IMAGE/qemu/fedora:latest needs: @@ -595,10 +594,13 @@ build-tools-and-docs-debian: extends: .native_build_job_template needs: job: amd64-debian-container + # when running on 'master' we use pre-existing container + optional: true variables: IMAGE: debian-amd64 MAKE_CHECK_ARGS: check-unit check-softfloat ctags TAGS cscope CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools + QEMU_JOB_PUBLISH: 1 artifacts: expire_in: 2 days paths: @@ -618,6 +620,7 @@ build-tools-and-docs-debian: # that users can see the results of their commits, regardless # of what topic branch they're currently using pages: + extends: .base_job_template image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest stage: test needs: @@ -635,10 +638,5 @@ pages: artifacts: paths: - public - rules: - - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' - when: never - - if: '$CI_PROJECT_NAMESPACE != "qemu-project"' - when: on_success + variables: + QEMU_JOB_PUBLISH: 1 diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 147e667744..b7963498a3 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -40,15 +40,13 @@ armhf-debian-cross-container: # We never want to build hexagon in the CI system and by default we # always want to refer to the master registry where it lives. hexagon-cross-container: + extends: .base_job_template image: docker:stable stage: containers - rules: - - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' - when: never - - when: always variables: NAME: debian-hexagon-cross GIT_DEPTH: 1 + QEMU_JOB_ONLY_FORKS: 1 services: - docker:dind before_script: diff --git a/.gitlab-ci.d/container-template.yml b/.gitlab-ci.d/container-template.yml index 1baecd9460..c434b9c8f3 100644 --- a/.gitlab-ci.d/container-template.yml +++ b/.gitlab-ci.d/container-template.yml @@ -1,4 +1,5 @@ .container_job_template: + extends: .base_job_template image: docker:stable stage: containers services: diff --git a/.gitlab-ci.d/crossbuild-template.yml b/.gitlab-ci.d/crossbuild-template.yml index 29c3c2b826..28b2142ec2 100644 --- a/.gitlab-ci.d/crossbuild-template.yml +++ b/.gitlab-ci.d/crossbuild-template.yml @@ -1,4 +1,5 @@ .cross_system_build_job: + extends: .base_job_template stage: build image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest timeout: 80m @@ -24,6 +25,7 @@ # KVM), and set extra options (such disabling other accelerators) via the # $EXTRA_CONFIGURE_OPTS variable. .cross_accel_build_job: + extends: .base_job_template stage: build image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest timeout: 30m @@ -36,6 +38,7 @@ - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS .cross_user_build_job: + extends: .base_job_template stage: build image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest script: diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index cf7724b8e5..1b2ede49e1 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -1,4 +1,5 @@ .shared_msys2_builder: + extends: .base_job_template tags: - shared-windows - windows diff --git a/docs/devel/ci-jobs.rst.inc b/docs/devel/ci-jobs.rst.inc index 4c7e30ab08..0b4926e537 100644 --- a/docs/devel/ci-jobs.rst.inc +++ b/docs/devel/ci-jobs.rst.inc @@ -73,6 +73,25 @@ The job results are only of interest to contributors prior to submitting code. They are not required as part of the gating CI pipeline. +QEMU_JOB_SKIPPED +~~~~~~~~~~~~~~~~ + +The job is not reliably successsful in general, so is not +currently suitable to be run by default. Ideally this should +be a temporary marker until the problems can be addressed, or +the job permanently removed. + +QEMU_JOB_PUBLISH +~~~~~~~~~~~~~~~~ + +The job is for publishing content after a branch has been +merged into the upstream default branch. + +QEMU_JOB_AVOCADO +~~~~~~~~~~~~~~~~ + +The job runs the Avocado integration test suite + Contributor controlled runtime variables ----------------------------------------