salsa-ci.yml 27.2 KB
Newer Older
Raul Benencia's avatar
Raul Benencia committed
1
---
2
3
4
5
6
7
# Copyright salsa-ci-team and others
# SPDX-License-Identifier: FSFAP
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice and
# this notice are preserved. This file is offered as-is, without any warranty.

8
9
10
11
12
workflow:
  rules:
    # If the branch matches the ones it should not run on, ignore it.
    - if: $SALSA_CI_IGNORED_BRANCHES && $CI_COMMIT_REF_NAME =~ $SALSA_CI_IGNORED_BRANCHES
      when: never
13
14
15
    # Avoid running on gbp pq's patch-queue branches
    - if: $CI_COMMIT_REF_NAME =~ /^patch-queue\/.*/
      when: never
16
    # Avoid duplicated pipelines, do not run detached pipelines.
Iñaki Malerba's avatar
Iñaki Malerba committed
17
18
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
19
20
21
22
23
24
25
    # Otherwise, if there's a debian/ folder, run.
    - exists:
        - debian/**
      when: always
    # Fallback to not running.
    - when: never

26
variables:
Iñaki Malerba's avatar
Iñaki Malerba committed
27
  GIT_DEPTH: 1
Agustin Henze's avatar
Agustin Henze committed
28
  DEBFULLNAME: "Salsa Pipeline"
29
  DEBEMAIL: "salsa-pipeline@debian.org"
Agustin Henze's avatar
Agustin Henze committed
30
  DEBIAN_FRONTEND: noninteractive
31
  WORKING_DIR: $CI_PROJECT_DIR/debian/output
32
  SOURCE_DIR: 'source_dir'
Raphaël Hertzog's avatar
Raphaël Hertzog committed
33
  VENDOR: 'debian'
34
35
36
  RELEASE:
    description: "The Debian release to build for"
    value: 'unstable'
37
38
  # the architecture of the builder
  BUILD_ARCH: 'amd64'
39
40
  # only set this for cross-compiling
  HOST_ARCH: ''
41
  SALSA_CI_MIRROR: 'http://deb.debian.org/debian'
42
  SALSA_CI_COMPONENTS: 'main'
43
  SALSA_CI_IMAGES: 'registry.salsa.debian.org/salsa-ci-team/pipeline'
44
  SALSA_CI_IMAGES_APTLY: ${SALSA_CI_IMAGES}/aptly
45
46
  SALSA_CI_IMAGES_AUTOPKGTEST: ${SALSA_CI_IMAGES}/autopkgtest
  SALSA_CI_IMAGES_BASE: ${SALSA_CI_IMAGES}/base:${RELEASE}
47
  SALSA_CI_IMAGES_BASE_I386: ${SALSA_CI_IMAGES}/i386/base:${RELEASE}
48
49
50
  SALSA_CI_IMAGES_BASE_ARM32V5: ${SALSA_CI_IMAGES}/arm32v5/base:${RELEASE}
  SALSA_CI_IMAGES_BASE_ARM32V7: ${SALSA_CI_IMAGES}/arm32v7/base:${RELEASE}
  SALSA_CI_IMAGES_BASE_ARM64: ${SALSA_CI_IMAGES}/arm64v8/base:${RELEASE}
Maximiliano Curia's avatar
Maximiliano Curia committed
51
  SALSA_CI_IMAGES_GENERIC_TESTS: ${SALSA_CI_IMAGES}/generic_tests:${RELEASE}
52
  SALSA_CI_IMAGES_BLHC: ${SALSA_CI_IMAGES}/blhc:latest
53
  SALSA_CI_IMAGES_GBP: ${SALSA_CI_IMAGES}/gbp:latest
54
  SALSA_CI_IMAGES_LINTIAN: ${SALSA_CI_IMAGES}/lintian:latest
55
56
  SALSA_CI_IMAGES_PIUPARTS: ${SALSA_CI_IMAGES}/piuparts:latest
  SALSA_CI_IMAGES_REPROTEST: ${SALSA_CI_IMAGES}/reprotest:latest
57
58
59
  SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE:
    description: "Set this to 1 to produce an in-depth comparision of reprotest results"
    value: 0
60
  SALSA_CI_AUTOPKGTEST_LXC: https://salsa.debian.org/salsa-ci-team/autopkgtest-lxc
61
62
63
  SALSA_CI_AUTOPKGTEST_ARGS:
    description: "debci_autopkgtest_args is set to this variable and used by autopkgtest"
    value: ''
64
  SALSA_CI_BLHC_ARGS: ''
65
  SALSA_CI_LINTIAN_SUPPRESS_TAGS: ""
66
  SALSA_CI_LINTIAN_FAIL_WARNING: ""
67
68
69
  SALSA_CI_LINTIAN_SHOW_OVERRIDES:
    description: "To make Lintian show overridden tags, set this to 1"
    value: 0
Raphaël Hertzog's avatar
Raphaël Hertzog committed
70
  SALSA_CI_PIUPARTS_ARGS: ''
71
72
  SALSA_CI_PIUPARTS_PRE_INSTALL_SCRIPT: ''
  SALSA_CI_PIUPARTS_POST_INSTALL_SCRIPT: ''
73
  SALSA_CI_DPKG_BUILDPACKAGE_ARGS: ''
74
  SALSA_CI_GBP_BUILDPACKAGE_ARGS: ''
75
  DOCKER_TLS_CERTDIR: ""
76
  SALSA_CI_DISABLE_APTLY: 1
77
78
79
80
81
  # These three ARM-related build jobs are disabled by default while there
  # isn't an ARM shared runner available
  SALSA_CI_DISABLE_BUILD_PACKAGE_ARMEL: 1
  SALSA_CI_DISABLE_BUILD_PACKAGE_ARMHF: 1
  SALSA_CI_DISABLE_BUILD_PACKAGE_ARM64: 1
82
  SALSA_CI_DISABLE_MISSING_BREAKS: 1
83
  SALSA_CI_DISABLE_RC_BUGS: 1
84
  SALSA_CI_IGNORED_BRANCHES: ''
85
  SALSA_CI_BUILD_TIMEOUT_ARGS: "2.75h"
86
87
  # Use fastzip to fix https://salsa.debian.org/salsa-ci-team/pipeline/-/issues/177
  FF_USE_FASTZIP: 'true'
88
89
90
  # Backward compatibility
  SALSA_CI_EXTRA_REPOSITORY: ${EXTRA_REPOSITORY}
  SALSA_CI_EXTRA_REPOSITORY_KEY: ${EXTRA_REPOSITORY_KEY}
91
  PYTHONIOENCODING: utf-8
Agustin Henze's avatar
Agustin Henze committed
92
93

stages:
94
  - provisioning
Agustin Henze's avatar
Agustin Henze committed
95
  - build
96
  - publish
Agustin Henze's avatar
Agustin Henze committed
97
98
  - test

99
.artifacts: &artifacts
Raul Benencia's avatar
Raul Benencia committed
100
101
102
103
  name: "$CI_JOB_NAME:$CI_COMMIT_REF_NAME"
  when: always
  paths:
    - ${WORKING_DIR}/
104
105
106
107
108

.artifacts-default-expire: &artifacts-default-expire
  artifacts:
    <<: *artifacts

109
110
111
112
113
114
115
116
117
118
119
.provisioning-extract-source: &provisioning-extract-source
  stage: provisioning
  image: $SALSA_CI_IMAGES_GBP
  extends:
    - .artifacts-default-expire
  variables:
    DB_BUILD_PARAM: ${SALSA_CI_DPKG_BUILDPACKAGE_ARGS}
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
  script:
120
    - mkdir -p ${WORKING_DIR}
121

122
    - gbp pull --ignore-branch --pristine-tar --track-missing
123

124
    - |
125
126
127
128
129
130
131
132
      if find . -maxdepth 3 -wholename "*/debian/source/format" -exec cat {} \; | grep -q '3.0 (gitarchive)'
      then
        apt-get update && eatmydata apt-get install --no-install-recommends -y \
          dpkg-source-gitarchive
        { DSC=$(dpkg-source --build . | tee /dev/fd/3 | sed -n 's/.* \(\S*.dsc$\)/\1/p') ; } 3>&1
        dpkg-source --extract --no-check ../$DSC ${WORKING_DIR}/${DSC%.dsc}
      else
        # Check if we can obtain the orig from the git branches
133

134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        if ! gbp export-orig --tarball-dir=${WORKING_DIR}; then
          # Fallback using origtargz
          apt-get update
          origtargz -dt
          cp ../*orig.tar* ${WORKING_DIR}
          SALSA_CI_GBP_BUILDPACKAGE_ARGS="--git-overlay ${SALSA_CI_GBP_BUILDPACKAGE_ARGS}"
        fi

        # As of 2020-09-09, gbp doesn't have a simpler method to extract the
        # debianized source package. Use --git-pbuilder=`/bin/true` for the moment:
        # https://bugs.debian.org/969952

        gbp buildpackage \
          --git-ignore-branch \
          --git-ignore-new \
          --git-no-create-orig \
          --git-export-dir=${WORKING_DIR} \
          --no-check-builddeps \
          --git-builder=/bin/true \
          --git-no-pbuilder \
          --git-no-hooks \
          --git-no-purge \
          ${SALSA_CI_GBP_BUILDPACKAGE_ARGS} |& filter-output
      fi
158

159
    - ls -lh ${WORKING_DIR}
160
161
162
    - cd ${WORKING_DIR}
    - DEBIANIZED_SOURCE=$(find . -maxdepth 3 -wholename "*/debian/changelog" | sed -e 's%/\w*/\w*$%%')
    - |
163
164
165
166
167
      if [ ! "${DEBIANIZED_SOURCE}" ] ; then
        echo "Error: No valid debianized source tree found."
        exit 1
      fi

168
    - mv ${DEBIANIZED_SOURCE} ${SOURCE_DIR}
169

170
.build-before-script: &build-before-script
171
172
173
174
175
176
177
  # Reported in https://salsa.debian.org/salsa-ci-team/pipeline/issues/104,
  # GitLab can only expand variables once. So at the beginning CCACHE_WORK_DIR
  # was assigned to `${WORKING_DIR}/.ccache`, and it will be expanded as
  # `$CI_PROJECT_DIR/debian/output/.ccache`, so it creates a folder named
  # "\$CI_PROJECT_DIR", which is then saved as build cache. To allow smooth
  # transition, that wrongly named folder has to be removed:
  - rm -rf '$CI_PROJECT_DIR'
178

179
180
  # salsa-ci-team/pipeline#107
  - rm -rf ${CI_PROJECT_DIR}/debian/output/.ccache
181

182
  - mkdir -p ${WORKING_DIR} ${CCACHE_WORK_DIR}
183

184
185
  # https://salsa.debian.org/salsa-ci-team/pipeline/-/merge_requests/230
  - rm -rf ${CCACHE_TMP_DIR}
186

187
188
  - mv ${CCACHE_WORK_DIR} ${CCACHE_TMP_DIR}
  - add_extra_repository.sh -v -e "${SALSA_CI_EXTRA_REPOSITORY}" -k "${SALSA_CI_EXTRA_REPOSITORY_KEY}"
Sven Hartge's avatar
Sven Hartge committed
189

190
191
  # are we cross-compiling? if not, unset HOST_ARCH
  - test "${BUILD_ARCH}" != "${HOST_ARCH}" || HOST_ARCH=""
192

193
.build-script: &build-script
194
195
196
197
198
199
  - export CCACHE_DIR=${CCACHE_TMP_DIR}

  # add target architecture if cross-compiliing
  - test -z "${HOST_ARCH}" || dpkg --add-architecture ${HOST_ARCH}

  # Add deb-src entries
200
201
202
203
204
205
206
207
  - |
    if [ -f /etc/apt/sources.list ]; then
      sed -n '/^deb\s/s//deb-src /p' /etc/apt/sources.list > /etc/apt/sources.list.d/deb-src.list
    fi
  - |
    if [ -f /etc/apt/sources.list.d/debian.sources ]; then
      sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources
    fi
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245

  - |
    apt-get update && eatmydata apt-get install --no-install-recommends -y \
      ccache \
      fakeroot \
      build-essential

  # in case we are cross-building, install some more dependencies
  # see #815172 why we need libc-dev and libstdc++-dev
  - |
    test -z "${HOST_ARCH}" || eatmydata apt-get satisfy --no-install-recommends -y \
      libc-dev:${HOST_ARCH} \
      libstdc++-dev:${HOST_ARCH} \
      crossbuild-essential-${HOST_ARCH}
  # when cross-compiling, add 'nocheck' to the DEB_BUILD_OPTIONS
  - test -z "${HOST_ARCH}" || export DEB_BUILD_OPTIONS=nocheck${DEB_BUILD_OPTIONS:+ }${DEB_BUILD_OPTIONS}

  # Enter source package dir
  - cd ${WORKING_DIR}/${SOURCE_DIR}

  # Install package build dependencies
  # use plain "apt-get build-dep" so that we can install only packages for
  # architecture indep or arch:any builds
  - aptopts=""
  - test "$DB_BUILD_TYPE" != "any" || aptopts="--arch-only"
  - test "$DB_BUILD_TYPE" != "all" || aptopts="--indep-only"
  # use aspcud solver for experimental and backports
  - |
    if [ "$RELEASE" = "experimental" ] || [[ "$RELEASE" =~ .*-backports$ ]]; then
      eatmydata apt-get install --no-install-recommends -y aspcud apt-cudf
      aptopts="$aptopts --solver aspcud -oAPT::Solver::Strict-Pinning=false -oAPT::Solver::aspcud::Preferences="
      # minimize number of packages from experimental and backports
      if [ "$RELEASE" = "experimental" ]; then
        aptopts="$aptopts-count(solution,APT-Release:=/a=experimental/),"
      elif [[ "$RELEASE" =~ .*-backports$ ]]; then
        aptopts="$aptopts-count(solution,APT-Release:~/a=.*-backports/),"
      fi
      aptopts="$aptopts-removed,-changed,-new"
246
     fi
247
  - eatmydata apt-get build-dep ${HOST_ARCH:+--host-architecture ${HOST_ARCH} -Pcross,nocheck} --no-install-recommends -y $aptopts .
248

249
250
251
252
253
  # If not disabled, bump package version
  - |
    if ! echo "$SALSA_CI_DISABLE_VERSION_BUMP" | grep -qE '^(1|yes|true)$'; then
      sed -i -e '1 s/)/+salsaci)/' debian/changelog
    fi
254

255
256
257
  # Generate ccache links
  - dpkg-reconfigure ccache
  - PATH="/usr/lib/ccache/:${PATH}"
258

259
260
  # Reset ccache stats
  - ccache -z
Iñaki Malerba's avatar
Iñaki Malerba committed
261

262
263
264
  # Create salsaci user and fix permissions
  - useradd salsaci
  - chown -R salsaci. ${WORKING_DIR} ${CCACHE_DIR}
Iñaki Malerba's avatar
Iñaki Malerba committed
265

266
267
268
269
270
271
  # Define buildlog filename
  - BUILD_LOGFILE_SOURCE=$(dpkg-parsechangelog -S Source)
  - BUILD_LOGFILE_VERSION=$(dpkg-parsechangelog -S Version)
  - BUILD_LOGFILE_VERSION=${BUILD_LOGFILE_VERSION#*:}
  - BUILD_LOGFILE_ARCH=${HOST_ARCH:-${BUILD_ARCH}}
  - BUILD_LOGFILE="${WORKING_DIR}/${BUILD_LOGFILE_SOURCE}_${BUILD_LOGFILE_VERSION}_${BUILD_LOGFILE_ARCH}.build"
272

273
274
  # Build package as user salsaci
  - su salsaci -c "timeout ${SALSA_CI_BUILD_TIMEOUT_ARGS} eatmydata dpkg-buildpackage ${HOST_ARCH:+--host-arch ${HOST_ARCH} -Pcross,nocheck} --build=${DB_BUILD_TYPE} ${DB_BUILD_PARAM}" |& OUTPUT_FILENAME=${BUILD_LOGFILE} filter-output
Iñaki Malerba's avatar
Iñaki Malerba committed
275

276
277
278
  # Restore PWD to ${WORKING_DIR}
  - cd ${WORKING_DIR}
  - rm -rf ${WORKING_DIR}/${SOURCE_DIR}
Iñaki Malerba's avatar
Iñaki Malerba committed
279

280
281
  # Print ccache stats on job log
  - ccache -s
282
283
284
285

.build-definition: &build-definition
  stage: build
  image: $SALSA_CI_IMAGES_BASE
286
  cache:
287
    key: "build-${BUILD_ARCH}_${HOST_ARCH}"
288
    paths:
289
      - .ccache
290
291
  variables:
    CCACHE_TMP_DIR: ${CI_PROJECT_DIR}/../.ccache
292
    CCACHE_WORK_DIR: ${CI_PROJECT_DIR}/.ccache
293
    DB_BUILD_PARAM: ${SALSA_CI_DPKG_BUILDPACKAGE_ARGS}
294
    DB_BUILD_TYPE: full
Agustin Henze's avatar
Agustin Henze committed
295
  script:
Sven Hartge's avatar
Sven Hartge committed
296
297
    - *build-before-script
    - *build-script
298
    - mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
299
300
301
  dependencies:
    - extract-source

302
.build-package: &build-package
303
304
305
  extends:
    - .build-definition
    - .artifacts-default-expire
306
307
308
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
Iñaki Malerba's avatar
Iñaki Malerba committed
309

310
311
312
313
314
.build-package-i386: &build-package-i386
  extends:
    - .build-package
  image: $SALSA_CI_IMAGES_BASE_I386
  variables:
315
    BUILD_ARCH: 'i386'
316
317
  except:
    variables:
318
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
319
320
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_I386 =~ /^(1|yes|true)$/

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
.build-package-armel: &build-package-armel
  extends:
    - .build-package
  image: $SALSA_CI_IMAGES_BASE_ARM32V5
  variables:
    BUILD_ARCH: 'armel'
  tags:
    - arm64
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_ARMEL =~ /^(1|yes|true)$/

.build-package-armhf: &build-package-armhf
  extends:
    - .build-package
  image: $SALSA_CI_IMAGES_BASE_ARM32V7
  variables:
    BUILD_ARCH: 'armhf'
  tags:
    - arm64
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_ARMHF =~ /^(1|yes|true)$/

.build-package-arm64: &build-package-arm64
  extends:
    - .build-package
  image: $SALSA_CI_IMAGES_BASE_ARM64
  variables:
    BUILD_ARCH: 'arm64'
  tags:
    - arm64
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_ARM64 =~ /^(1|yes|true)$/

360
.build-source-only: &build-source-only
361
362
  extends:
    - .build-definition
363
    - .artifacts-default-expire
364
365
  cache:
    paths: []  # Override cache for source builds
366
  variables:
367
    DB_BUILD_TYPE: source
368
    SALSA_CI_DISABLE_VERSION_BUMP: 1
369
370
371
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
Iñaki Malerba's avatar
Iñaki Malerba committed
372

373
.test-build-package-any: &test-build-package-any
374
375
  extends:
    - .build-definition
376
  stage: test
Sven Hartge's avatar
Sven Hartge committed
377
378
  script:
    - *build-before-script
379
    - LOCAL_ARCH=$(dpkg --print-architecture)
Sven Hartge's avatar
Sven Hartge committed
380
    - |
381
382
383
384
385
386
387
388
389
390
391
392
393
      if egrep -q "^Architecture:.*(any|[^\!]${LOCAL_ARCH})" debian/control; then
        DB_BUILD_TYPE="any"
      else
        echo "###########################################"
        echo "### No binary package of type any or ${LOCAL_ARCH} found"
        echo "### Stopping test-build-any test."
        echo "###########################################"
        echo "You should disable this job via:"
        echo "variables:"
        echo "  SALSA_CI_DISABLE_BUILD_PACKAGE_ANY: '1'"
        mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
        exit 0
      fi
Sven Hartge's avatar
Sven Hartge committed
394
395
396
397
    - *build-script
    - mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
  except:
    variables:
398
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
399
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_ANY =~ /^(1|yes|true)$/
Sven Hartge's avatar
Sven Hartge committed
400

401

402
.test-build-package-all: &test-build-package-all
403
404
  extends:
    - .build-definition
405
  stage: test
Sven Hartge's avatar
Sven Hartge committed
406
407
  script:
    - *build-before-script
408
    - LOCAL_ARCH=$(dpkg --print-architecture)
Sven Hartge's avatar
Sven Hartge committed
409
    - |
410
411
412
413
414
415
416
417
418
419
420
421
422
      if grep -q "^Architecture: all" debian/control; then
        DB_BUILD_TYPE="all"
      else
        echo "###########################################"
        echo "### No binary package of type all found"
        echo "### Stopping test-build-all test."
        echo "###########################################"
        echo "You should disable this job via:"
        echo "variables:"
        echo "  SALSA_CI_DISABLE_BUILD_PACKAGE_ALL: '1'"
        mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
        exit 0;
      fi
Sven Hartge's avatar
Sven Hartge committed
423
424
425
426
    - *build-script
    - mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
  except:
    variables:
427
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
428
      - $SALSA_CI_DISABLE_BUILD_PACKAGE_ALL =~ /^(1|yes|true)$/
429

430
431
432
433
434
435
436
.test-crossbuild-package-arm64: &test-crossbuild-package-arm64
  extends:
    - .build-definition
  stage: test
  script:
    - *build-before-script
    - |
437
438
439
440
441
442
443
444
445
446
447
448
449
      if test -n ${HOST_ARCH} && egrep -q "^Architecture:.*(any|[^\!]${HOST_ARCH})" debian/control; then
        DB_BUILD_TYPE="any"
      else
        echo "###########################################"
        echo "### No binary package of type any or ${HOST_ARCH} found"
        echo "### Stopping ${CI_JOB_NAME} test."
        echo "###########################################"
        echo "You should disable this job via:"
        echo "variables:"
        echo "  SALSA_CI_DISABLE_CROSSBUILD_ARM64: '1'"
        mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
        exit 0
      fi
450
451
452
453
454
455
456
457
458
459
460
    - *build-script
    - mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
  variables:
    HOST_ARCH: arm64
  except:
    variables:
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
      - $SALSA_CI_DISABLE_CROSSBUILD_ARM64 =~ /^(1|yes|true)$/
      - $BUILD_ARCH == $HOST_ARCH
      - $HOST_ARCH == ""

461
.test-autopkgtest: &test-autopkgtest
Agustin Henze's avatar
Agustin Henze committed
462
  stage: test
463
  image: $SALSA_CI_IMAGES_AUTOPKGTEST
464
465
  except:
    variables:
466
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
467
      - $SALSA_CI_DISABLE_AUTOPKGTEST =~ /^(1|yes|true)$/
Agustin Henze's avatar
Agustin Henze committed
468
  script:
469
    - wget --progress=dot:giga ${SALSA_CI_AUTOPKGTEST_LXC}/-/jobs/artifacts/master/raw/artifacts/lxc.tar?job=${RELEASE} -O lxc.tar
470
    - mkdir ${SCI_LXC_PATH} && tar xf lxc.tar -C ${SCI_LXC_PATH}
471
    - sed -i "/lxc.rootfs.path/ s@dir:.*/lxc/@dir:${SCI_LXC_PATH}/@" ${SCI_LXC_PATH}/autopkgtest-${RELEASE}-amd64/config
472
    - |
473
474
475
      cat >/etc/lxc/lxc.conf <<EOT
      lxc.lxcpath=${SCI_LXC_PATH}
      EOT
476
    - add_extra_repository.sh -v -e "${SALSA_CI_EXTRA_REPOSITORY}" -k "${SALSA_CI_EXTRA_REPOSITORY_KEY}"
477
      -t "${SCI_LXC_PATH}/autopkgtest-${RELEASE}-amd64/rootfs/etc"
478
479
480
    - umount -R /sys/fs/cgroup && mount -a
    - /etc/init.d/lxc-net start
    - /etc/init.d/lxc start
Santiago R.R.'s avatar
Santiago R.R. committed
481
    - chown -R debci. ${WORKING_DIR}
482
    - export debci_autopkgtest_args="${SALSA_CI_AUTOPKGTEST_ARGS}"
483
484
485
    # su's -P is required to have ownership over /dev/stderr, /dev/stdout and
    # /dev/stdin, and then fix #256
    - su -P debci -c "debci localtest $WORKING_DIR/*.changes --suite ${RELEASE} --logs-dir ${DEBCI_LOG_PATH}" || ( ret=$?; [ $ret -eq 8 ] || [ $ret -eq 2 ] )
486
    - rm -rf ${WORKING_DIR}/debci/binaries
487
  variables:
488
    GIT_STRATEGY: fetch
Raul Benencia's avatar
Raul Benencia committed
489
    SCI_LXC_PATH: ${CI_PROJECT_DIR}/lxc
490
491
492
493
494
    DEBCI_LOG_PATH: ${WORKING_DIR}/debci
  artifacts:
    when: always
    paths:
      - ${WORKING_DIR}/debci
495
496
497
  needs:
    - job: build
      artifacts: true
Agustin Henze's avatar
Agustin Henze committed
498

Sven Hartge's avatar
Sven Hartge committed
499
.test-blhc: &test-blhc
Sven Hartge's avatar
Sven Hartge committed
500
  stage: test
501
  image: $SALSA_CI_IMAGES_BLHC
502
503
  except:
    variables:
504
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
505
      - $SALSA_CI_DISABLE_BLHC =~ /^(1|yes|true)$/
Sven Hartge's avatar
Sven Hartge committed
506
  script:
507
    - blhc --debian --line-numbers --color ${SALSA_CI_BLHC_ARGS} ${WORKING_DIR}/*.build || [ $? -eq 1 ]
Sven Hartge's avatar
Sven Hartge committed
508
  variables:
Raul Benencia's avatar
Raul Benencia committed
509
    GIT_STRATEGY: none
510
511
512
  needs:
    - job: build
      artifacts: true
Sven Hartge's avatar
Sven Hartge committed
513

514
.test-lintian: &test-lintian
Agustin Henze's avatar
Agustin Henze committed
515
  stage: test
516
  image: $SALSA_CI_IMAGES_LINTIAN
517
518
  except:
    variables:
519
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
520
      - $SALSA_CI_DISABLE_LINTIAN =~ /^(1|yes|true)$/
Agustin Henze's avatar
Agustin Henze committed
521
  script:
522
    - lintian --version
523
    - |
524
525
526
      if lintian --fail-on error --allow-root > /dev/null ; then
        if echo "${SALSA_CI_LINTIAN_FAIL_WARNING}" | grep -qE '^(1|yes|true)$'; then
          SALSA_CI_LINTIAN_FAIL_ARG='--fail-on error --fail-on warning'
527
        else
528
          SALSA_CI_LINTIAN_FAIL_ARG='--fail-on error'
529
        fi
530
531
      else
        SALSA_CI_LINTIAN_FAIL_ARG=''
Agustin Henze's avatar
Agustin Henze committed
532
      fi
533
534
535
      if echo "${SALSA_CI_LINTIAN_SHOW_OVERRIDES}" | grep -qE '^(1|yes|true)$'; then
        SALSA_CI_LINTIAN_SHOW_OVERRIDES_ARG='--show-overrides'
      fi
536
    - lintian --suppress-tags "${SALSA_CI_LINTIAN_SUPPRESS_TAGS}" --display-info --pedantic ${SALSA_CI_LINTIAN_FAIL_ARG} --allow-root ${SALSA_CI_LINTIAN_SHOW_OVERRIDES_ARG} ${WORKING_DIR}/*.changes | tee lintian.output || ECODE=$?
537
538
    - |
        if echo "${SALSA_CI_LINTIAN_FAIL_WARNING}" | grep -qE '^(1|yes|true)$'; then
539
          grep -q '^W: ' lintian.output && ECODE=3
540
        fi
541
    - lintian2junit.py --lintian-file lintian.output > ${WORKING_DIR}/lintian.xml
542
543
    # 🗂️ Generate HTML report.
    - lintian --suppress-tags "${SALSA_CI_LINTIAN_SUPPRESS_TAGS}"  --display-info --pedantic --allow-root ${SALSA_CI_LINTIAN_SHOW_OVERRIDES_ARG} --exp-output format=html ${WORKING_DIR}/*.changes > ${WORKING_DIR}/lintian.html || true
544
    - exit ${ECODE-0}
545
  variables:
Raul Benencia's avatar
Raul Benencia committed
546
    GIT_STRATEGY: none
547
548
549
  artifacts:
    reports:
      junit: ${WORKING_DIR}/lintian.xml
550
    paths:
551
      - ${WORKING_DIR}/lintian.html
Iñaki Malerba's avatar
Iñaki Malerba committed
552
    when: always
553
554
555
  needs:
    - job: build
      artifacts: true
Agustin Henze's avatar
Agustin Henze committed
556

557
.test-reprotest: &test-reprotest
Agustin Henze's avatar
Agustin Henze committed
558
  stage: test
559
  image: $SALSA_CI_IMAGES_REPROTEST
560
561
  except:
    variables:
562
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
563
      - $SALSA_CI_DISABLE_REPROTEST =~ /^(1|yes|true)$/
564
  artifacts:
565
    name: "$CI_JOB_NAME:$CI_COMMIT_REF_NAME"
566
    paths:
567
568
      - $WORKING_DIR/reprotest
      - $WORKING_DIR/reprotest.log
569
    when: always
Agustin Henze's avatar
Agustin Henze committed
570
  script:
571
    - add_extra_repository.sh -v -e "${SALSA_CI_EXTRA_REPOSITORY}" -k "${SALSA_CI_EXTRA_REPOSITORY_KEY}"
Agustin Henze's avatar
Agustin Henze committed
572
    - apt-get update
573
    - eatmydata apt-get build-dep -y ${WORKING_DIR}/*.dsc
574
    - |
575
576
577
      if ! echo "${SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE}" | grep -q -E '^(1|yes|true)$'; then
        SALSA_CI_REPROTEST_ARGS="${SALSA_CI_REPROTEST_ARGS} --no-diffoscope"
      fi
578
    - |
579
580
581
582
      timeout ${SALSA_CI_BUILD_TIMEOUT_ARGS} reprotest \
        --min-cpus $(nproc --all) \
        --store-dir ${WORKING_DIR}/reprotest \
        --verbosity=2  \
Luca Boccassi's avatar
Luca Boccassi committed
583
        --vary=-time \
584
585
        ${SALSA_CI_REPROTEST_ARGS} \
        ${WORKING_DIR}/*.dsc -- null |& OUTPUT_FILENAME=reprotest.log filter-output
586
  variables:
587
    GIT_STRATEGY: none
588
589
590
  needs:
    - job: build
      artifacts: true
Agustin Henze's avatar
Agustin Henze committed
591

592
# Only for compat with the old way of enabling diffoscope
593
.test-reprotest-diffoscope: &test-reprotest-diffoscope
594
595
  extends:
    - .test-reprotest
596
  variables:
597
    SALSA_CI_REPROTEST_ENABLE_DIFFOSCOPE: '1'
598

599
.test-piuparts: &test-piuparts
Agustin Henze's avatar
Agustin Henze committed
600
  stage: test
601
  image: $SALSA_CI_IMAGES_PIUPARTS
602
603
  except:
    variables:
604
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
605
      - $SALSA_CI_DISABLE_PIUPARTS =~ /^(1|yes|true)$/
Agustin Henze's avatar
Agustin Henze committed
606
  services:
607
    - docker:20.10.12-dind
Agustin Henze's avatar
Agustin Henze committed
608
  script:
609
    - CHROOT_PATH="/tmp/debian-chroot"
610
    - CONTAINER_ID=$(docker run --rm -d "${SALSA_CI_IMAGES_BASE}" sleep infinity)
Agustin Henze's avatar
Agustin Henze committed
611
612
613
614
    - docker exec ${CONTAINER_ID} bash -c "apt-get update && apt-get install eatmydata -y"
    - mkdir -p ${CHROOT_PATH}
    - docker export ${CONTAINER_ID} | tar -C ${CHROOT_PATH} -xf -
    - mknod -m 666 ${CHROOT_PATH}/dev/urandom c 1 9
615
    - mkdir -p /srv/local-apt-repository/ && cp -a ${WORKING_DIR}/*.deb /srv/local-apt-repository/ && /usr/lib/local-apt-repository/rebuild
616
    - mkdir -p ${CHROOT_PATH}/etc-target/apt/sources.list.d ${CHROOT_PATH}/etc-target/apt/preferences.d
617
    - cp -Hv /etc/apt/sources.list.d/local-apt-repository.list ${CHROOT_PATH}/etc-target/apt/sources.list.d/
618
    - cp -aTLv /etc/apt/preferences.d  ${CHROOT_PATH}/etc-target/apt/preferences.d
619
620
    - cp -aTLv /srv/local-apt-repository ${CHROOT_PATH}/srv/local-apt-repository
    - cp -aTLv /var/lib/local-apt-repository/ ${CHROOT_PATH}/var/lib/local-apt-repository/
621
622
    - test -n "${SALSA_CI_PIUPARTS_PRE_INSTALL_SCRIPT}" && cp -aTLv "${SALSA_CI_PIUPARTS_PRE_INSTALL_SCRIPT}" /etc/piuparts/scripts/pre_install_salsa_ci && chmod 755 /etc/piuparts/scripts/pre_install_salsa_ci
    - test -n "${SALSA_CI_PIUPARTS_POST_INSTALL_SCRIPT}" && cp -aTLv "${SALSA_CI_PIUPARTS_POST_INSTALL_SCRIPT}" /etc/piuparts/scripts/post_install_salsa_ci && chmod 755 /etc/piuparts/scripts/post_install_salsa_ci
623
624
    - add_extra_repository.sh -v -e "${SALSA_CI_EXTRA_REPOSITORY}"
      -k "${SALSA_CI_EXTRA_REPOSITORY_KEY}" -t "${CHROOT_PATH}/etc-target"
625
    - sed  '/127.0.0.1/s/localhost/pipeline.salsa.debian.org localhost/' /etc/hosts > ${CHROOT_PATH}/etc/hosts
Raphaël Hertzog's avatar
Raphaël Hertzog committed
626
627
    - PIUPARTS_DISTRIBUTION_ARG="--distribution $RELEASE"
    - |
628
629
630
631
      if [ "$VENDOR" = "debian" ]; then \
        CODENAME=$(wget -O - ${SALSA_CI_MIRROR}/dists/${RELEASE}/Release | awk "/^Codename:/ { print \$2 }" | cut -d- -f1); \
        PIUPARTS_DISTRIBUTION_ARG="--distribution ${CODENAME}"; \
      fi
632
    - |
633
634
635
      (for PACKAGE in $(ls ${WORKING_DIR}/*.deb); do
        piuparts --mirror "${SALSA_CI_MIRROR} ${SALSA_CI_COMPONENTS}" ${SALSA_CI_PIUPARTS_ARGS} --scriptsdir /etc/piuparts/scripts --allow-database --warn-on-leftovers-after-purge --hard-link -e ${CHROOT_PATH} ${PIUPARTS_DISTRIBUTION_ARG} ${PACKAGE}
      done) | filter-output
636
  variables:
Agustin Henze's avatar
Agustin Henze committed
637
638
639
640
    # To make the repository available in this job,
    # so SALSA_CI_PIUPARTS_{PRE,POST}_INSTALL_SCRIPT
    # can refer to committed scripts
    GIT_STRATEGY: fetch
641
642
643
  needs:
    - job: build
      artifacts: true
Maximiliano Curia's avatar
Maximiliano Curia committed
644
645
646

.test-rc-bugs: &test-rc-bugs
  stage: test
647
  image: $SALSA_CI_IMAGES_GENERIC_TESTS
648
649
  except:
    variables:
650
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
651
      - $SALSA_CI_DISABLE_RC_BUGS =~ /^(1|yes|true)$/
Maximiliano Curia's avatar
Maximiliano Curia committed
652
  script:
653
    - check_rc_bugs.py -v -o ${WORKING_DIR}/rc_bugs.xml --changes-file ${WORKING_DIR}/*.changes
Maximiliano Curia's avatar
Maximiliano Curia committed
654
655
656
  artifacts:
    reports:
      junit: ${WORKING_DIR}/rc_bugs.xml
657
  variables:
Raul Benencia's avatar
Raul Benencia committed
658
    GIT_STRATEGY: none
659
660
661
  needs:
    - job: build
      artifacts: true
Maximiliano Curia's avatar
Maximiliano Curia committed
662
663
664
665

.test-missing-breaks: &test-missing-breaks
  stage: test
  image: $SALSA_CI_IMAGES_GENERIC_TESTS
666
667
  except:
    variables:
668
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
669
      - $SALSA_CI_DISABLE_MISSING_BREAKS =~ /^(1|yes|true)$/
Maximiliano Curia's avatar
Maximiliano Curia committed
670
671
672
673
674
675
676
  script:
    - apt-get update
    - check_for_missing_breaks_replaces.py -o ${WORKING_DIR}/missing_breaks.xml --changes-file ${WORKING_DIR}/*.changes
  artifacts:
    reports:
      junit: ${WORKING_DIR}/missing_breaks.xml
  variables:
Raul Benencia's avatar
Raul Benencia committed
677
    GIT_STRATEGY: none
678
679
680
  needs:
    - job: build
      artifacts: true
681
682
683
684
685
686
687
688
689
690

.publish-aptly: &publish-aptly
  stage: publish
  image: $SALSA_CI_IMAGES_APTLY
  variables:
    GIT_STRATEGY: none
    REPO_PATH: 'aptly'
    PUBKEY_FILENAME: 'public-key.asc'
  except:
    variables:
691
      - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/
692
693
      - $SALSA_CI_DISABLE_APTLY =~ /^(1|yes|true)$/
  script:
694
    - export REPO_URL="${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/${REPO_PATH}"
695
    - export REPO_PUBKEY_URL="${REPO_URL}/${PUBKEY_FILENAME}"
696
    - export GPG_TTY=$(tty)
697
698
699
    - aptly repo create -distribution ${RELEASE} -component main ${CI_PROJECT_NAME}
    - aptly repo add ${CI_PROJECT_NAME} "${WORKING_DIR}"
    - aptly repo show -with-packages ${CI_PROJECT_NAME}
700
    # accept miss-spelled var for backwards-compatibility (see https://salsa.debian.org/salsa-ci-team/pipeline/issues/114)
701
    - export SALSA_CI_APTLY_GPG_PASSPHRASE="${SALSA_CI_APTLY_GPG_PASSPHRASE:=${SALSA_CI_APTLY_GPG_PASSPHASE}}"
702
    - |
703
704
705
706
707
708
709
710
711
712
      if [ -n "${SALSA_CI_APTLY_GPG_KEY}" ]; then \
        echo "${SALSA_CI_APTLY_GPG_KEY}" \
        | gpg --import ${SALSA_CI_APTLY_GPG_PASSPHRASE:+ --batch --passphrase "${SALSA_CI_APTLY_GPG_PASSPHRASE}"}; \
      else \
        export SALSA_CI_APTLY_GPG_PASSPHRASE="${SALSA_CI_APTLY_GPG_PASSPHRASE:-${CI_PROJECT_NAME}:${CI_PIPELINE_ID}}"; \
        rngd -r /dev/urandom; \
        printf "Key-Type: RSA\nKey-Length: 2048\nName-Real: ${DEBFULLNAME}\nName-Email: ${DEBEMAIL}\nExpire-Date: 0\nPassphrase: ${SALSA_CI_APTLY_GPG_PASSPHRASE}\n%%commit" | \
        gpg --batch --gen-key; \
      fi
      gpg --export --armor > "${PUBKEY_FILENAME}"
713
    - |
714
715
716
      ARCHITECTURES=$(aptly repo show -with-packages ${CI_PROJECT_NAME} | \
        awk 'BEGIN {FS="_"} /^Packages:/ {x=NR} (x && NR>x) {print $3}' | \
        sort -u | tr '\n' ','); \
717
718
        ARCHITECTURES=${ARCHITECTURES%,}; \
        aptly publish repo -batch \
719
720
721
        ${ARCHITECTURES:+ -architectures=${ARCHITECTURES}} \
        ${SALSA_CI_APTLY_GPG_PASSPHRASE:+ -passphrase="${SALSA_CI_APTLY_GPG_PASSPHRASE}"} \
        ${CI_PROJECT_NAME}
722
    - |
723
      mkdir -p "${CI_PROJECT_DIR}/${REPO_PATH}"
724
725
726
727
728
729
        cp -a ~/.aptly/public/. "${CI_PROJECT_DIR}/${REPO_PATH}"
        mv "${PUBKEY_FILENAME}" "${CI_PROJECT_DIR}/${REPO_PATH}/${PUBKEY_FILENAME}"
        envsubst < /etc/aptly/index.html.template > "${CI_PROJECT_DIR}/${REPO_PATH}/index.html"
  artifacts:
    paths:
      - ${CI_PROJECT_DIR}/${REPO_PATH}
730
731
732
  needs:
    - job: build
      artifacts: true