Sync suitable parts of MariaDB 10.3 gitlab-ci.yml to this repository

Also cherry-pick autopkg tests fixes from
mariadb-10.3@1ba2a673

debian/gitlab-ci.yml

@@ -8,7 +8,8 @@ variables:
 
 stages:
   - build
-  - test
+  - test quality
+  - upgrades in Debian stable
 
 build package:
   stage: build
@@ -37,75 +38,48 @@ build package:
     - du -shc ${WORKING_DIR}/* # Show total file size of artifacts. Must stay are under 100 MB.
     - mv ${CCACHE_TMP_DIR} ${CCACHE_WORK_DIR}
 
-# Autopkgtest must be disabled, because the test 'upstream' will be skipped and
-# thus the command returns exit code 2 and is considered failed by Gitlab CI.
-# Removing 'breaks-testbed' stanza from tests/control will result in that the
-# test is run on Gitlab CI, but then it will fail in multiple tests as it cannot
-# bind to network interfaces and other needes stuff inside Gitlab CI containers.
-#
-#run autopkgtest:
-#  stage: test
-#  script:
-#    - apt-get update && apt-get install autopkgtest eatmydata -y --no-install-recommends
-#    - eatmydata autopkgtest ${WORKING_DIR}/*.deb -- null
-
-.run_lintian: &run_lintian
-  stage: test
-  image: registry.salsa.debian.org/salsa-ci-team/images/lintian:${RELEASE}
-  script:
-    - apt-get update
-    - apt-get install -y dpkg-dev
-    - lintian -iI ${WORKING_DIR}/*.changes
-
-run lintian stretch:
-  <<: *run_lintian
+run lintian:
+  stage: test quality
   dependencies:
     - build package
+  image: registry.salsa.debian.org/salsa-ci-team/images/lintian:${RELEASE}
+  script:
+    - lintian ${WORKING_DIR}/*.changes | tee lintian.output
+    - cat lintian.output | /root/go/bin/lintian2junit > ${WORKING_DIR}/lintian.xml
   variables:
-    RELEASE: stretch
-
-# Reprotest exceeds the 2 hour timeout limit on Github CI, so results in failure
-# and must be disabled until a quicker (times two) build is possible.
-#run reprotest:
-#  stage: test
-#  image: genericpipeline/reprotest-docker
-#  artifacts:
-#    name: "$CI_BUILD_NAME"
-#    expire_in: 180 day
-#    paths:
-#      - ./reprotest.log
-#    when: always
-#  script:
-#    - apt-get update && apt-get install eatmydata -y
-#    - eatmydata apt-get build-dep -y .
-#    - export DEB_BUILD_OPTIONS=nocheck
-#    - eatmydata reprotest . -- null &> reprotest.log
-#  tags:
-#    - privileged
+      GIT_STRATEGY: none
+  artifacts:
+    reports:
+      junit: ${WORKING_DIR}/lintian.xml
 
-# Piuparts is passing on official piuparts.debian.org, but fail on Gitlab CI
-# with 'FAIL: Package purging left files on system'. Disabling for now.
-#run piuparts:
-#  stage: test
-#  image: genericpipeline/piuparts-docker
-#  services:
-#    - docker:dind
-#  script:
-#    - CHROOT_PATH=/tmp/debian-unstable
-#    - CONTAINER_ID=$(docker run --rm -d debian:unstable sleep infinity)
-#    - docker exec ${CONTAINER_ID} bash -c "apt-get update && apt-get install eatmydata -y"
-#    - mkdir -p ${CHROOT_PATH}
-#    - docker export ${CONTAINER_ID} | tar -C ${CHROOT_PATH} -xf -
-#    - mknod -m 666 ${CHROOT_PATH}/dev/urandom c 1 9
-#    - piuparts --hard-link -e ${CHROOT_PATH} ${WORKING_DIR}/*.deb
-#  tags:
-#    - privileged
+fresh install:
+  stage: test quality
+  image: debian:${RELEASE}
+  artifacts:
+    when: always
+    name: "$CI_BUILD_NAME"
+    paths:
+      - ${WORKING_DIR}/debug
+  script:
+    - cd ${WORKING_DIR} # Don't repeat this step, it's just cd ./debian/output
+    - mkdir debug
+    - dpkg -l | grep -iE '"'"'maria|mysql|galera'"'"' || true
+    - apt-get update
+    - apt-get install -y ./*.deb
+    - mariadb --version
+    - find /var/lib/mysql -ls > debug/var-lib-mysql.list
+    - cp -ra /etc/mysql debug/etc-mysql
+    - service mysql restart || true # Always proceed even if init failed
+    - service mysql status || true
+    - cp -ra /var/log/mysql debug/var-log-mysql
+    - mariadb --skip-column-names -e "select @@version, @@version_comment"
+    - mariadb -e "create database test; use test; create table t(a int primary key) engine=innodb; insert into t values (1); select * from t; drop table t; drop database test;"
 
-test upgrade stretch:
-  stage: test
+upgrade on stretch:
+  stage: upgrades in Debian stable
   dependencies:
     - build package
-  image: debian:stretch
+  image: debian:stable
   artifacts:
     when: always
     name: "$CI_BUILD_NAME"
@@ -113,15 +87,18 @@ test upgrade stretch:
       - ${WORKING_DIR}/debug
   script:
     - cd ${WORKING_DIR} # Don't repeat this step, it's just cd ./debian/output
-    - mkdir debug
-    - dpkg -l | grep -iE '"'"'maria|mysql|galera'"'"' || true
     - apt-get update
-    - apt-get install -y mariadb-server-10.1 # Debian Stretch has this and other -10.1 packages
+    - apt-get install -y 'default-mysql*' 'mariadb-*' 'libmariadbd*' 'libmariadbclient*' # Install almost everything curently in Debian Stretch
+    - dpkg -l | grep -iE '"'"'maria|mysql|galera'"'"' || true
     - service mysql status || true # Always proceed even if init failed
     - service mysql restart || true
     - mariadb --skip-column-names -e "select @@version, @@version_comment"
+    - echo 'deb http://deb.debian.org/debian sid main' > /etc/apt/sources.list
+    - apt-get update; apt-get install -y apt # Uprade minimal stack first
     - apt-get install -y ./*.deb
+    - dpkg -l | grep -iE '"'"'maria|mysql|galera'"'"' || true
     - mariadb --version
+    - mkdir -p debug # Ensure dir exists before using it
     - find /var/lib/mysql -ls > debug/var-lib-mysql.list
     - cp -ra /etc/mysql debug/etc-mysql
     - service mysql restart || true # Always proceed even if init failed
@@ -130,8 +107,10 @@ test upgrade stretch:
     - mariadb --skip-column-names -e "select @@version, @@version_comment"
     - mariadb -e "create database test; use test; create table t(a int primary key) engine=innodb; insert into t values (1); select * from t; drop table t; drop database test;"
 
-fresh install:
-  stage: test
+test basic features:
+  stage: test quality
+  dependencies:
+    - build package
   image: debian:${RELEASE}
   artifacts:
     when: always
@@ -140,15 +119,99 @@ fresh install:
       - ${WORKING_DIR}/debug
   script:
     - cd ${WORKING_DIR} # Don't repeat this step, it's just cd ./debian/output
-    - mkdir debug
     - dpkg -l | grep -iE '"'"'maria|mysql|galera'"'"' || true
     - apt-get update
     - apt-get install -y ./*.deb
     - mariadb --version
+    - mkdir -p debug # Ensure dir exists before using it
     - find /var/lib/mysql -ls > debug/var-lib-mysql.list
     - cp -ra /etc/mysql debug/etc-mysql
     - service mysql restart || true # Always proceed even if init failed
     - service mysql status || true
     - cp -ra /var/log/mysql debug/var-log-mysql
+    # Print info about server
     - mariadb --skip-column-names -e "select @@version, @@version_comment"
-    - mariadb -e "create database test; use test; create table t(a int primary key) engine=innodb; insert into t values (1); select * from t; drop table t; drop database test;"
+    - mariadb --skip-column-names -e "select engine, support, transactions, savepoints from information_schema.engines order by engine" | sort
+    - mariadb --skip-column-names -e "select plugin_name, plugin_status, plugin_type, plugin_library, plugin_license from information_schema.all_plugins order by plugin_name, plugin_library"
+    # Test various features
+    - mariadb -e "CREATE DATABASE db"
+    - mariadb -e "CREATE TABLE db.t_innodb(a1 SERIAL, c1 CHAR(8)) ENGINE=InnoDB; INSERT INTO db.t_innodb VALUES (1,'"'"'foo'"'"'),(2,'"'"'bar'"'"')"
+    - mariadb -e "CREATE TABLE db.t_myisam(a2 SERIAL, c2 CHAR(8)) ENGINE=MyISAM; INSERT INTO db.t_myisam VALUES (1,'"'"'foo'"'"'),(2,'"'"'bar'"'"')"
+    - mariadb -e "CREATE TABLE db.t_aria(a3 SERIAL, c3 CHAR(8)) ENGINE=Aria; INSERT INTO db.t_aria VALUES (1,'"'"'foo'"'"'),(2,'"'"'bar'"'"')"
+    - mariadb -e "CREATE TABLE db.t_memory(a4 SERIAL, c4 CHAR(8)) ENGINE=MEMORY; INSERT INTO db.t_memory VALUES (1,'"'"'foo'"'"'),(2,'"'"'bar'"'"')"
+    - mariadb -e "CREATE ALGORITHM=MERGE VIEW db.v_merge AS SELECT * FROM db.t_innodb, db.t_myisam, db.t_aria"
+    - mariadb -e "CREATE ALGORITHM=TEMPTABLE VIEW db.v_temptable AS SELECT * FROM db.t_innodb, db.t_myisam, db.t_aria"
+    - mariadb -e "CREATE PROCEDURE db.p() SELECT * FROM db.v_merge"
+    - mariadb -e "CREATE FUNCTION db.f() RETURNS INT DETERMINISTIC RETURN 1"
+    # Test that the features still work (this step can be done e.g. after an upgrade)
+    - mariadb -e "SHOW TABLES IN db"
+    - mariadb -e "SELECT * FROM db.t_innodb; INSERT INTO db.t_innodb VALUES (3,'"'"'foo'"'"'),(4,'"'"'bar'"'"')"
+    - mariadb -e "SELECT * FROM db.t_myisam; INSERT INTO db.t_myisam VALUES (3,'"'"'foo'"'"'),(4,'"'"'bar'"'"')"
+    - mariadb -e "SELECT * FROM db.t_aria; INSERT INTO db.t_aria VALUES (3,'"'"'foo'"'"'),(4,'"'"'bar'"'"')"
+    - mariadb -e "SELECT * FROM db.t_memory; INSERT INTO db.t_memory VALUES (3,'"'"'foo'"'"'),(4,'"'"'bar'"'"')"
+    - mariadb -e "SELECT COUNT(*) FROM db.v_merge"
+    - mariadb -e "SELECT COUNT(*) FROM db.v_temptable"
+    - mariadb -e "CALL db.p()"
+    - mariadb -e "SELECT db.f()"
+
+# Stanza 'breaks-testbed' must be removed because the test 'upstream' will otherwise be skipped and
+# thus the command would return exit code 2, and be considered failed by Gitlab CI.
+run autopkgtest:
+  stage: test quality
+  dependencies:
+    - build package
+  image: registry.salsa.debian.org/salsa-ci-team/images/autopkgtest:${RELEASE}
+  script:
+    - sed s/breaks-testbed//g -i debian/tests/control
+    - cat debian/tests/control
+    - eatmydata autopkgtest -U --override-control=debian/tests/control ${WORKING_DIR}/*.changes -- null || [ $? -eq 8 ]
+  # Repo needs to be checked out so the debian/tests/control fiel can be modified on-the-fly.
+  #variables:
+  #  GIT_STRATEGY: none
+
+
+# Build a piece of software that was designed for libmysqlclient-dev but using the
+# libmariadb-dev-compat layer. Should always end up using libmariadb.so.3 run-time.
+build mariadbclient consumer Python-MySQLdb:
+  stage: test quality
+  dependencies:
+    - build package
+  image: debian:${RELEASE}
+  script:
+    - cd ${WORKING_DIR} # Don't repeat this step, it's just cd ./debian/output
+    - mkdir -p debug # Ensure dir exists before using it
+    - apt-get update
+    - apt-get install -y pkg-config ./libmariadbclient-dev*.deb ./libmariadbclient18_*.deb ./mariadb-common*.deb
+    - pkg-config --list-all # See what MySQLdb builds with
+    - ldconfig -p | grep -e mariadb -e mysql
+    - mysql_config # Must be present for mysqlclient Python module to build
+    - apt-get install -y python3-pip
+    - pip3 install mysqlclient # Compiles module against libmysqlclient
+    - apt-get purge -y libmariadb-dev # Not needed for run-time
+    - python3 -c "import MySQLdb; print(MySQLdb.get_client_info())"
+
+mysql-5.5 to mariadb upgrade:
+  stage: upgrades in Debian stable
+  dependencies:
+    - build package
+  image: debian:jessie
+  artifacts:
+    when: always
+    name: "$CI_BUILD_NAME"
+    paths:
+      - ${WORKING_DIR}/debug
+  script:
+    - apt-get update
+    - apt-get install -y mysql-server
+    - /etc/init.d/mysql restart
+    - mysql --version
+    - echo 'SHOW DATABASES;' | mysql
+    - cd ${WORKING_DIR}; apt-get install -y ./*.deb || true # Allow to proceed so debug artifacts get collected
+    - service mysql restart || true # Always proceed even if init failed
+    - mkdir -p debug # Ensure dir exists before using it
+    - find /var/lib/mysql -ls > debug/var-lib-mysql.list
+    - cp -ra /etc/mysql debug/etc-mysql
+    - cp -ra /var/log/mysql debug/var-log-mysql
+    - service mysql status
+    - mariadb --version
+    - echo 'SHOW DATABASES;' | mariadb

debian/tests/upstream

@@ -27,6 +27,14 @@ main.ctype_uca : Requires writable /usr
 rpl.rpl_gtid_mode : Requires starting server as root ref http://bugs.mysql.com/bug.php?id=70517
 EOF
 
+# Skip tests that cannot run properly on Gitlab-CI
+if [ ! -z "$GITLAB_CI" ]
+then
+  cat >> $SKIP_TEST_LST << EOF
+main.mysqld--help : For unknown reason table-cache is 4000 instead of default 421
+EOF
+fi
+
 ARCH=$(dpkg --print-architecture)
 if [ "$ARCH" = "s390x" ]; then
     echo "main.func_regexp_pcre : recursion fails on s390x https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1723947" >> $SKIP_TEST_LST