Jenkinsfile

-/**
- * Remember that the build failed because one of the untrusted files were
- * modified.
- */
-untrusted = false
-
 /**
  * SSSD CI.
  *
@@ -41,6 +35,11 @@ class CI {
    */
   public static String SuiteDir = this.BaseDir + '/sssd-test-suite'
 
+  /**
+   * Path to SSSD CI tools on Jenkins slave.
+   */
+  public static String CIDir = this.BaseDir + '/sssd-ci'
+
   /**
    * Workaround for https://issues.jenkins-ci.org/browse/JENKINS-39203
    *
@@ -52,6 +51,7 @@ class CI {
    * parallel failed.
    */
   public static def Results = [:]
+  public static def RebaseResults = [:]
 
   /**
    * Mark build as successfull.
@@ -67,6 +67,20 @@ class CI {
     return this.Results[build] == "success"
   }
 
+  /**
+   * Mark build as successfully rebased.
+   */
+  public static def RebaseSuccessful(build) {
+    this.RebaseResults[build] = "success"
+  }
+
+  /**
+   * Return true if the rebase was successful.
+   */
+  public static def IsRebaseSuccessful(build) {
+    return this.RebaseResults[build] == "success"
+  }
+
   /**
    * Send commit status to Github for sssd-ci context.
    */
@@ -93,18 +107,66 @@ class CI {
       )
   }
 
+  public static def Rebase(ctx) {
+    if (!ctx.env.CHANGE_TARGET) {
+      this.RebaseSuccessful(ctx.env.TEST_SYSTEM)
+      return
+    }
+
+    ctx.echo String.format('Rebasing on %s', ctx.env.CHANGE_TARGET)
+
+    ctx.sh String.format(
+      'git -C %s fetch --no-tags --progress origin +refs/heads/%s:refs/remotes/origin/%s',
+      "${ctx.env.WORKSPACE}/sssd",
+      ctx.env.CHANGE_TARGET,
+      ctx.env.CHANGE_TARGET
+    )
+
+    // Remove left overs from previous rebase if there are any
+    ctx.sh String.format(
+      'git -C %s rebase --abort || :',
+      "${ctx.env.WORKSPACE}/sssd"
+    )
+
+    // Just to be sure
+    ctx.sh String.format(
+      'rm -fr "%s/.git/rebase-apply" || :',
+      "${ctx.env.WORKSPACE}/sssd"
+    )
+
+    ctx.sh String.format(
+      'git -C %s rebase origin/%s',
+      "${ctx.env.WORKSPACE}/sssd",
+      ctx.env.CHANGE_TARGET
+    )
+
+    this.RebaseSuccessful(ctx.env.TEST_SYSTEM)
+  }
+
   /**
    * Run tests. TEST_SYSTEM environment variable must be defined.
    */
   public static def RunTests(ctx) {
+    ctx.echo "Running on ${ctx.env.NODE_NAME}"
     this.NotifyBuild(ctx, 'PENDING', 'Build is in progress.')
+    this.Rebase(ctx)
+
+    ctx.echo String.format(
+      'Executing tests, started at %s',
+      (new Date()).format('dd. MM. yyyy HH:mm:ss')
+    )
 
     ctx.sh String.format(
-      './sssd/contrib/test-suite/run.sh %s %s %s %s',
-      "${ctx.env.WORKSPACE}/sssd",
+      '%s/sssd-test-suite -c "%s" run --sssd "%s" --artifacts "%s" --update --prune',
       "${this.SuiteDir}",
-      "${ctx.env.WORKSPACE}/artifacts/${ctx.env.TEST_SYSTEM}",
-      "${this.BaseDir}/configs/${ctx.env.TEST_SYSTEM}.json"
+      "${this.BaseDir}/configs/${ctx.env.TEST_SYSTEM}.json",
+      "${ctx.env.WORKSPACE}/sssd",
+      "${ctx.env.WORKSPACE}/artifacts/${ctx.env.TEST_SYSTEM}"
+    )
+
+    ctx.echo String.format(
+      'Finished at %s',
+      (new Date()).format('dd. MM. yyyy HH:mm:ss')
     )
 
     this.BuildSuccessful(ctx.env.TEST_SYSTEM)
@@ -114,12 +176,19 @@ class CI {
    * Archive artifacts and notify Github about build result.
    */
   public static def WhenCompleted(ctx) {
+    if (!this.IsRebaseSuccessful(ctx.env.TEST_SYSTEM)) {
+      ctx.echo "Unable to rebase on target branch."
+      this.NotifyBuild(ctx, 'FAILURE', 'Unable to rebase on target branch.')
+      return
+    }
+
     ctx.archiveArtifacts artifacts: "artifacts/**", allowEmptyArchive: true
     ctx.sh String.format(
-      "${this.BaseDir}/scripts/archive.sh %s %s %s",
+      '%s/sssd-ci archive --name "%s" --system "%s" --artifacts "%s"',
+      "${this.CIDir}",
+      "${ctx.env.BRANCH_NAME}/${ctx.env.BUILD_ID}",
       ctx.env.TEST_SYSTEM,
-      "${ctx.env.WORKSPACE}/artifacts/${ctx.env.TEST_SYSTEM}",
-      "${ctx.env.BRANCH_NAME}/${ctx.env.BUILD_ID}"
+      "${ctx.env.WORKSPACE}/artifacts/${ctx.env.TEST_SYSTEM}"
     )
     ctx.sh "rm -fr ${ctx.env.WORKSPACE}/artifacts/${ctx.env.TEST_SYSTEM}"
 
@@ -152,7 +221,6 @@ def CI_Notify(status, message) { CI.Notify(this, status, message) }
 pipeline {
   agent none
   options {
-    timeout(time: 10, unit: 'HOURS')
     checkoutToSubdirectory('sssd')
   }
   stages {
@@ -161,19 +229,6 @@ pipeline {
         CI_Notify('PENDING', 'Running tests.')
       }
     }
-    stage('Read trusted files') {
-      steps {
-        readTrusted './contrib/test-suite/run.sh'
-        readTrusted './contrib/test-suite/run-client.sh'
-      }
-      post {
-        failure {
-          script {
-            untrusted = true
-          }
-        }
-      }
-    }
     stage('Run Tests') {
       parallel {
         stage('Test on Fedora 28') {
@@ -217,14 +272,8 @@ pipeline {
   }
   post {
     failure {
-      script {
-        if (untrusted) {
-          CI_Notify('ERROR', 'Untrusted files were modified.')
-        } else {
       CI_Notify('FAILURE', 'Some tests failed.')
     }
-      }
-    }
     aborted {
       CI_Notify('ERROR', 'Builds were aborted.')
     }

Makefile.am

@@ -346,12 +346,14 @@ PYTHON_TESTS =
 if BUILD_PYTHON2_BINDINGS
 PYTHON_TESTS += src/config/SSSDConfigTest.py2.sh \
                 src/tests/pyhbac-test.py2.sh \
+                src/tests/pysss-test.py2.sh \
                 src/tests/pysss_murmur-test.py2.sh \
                 $(NULL)
 endif
 if BUILD_PYTHON3_BINDINGS
 PYTHON_TESTS += src/config/SSSDConfigTest.py3.sh \
                 src/tests/pyhbac-test.py3.sh \
+                src/tests/pysss-test.py3.sh \
                 src/tests/pysss_murmur-test.py3.sh \
                 $(NULL)
 endif
@@ -456,6 +458,9 @@ dist_noinst_SCRIPTS = \
     src/tests/pyhbac-test.py \
     src/tests/pyhbac-test.py2.sh \
     src/tests/pyhbac-test.py3.sh \
+    src/tests/pysss-test.py \
+    src/tests/pysss-test.py2.sh \
+    src/tests/pysss-test.py3.sh \
     src/tests/pysss_murmur-test.py \
     src/tests/pysss_murmur-test.py2.sh \
     src/tests/pysss_murmur-test.py3.sh \
@@ -473,6 +478,7 @@ dist_noinst_DATA = \
     src/config/testconfigs/sssd-invalid.conf \
     src/config/testconfigs/sssd-invalid-badbool.conf \
     src/config/testconfigs/sssd-nonexisting-services-domains.conf \
+    src/config/testconfigs/sssd-test-parse.conf \
     src/config/etc/sssd.api.d/crash_test_dummy \
     contrib/ci/README.md \
     contrib/ci/configure.sh \
@@ -665,6 +671,7 @@ dist_noinst_HEADERS = \
     src/util/sss_nss.h \
     src/util/sss_ldap.h \
     src/util/sss_python.h \
+    src/util/sss_regexp.h \
     src/util/sss_krb5.h \
     src/util/sss_selinux.h \
     src/util/sss_sockets.h \
@@ -939,7 +946,7 @@ if HAVE_NSS
                         src/util/crypto/nss/nss_obfuscate.c \
                         src/util/crypto/nss/nss_nite.c \
                         src/util/crypto/nss/nss_util.c \
-			src/util/crypto/sss_crypto.c \
+                        src/util/crypto/nss/nss_prng.c \
                         src/util/atomic_io.c \
                         $(NULL)
     SSS_CRYPT_CFLAGS = $(NSS_CFLAGS)
@@ -962,7 +969,7 @@ else
                         src/util/crypto/libcrypto/crypto_sha512crypt.c \
                         src/util/crypto/libcrypto/crypto_obfuscate.c \
                         src/util/crypto/libcrypto/crypto_nite.c \
-			src/util/crypto/sss_crypto.c \
+                        src/util/crypto/libcrypto/crypto_prng.c \
                         src/util/atomic_io.c \
                         $(NULL)
     SSS_CRYPT_CFLAGS = $(CRYPTO_CFLAGS)
@@ -1269,6 +1276,7 @@ libsss_util_la_SOURCES = \
     src/util/sss_ptr_hash.c \
     src/util/files.c \
     src/util/selinux.c \
+    src/util/sss_regexp.c \
     $(NULL)
 libsss_util_la_CFLAGS = \
     $(AM_CFLAGS) \
@@ -3034,12 +3042,22 @@ test_ipa_idmap_LDADD = \
 test_utils_SOURCES = \
     src/tests/cmocka/test_utils.c \
     src/tests/cmocka/test_string_utils.c \
+    src/p11_child/p11_child_common_utils.c \
     $(NULL)
 if BUILD_SSH
 test_utils_SOURCES += src/tests/cmocka/test_sss_ssh.c
 endif
 test_utils_CFLAGS = \
     $(AM_CFLAGS)
+if HAVE_NSS
+test_utils_CFLAGS += \
+    $(NSS_CFLAGS) \
+    $(NULL)
+else
+test_utils_CFLAGS += \
+    $(P11_KIT_CFLAGS) \
+    $(NULL)
+endif
 test_utils_LDADD = \
     $(CMOCKA_LIBS) \
     $(POPT_LIBS) \
@@ -4019,6 +4037,7 @@ intgcheck-prepare:
 	    --with-secrets \
 	    --with-session-recording-shell=/bin/false \
 	    --enable-local-provider \
+	    --enable-files-domain \
 	    $(INTGCHECK_CONFIGURE_FLAGS) \
 	    CFLAGS="-O2 -g $$CFLAGS"; \
 	$(MAKE) $(AM_MAKEFLAGS) ; \
@@ -4430,6 +4449,7 @@ libsss_ipa_la_SOURCES = \
     src/providers/ipa/ipa_srv.c \
     src/providers/ipa/ipa_idmap.c \
     src/providers/ipa/ipa_dn.c \
+    src/providers/ipa/ipa_refresh.c \
     src/providers/ad/ad_opts.c \
     src/providers/ad/ad_common.c \
     src/providers/ad/ad_dyndns.c \
@@ -4502,7 +4522,10 @@ libsss_ad_la_SOURCES = \
     src/providers/ad/ad_gpo_ndr.c \
     src/providers/ad/ad_srv.c \
     src/providers/ad/ad_subdomains.c \
-    src/providers/ad/ad_domain_info.c
+    src/providers/ad/ad_domain_info.c \
+    src/providers/ad/ad_refresh.c \
+    $(NULL)
+
 
 if BUILD_SUDO
 libsss_ad_la_SOURCES += \
@@ -4660,6 +4683,7 @@ proxy_child_LDADD = \
 
 p11_child_SOURCES = \
     src/p11_child/p11_child_common.c \
+    src/p11_child/p11_child_common_utils.c \
     src/util/atomic_io.c \
     src/util/util.c \
     src/util/util_ext.c \

configure.ac

@@ -322,7 +322,7 @@ AC_CHECK_PROG(HAVE_PYTHON3, python3, yes, no)
 AS_IF([test x$HAVE_PYTHON3 = xyes],
       [AC_PATH_PROG(PYTHON3, python3)])
 
-if test x$HAVE_PYTHON2_BINDINGS = x1; then
+if test x$HAVE_PYTHON2_BINDINGS = xyes; then
     AS_IF([test x$HAVE_PYTHON2 != xyes],
           [AC_MSG_ERROR([
 The program python2 was not found in search path.
@@ -348,7 +348,7 @@ them please use argument --without-python2-bindings when running configure.])])
     SSS_CLEAN_PYTHON_VARIABLES
 fi
 
-if test x$HAVE_PYTHON3_BINDINGS = x1; then
+if test x$HAVE_PYTHON3_BINDINGS = xyes; then
     AS_IF([test x$HAVE_PYTHON3 != xyes],
           [AC_MSG_ERROR([
 The program python3 was not found in search path.
@@ -374,7 +374,7 @@ them please use argument --without-python3-bindings when running configure.])])
     SSS_CLEAN_PYTHON_VARIABLES
 fi
 
-if test x$HAVE_PYTHON3_BINDINGS = x1; then
+if test x$HAVE_PYTHON3_BINDINGS = xyes; then
     PYTHON_EXEC=$PYTHON3
 else
     PYTHON_EXEC=$PYTHON2
@@ -385,8 +385,6 @@ AM_CONDITIONAL([BUILD_PYTHON_BINDINGS],
                [test x"$with_python2_bindings" = xyes \
                      -o x"$with_python3_bindings" = xyes])
 
-AM_PYTHON2_MODULE([ldap])
-
 if test x$HAVE_SELINUX != x; then
     AM_CHECK_SELINUX
 fi

contrib/ci/deps.sh

@@ -40,10 +40,6 @@ if [[ "$DISTRO_BRANCH" == -redhat-* ]]; then
         nss_wrapper
         openldap-clients
         openldap-servers
-        pytest
-        python-ldap
-        python-psutil
-        pyldb
         rpm-build
         uid_wrapper
         pam_wrapper
@@ -51,16 +47,29 @@ if [[ "$DISTRO_BRANCH" == -redhat-* ]]; then
         curl-devel
         krb5-server
         krb5-workstation
-        dbus-python
     )
 
-    if [[ "$DISTRO_BRANCH" == -redhat-fedora-3[1-9]* ]]; then
+    if [[ "$DISTRO_BRANCH" == -redhat-fedora-3[1-9]* ||
+          "$DISTRO_BRANCH" == -redhat-redhatenterprise*-8.*- ||
+          "$DISTRO_BRANCH" == -redhat-centos-8.*- ]]; then
         DEPS_LIST+=(
-            python3-pep8
+            python3-dbus
+            python3-ldap
+            python3-ldb
+            python3-psutil
+            python3-pycodestyle
+            python3-pytest
+            python3-requests
         )
     else
         DEPS_LIST+=(
+            dbus-python
+            pyldb
+            pytest
+            python-ldap
             python-pep8
+            python-psutil
+            python-requests
         )
     fi
 

contrib/ci/run

@@ -58,6 +58,12 @@ declare BASE_DIR=`pwd`
 declare MODERATE=false
 declare RIGOROUS=false
 
+# pep8 was renamed to pycodestyle
+declare PEP8_BIN="pep8"
+if which pycodestyle &> /dev/null; then
+  PEP8_BIN="pycodestyle"
+fi
+
 # Output program usage information.
 function usage()
 {
@@ -398,7 +404,7 @@ if [[ "$DISTRO_BRANCH" != redhat-* ]]; then
     PEP8_IGNORE+=",E722"
 fi
 stage pep8          find . -path ./src/config -prune -o \
-                           -name \*.py -exec pep8 $PEP8_IGNORE {} +
+                           -name \*.py -exec $PEP8_BIN $PEP8_IGNORE {} +
 stage autoreconf    autoreconf --install --force
 run_build debug     build_debug
 if "$RIGOROUS"; then

contrib/ci/sssd.supp

@@ -64,6 +64,8 @@
    Memcheck:Leak
    fun:malloc
    fun:pcre_compile2
+   fun:sss_regexp_pcre1_compile
+   fun:sss_regexp_new
    fun:sss_names_init_from_args
    ...
 }

contrib/test-suite/README.md

-# Run SSSD Test Suite
+# SSSD Test Suite
 
-Script `run.sh` will run all available SSSD test on a set of virtual machines created by vagrant. These virtual machines are part of separate project located at `https://github.com/SSSD/sssd-test-suite`.
+SSSD Test Suite is set of test that are being run automatically as part of Pull Request CI.
 
-## Automated Testing
+## Steps to run the tests manually on local machine
 
-These test are run automatically when you submit a Pull Request to SSSD project. Status report together with logs will be available in the Pull Request when testing is finished.
-
-## Steps to run the tests manually
+You need to clone and configure `sssd-test-suite` project to run these test manually on your local machine.
 
 1. Checkout `https://github.com/SSSD/sssd-test-suite`
 2. Configure and setup SSSD test suite per instructions located at project readme.
 3. Make sssd-test-suite use already provisioned boxes (either manually created or maintained by SSSD team at https://app.vagrantup.com/sssd-vagrant).
-4. Run `run.sh`, please note that this script will call `vagrant destroy` and it will thus destroy your existing guests.
+4. Run the tests with `sssd-test-suite` command line interface
 
+```bash
+$ git clone https://github.com/SSSD/sssd-test-suite
+$ cd sssd-test-suite
+$ cp ./configs/sssd-f30.json ./config.json
+$ ./sssd-test-suite run --sssd $path-to-sssd --artifacts /tmp/sssd-artifacts
 ```
-run.sh SSSD-SOURCE-DIR TEST-SUITE-DIR ARTIFACTS-DIR CONFIG-FILE
-  SSSD-SOURCE-DIR Path to SSSD source directory.
-  TEST-SUITE-DIR  Path to sssd-test-suite_dir directory.
-  ARTIFACTS-DIR   Path to directory where artifacts should be stored.
-  CONFIG-FILE     Path to sssd-test-suite_dir configuration file to use.
-```
-
-At this moment only `client` guest is required. We need to expand our test cases to test agains FreeIPA and Active Directory.
-
-## SSSD CI Architecture
-
-Jenkins master polls github for new branches and pull requests. When it discovers new pull request or branch or changes to existing pull request or branch it will allocate a jenkins agent and executes pipeline defined in `./Jenkinsfile` (in SSSD source) on this agent.
-
-The pipeline executes `./contrib/test-suite/run.sh` and archives logs when testing is finished. Script `./contrib/test-suite/run.sh` prepares sssd-test-suite, starts the vagrant machines and copy SSSD source code to the client machine. Then it calls `./contrib/test-suite/run-client.sh` on the client machine which runs continuous integration tests.
-
-### Extending current tests
-To extend current testing capabilities, modify `./contrib/test-suite/run.sh` and `./contrib/test-suite/run-client.sh` to new requirements. These files can be modified by anyone but are considered untrusted from contributor that is not an administrator of SSSD repository. This means that if a public contributor submits a pull request that changes those files, Jenkins will refuse to run tests.
 
-### Adding additional distribution to test on
-You need to modify `./Jenkinsfile`. Simply copy, paste and amend existing Fedora 28 stage. This file is also considered untrusted so only administrators can modify it within a pull request.
+See [sssd-test-suite documentation](https://github.com/SSSD/sssd-test-suite/blob/master/readme.md) for more information.
+See [running the tests documentation](https://github.com/SSSD/sssd-test-suite/blob/master/docs/running-tests.md) for more information about the process.
 
-You also need to extend `sssd-test-suite` and prepare vagrant boxes for this distro.

contrib/test-suite/run-client.sh

-#!/bin/bash
-#
-# DO NOT RUN THIS MANUALLY
-#
-
-sssd_source="/shared/sssd"
-artifacts_dir="/shared/artifacts"
-
-archive-artifacts() {
-    echo "Archiving artifacts..."
-
-    cp -f $sssd_source/ci-*.log $artifacts_dir
-    cp -f $sssd_source/ci-build-debug/ci-*.log $artifacts_dir
-    cp -f $sssd_source/ci-build-debug/test-suite.log $artifacts_dir
-}
-
-success-or-die() {
-    ret=$1
-    msg=$2
-    if [ $ret -eq 0 ]; then
-        return 0
-    fi
-
-    echo $msg
-    archive-artifacts
-
-    exit $ret
-}
-
-cd $sssd_source
-
-echo "[1/1] Running Continuous Integration Tests"
-./contrib/ci/run --moderate --no-deps
-success-or-die $? "CI Failed!"
-
-archive-artifacts
-exit 0

contrib/test-suite/run.sh

-#!/bin/bash
-
-print-usage() {
-    cat <<EOF
-Run SSSD Continuous Integration Tests
-Make sure to checkout and setup https://github.com/SSSD/sssd-test-suite
-
-run.sh SSSD-SOURCE-DIR TEST-SUITE-DIR ARTIFACTS-DIR CONFIG-FILE
-  SSSD-SOURCE-DIR Path to SSSD source directory.
-  TEST-SUITE-DIR  Path to sssd-test-suite_dir directory.
-  ARTIFACTS-DIR   Path to directory where artifacts should be stored.
-  CONFIG-FILE     Path to sssd-test-suite_dir configuration file to use.
-EOF
-}
-
-print-help-if-asked() {
-    while test $# -gt 0
-    do
-        case "$1" in
-            --help)
-                print-usage ; exit 0
-                ;;
-            -h) print-usage ; exit 0
-                ;;
-            -?) print-usage ; exit 0
-                ;;
-        esac
-        shift
-    done
-}
-
-success-or-die() {
-    if [ $1 -ne 0 ]; then
-        echo $2
-        exit 1
-    fi
-}
-
-print-help-if-asked "$@"
-if [[ $# -ne 4 ]]; then
-    print-usage
-    exit 1
-fi
-
-sssd_source=$1
-suite_dir=$2
-artifacts_dir=$3
-config=$4
-
-guest_source="/shared/sssd"
-guest_artifacts="/shared/artifacts"
-
-# Currently only client machine is needed.
-guests="client"
-
-run-vagrant() {
-    VAGRANT_CWD="$suite_dir" \
-    SSSD_TEST_SUITE_RSYNC="$sssd_source:$guest_source" \
-    SSSD_TEST_SUITE_SSHFS="$artifacts_dir:$guest_artifacts" \
-    SSSD_TEST_SUITE_CONFIG="$config" \
-    vagrant "$@"
-}
-
-start-guest() {
-    # This may fail if guest's box was not yet downloaded. We will ignore it.
-    run-vagrant destroy $1 &> /dev/null
-
-    run-vagrant box update $1
-    success-or-die $? "Unable to update guest: $1"
-
-    run-vagrant up $1
-    success-or-die $? "Unable to start guest: $1"
-}
-
-stop-guest() {
-    run-vagrant halt $1
-    success-or-die $? "Unable to halt guest: $1"
-}
-
-echo "[1/5] Creating $artifacts_dir"
-mkdir -p "$artifacts_dir"
-success-or-die $? "Unable to create directory: $artifacts_dir"
-
-echo "[2/5] Updating sssd-test-suite"
-git -C "$suite_dir" pull --rebase
-success-or-die $? "Unable to rebase sssd-test-suite at: $suite_dir"
-
-echo "[3/5] Preparing vagrant machines"
-for guest in $guests; do
-    start-guest $guest
-done
-
-echo "[4/5] Running tests"
-run-vagrant ssh client -- "$guest_source/contrib/test-suite/run-client.sh"
-success-or-die $? "SSSD Test Suite Failed: $?"
-
-echo "[5/5] Shutdown machines"
-for guest in $guests; do
-    stop-guest $guest
-done

contrib/test-suite/test-suite.yml

+- name: Integration Tests
+  machines:
+  - client
+  tasks:
+  - name: Running ./contrib/ci/run
+    shell: ./contrib/ci/run --moderate --no-deps
+  artifacts:
+  - ci-*.log
+  - ci-build-debug/ci-*.log
+  - ci-build-debug/test-suite.log
+  timeout: 6 hours