[openssh-commits] [openssh] 01/01: Add self-hosted runners for VMs of other platforms.

git+noreply at mindrot.org git+noreply at mindrot.org
Wed Feb 17 18:24:23 AEDT 2021


This is an automated email from the git hooks/post-receive script.

dtucker pushed a commit to branch master
in repository openssh.

commit f031366535650b88248ed7dbf23033afdf466240
Author: Darren Tucker <dtucker at dtucker.net>
Date:   Fri Jan 15 14:11:43 2021 +1100

    Add self-hosted runners for VMs of other platforms.
    
    Github only hosts a limited number of platforms, and the runner code
    is only supported on slightly wider range of platforms.  To increase
    our test coverage beyond that, we run the runner natively on a VM host,
    where it runs a jobs that boot VMs of other platforms, waits for them
    to come up then runs the build and test by ssh'ing into the guest.
    This means that the minimum dependencies for the guests are quite low
    (basically just sshd, a compiler and make).
    
    The interface to the VM host is fairly simple (basically 3 scripts:
    vmstartup, vmrun and vmshutdown), but those are specific to the VM host
    so are not in the public repo.  We also mount the working directory on the
    host via sshfs, so things like artifact upload by the runner also work.
    
    As part of this we are moving the per-test-target configs into a single
    place (.github/configs) where there will be referenced by a single short
    "config" key.  I plan to make the github-hosted runners use this too.
    
    The self-hosted runners are run off a private repo on github since that
    prevents third parties from accessing them[0], and since runner quota is
    limited on private repos, we avoid running the tests we run on the public
    repo.
    
    [0] https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners#self-hosted-runner-security-with-public-repositories
---
 .github/configs                  | 65 ++++++++++++++++++++++++++++++++++++++
 .github/configure.sh             |  6 ++++
 .github/run_test.sh              | 23 ++++++--------
 .github/setup_ci.sh              |  2 +-
 .github/workflows/c-cpp.yml      |  4 +++
 .github/workflows/selfhosted.yml | 67 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 153 insertions(+), 14 deletions(-)

diff --git a/.github/configs b/.github/configs
new file mode 100755
index 00000000..1cf80027
--- /dev/null
+++ b/.github/configs
@@ -0,0 +1,65 @@
+#!/bin/sh
+#
+# usage: configs vmname test_config (or '' for default)
+#
+# Sets the following variables:
+# CONFIGFLAGS           options to ./configure
+# SSHD_CONFOPTS         sshd_config options
+# TEST_TARGET           make target used when testing.  defaults to "tests".
+# LTESTS
+
+config=$1
+
+TEST_TARGET="tests"
+LTESTS=""
+SUDO=sudo	# run with sudo by default
+TEST_SSH_UNSAFE_PERMISSIONS=1
+
+CONFIGFLAGS=""
+LIBCRYPTOFLAGS=""
+
+case "$config" in
+    default|sol64)
+	;;
+    *pam)
+	CONFIGFLAGS="--with-pam"
+	SSHD_CONFOPTS="UsePam yes"
+	;;
+    without-openssl)
+	LIBCRYPTOFLAGS="--without-openssl"
+        TEST_TARGET=t-exec
+	;;
+    *)
+	echo "Unknown configuration $config"
+	exit 1
+	;;
+esac
+
+# The Solaris 64bit targets are special since they need a non-flag arg.
+case "$config" in
+    sol64*)
+	CONFIGFLAGS="x86_64 --with-cflags=-m64 --with-ldflags=-m64 ${CONFIGFLAGS}"
+	LIBCRYPTOFLAGS="--with-ssl-dir=/usr/local/ssl64"
+	;;
+esac
+
+case "${TARGET_HOST}" in
+    sol10)
+	# This VM is 32bit and the unit tests are slow.
+	TEST_TARGET="tests SKIP_UNIT=1"
+	;;
+esac
+
+# If we have a local openssl/libressl, use that.
+if [ -z "${LIBCRYPTOFLAGS}" ]; then
+	# last-match
+	for i in /usr/local /usr/local/ssl; do
+		if [ -x ${i}/bin/openssl ]; then
+			LIBCRYPTOFLAGS="--with-ssl-dir=${i}"
+		fi
+	done
+fi
+
+CONFIGFLAGS="${CONFIGFLAGS} ${LIBCRYPTOFLAGS}"
+
+export LTESTS SUDO TEST_TARGET TEST_SSH_UNSAFE_PERMISSIONS
diff --git a/.github/configure.sh b/.github/configure.sh
new file mode 100755
index 00000000..869dc824
--- /dev/null
+++ b/.github/configure.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+. .github/configs $1 $2
+
+set -x
+./configure ${CONFIGFLAGS}
diff --git a/.github/run_test.sh b/.github/run_test.sh
index 5a0e6538..c2173020 100755
--- a/.github/run_test.sh
+++ b/.github/run_test.sh
@@ -1,23 +1,11 @@
 #!/usr/bin/env bash
 
-TARGETS=$@
-
-TEST_TARGET="tests"
-LTESTS=""  # all tests by default
+. .github/configs $1 $2
 
 [ -z "${SUDO}" ] || ${SUDO} mkdir -p /var/empty
 
 set -ex
 
-for TARGET in $TARGETS; do
-    case $TARGET in
-    --without-openssl)
-        # When built without OpenSSL we can't do the file-based RSA key tests.
-        TEST_TARGET=t-exec
-        ;;
-    esac
-done
-
 if [ -z "$LTESTS" ]; then
     make $TEST_TARGET
     result=$?
@@ -26,6 +14,15 @@ else
     result=$?
 fi
 
+if [ ! -z ${SSHD_CONFOPTS} ]; then
+    echo "rerunning tests with TEST_SSH_SSHD_CONFOPTS='${SSHD_CONFOPTS}'"
+    make t-exec TEST_SSH_SSHD_CONFOPTS="${SSHD_CONFOPTS}"
+    result2=$?
+    if [ "${result2}" -ne 0 ]; then
+        result="${result2}"
+    fi
+fi
+
 if [ "$result" -ne "0" ]; then
     for i in regress/failed*; do
         echo -------------------------------------------------------------------------
diff --git a/.github/setup_ci.sh b/.github/setup_ci.sh
index 67a76a5d..187a4fad 100755
--- a/.github/setup_ci.sh
+++ b/.github/setup_ci.sh
@@ -20,7 +20,7 @@ lsb_release -a
 
 for TARGET in $TARGETS; do
     case $TARGET in
-    ""|--without-openssl|--without-zlib|--with-Werror|--with-rpath*)
+    ""|--without-openssl|--without-zlib|--with-Werror|--with-rpath*|--with-ssl-dir=*|--with-zlib=*)
         # nothing to do
         ;;
     "--with-kerberos5")
diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml
index e546a9c5..7d02bc31 100644
--- a/.github/workflows/c-cpp.yml
+++ b/.github/workflows/c-cpp.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   ubuntu-20_04:
+    if: github.repository != 'openssh/openssh-portable-selfhosted'
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -37,6 +38,7 @@ jobs:
 
 
   ubuntu-18_04:
+    if: github.repository != 'openssh/openssh-portable-selfhosted'
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -69,6 +71,7 @@ jobs:
 
 
   ubuntu-16_04:
+    if: github.repository != 'openssh/openssh-portable-selfhosted'
     runs-on: ubuntu-16.04
     strategy:
       matrix:
@@ -95,6 +98,7 @@ jobs:
 
 
   macos:
+    if: github.repository != 'openssh/openssh-portable-selfhosted'
     strategy:
       matrix:
         os: [ macos-10.15, macos-11.0 ]
diff --git a/.github/workflows/selfhosted.yml b/.github/workflows/selfhosted.yml
new file mode 100644
index 00000000..835bfb32
--- /dev/null
+++ b/.github/workflows/selfhosted.yml
@@ -0,0 +1,67 @@
+name: C/C++ CI self-hosted
+
+on:
+  push:
+    branches: [ master, ci ]
+
+jobs:
+  selfhosted:
+    runs-on: ${{ matrix.vm }}
+    env:
+      TARGET_HOST: ${{ matrix.vm }}
+      SUDO: sudo
+    strategy:
+      fail-fast: false
+      # We use a matrix in two parts: firstly all of the VMs are tested with the
+      # default config.  "vm" corresponds to a label associated with the worker.
+      matrix:
+        vm: [dfly30, dfly48, dfly58, sol10, sol11]
+        configs:
+          - default
+        # Then we include any extra configs we want to test for specific VMs.
+        include:
+         - vm: dfly30
+           configs: without-openssl
+         - vm: dfly48
+           configs: pam
+         - vm: dfly58
+           configs: pam
+         - vm: sol10
+           configs: pam
+         - vm: sol11
+           configs: pam
+         - vm: sol11
+           configs: sol64
+         - vm: sol11
+           configs: sol64-pam
+    steps:
+    - uses: actions/checkout at v2
+    - name: autoreconf
+      run: autoreconf
+    - name: shutdown VM if running
+      run: vmshutdown
+    - name: startup VM
+      run: vmstartup
+    - name: configure
+      run: vmrun ./.github/configure.sh ${{ matrix.configs }}
+    - name: save config files
+      if: always()
+      uses: actions/upload-artifact at v2
+      with:
+        name: ${{ matrix.vm }}-${{ matrix.configs }}-config-files
+        path: |
+          config.h
+          config.log
+    - name: make
+      run: vmrun make
+    - name: make tests
+      run: vmrun ./.github/run_test.sh ${{ matrix.configs }}
+    - name: save regress logs
+      if: failure()
+      uses: actions/upload-artifact at v2
+      with:
+        name: ${{ matrix.vm }}-${{ matrix.configs }}-regress-logs
+        path: regress/*.log
+    - name: shutdown VM
+      if: always()
+      run: vmshutdown

-- 
To stop receiving notification emails like this one, please contact
djm at mindrot.org.


More information about the openssh-commits mailing list