From f1060d82bf4b4efcd3e7c4d2e2757b137601e2ba Mon Sep 17 00:00:00 2001 From: Devan Carpenter Date: Tue, 13 Jul 2021 16:05:51 -0700 Subject: [PATCH] CI: build CentOS/RedHat 8 RPM packages This commit cleans up the existing CI scripts a bit while adding jobs to build RPM packages. The directory structure allows for easily adding more package targets. RPM outputs will only be uploaded on tagged commits. --- .gitlab-ci-files/common-prepare.yml | 35 +++++++++++- .gitlab-ci.yml | 46 +++++++++++++++ scripts/Makefile | 56 +++++++++++++++++++ scripts/centos8/Makefile | 15 ----- .../centos8/libpEpAdapter.centos8.Dockerfile | 23 -------- scripts/centos8/packages/rpm/Dockerfile | 18 ++++++ .../packages/rpm/create-libpepadapter-rpm.sh | 27 +++++++++ scripts/centos8/packages/rpm/install.sh | 32 +++++++++++ .../build_libpEpAdapter.sh | 0 .../libpEpAdapter.linux.Dockerfile} | 2 +- scripts/debian10/Makefile | 27 --------- scripts/debian10/build_libpEpAdapter.sh | 8 --- 12 files changed, 212 insertions(+), 77 deletions(-) create mode 100644 scripts/Makefile delete mode 100644 scripts/centos8/Makefile delete mode 100644 scripts/centos8/libpEpAdapter.centos8.Dockerfile create mode 100644 scripts/centos8/packages/rpm/Dockerfile create mode 100755 scripts/centos8/packages/rpm/create-libpepadapter-rpm.sh create mode 100755 scripts/centos8/packages/rpm/install.sh rename scripts/{centos8 => common}/build_libpEpAdapter.sh (100%) rename scripts/{debian10/libpEpAdapter.debian10.Dockerfile => common/libpEpAdapter.linux.Dockerfile} (89%) delete mode 100644 scripts/debian10/Makefile delete mode 100755 scripts/debian10/build_libpEpAdapter.sh diff --git a/.gitlab-ci-files/common-prepare.yml b/.gitlab-ci-files/common-prepare.yml index 8053794..2503270 100644 --- a/.gitlab-ci-files/common-prepare.yml +++ b/.gitlab-ci-files/common-prepare.yml @@ -8,15 +8,44 @@ - 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )' - 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )' +.add_ssh_keys: &add_ssh_keys + # Add the SSH key (stored in the SSH_PRIVATE_KEY variable) to the agent. + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + +.verify_ssh_host_keys: &verify_ssh_host_keys + # Verify SSH host keys + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + .standard_job: tags: [kvm] before_script: - *ensure_docker - - *ensure_rsync .make_in_docker: extends: .standard_job script: - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST} - - cd scripts/${CI_DISTRO_TARGET} - - make + - cd scripts + - make ${MAKE_TARGET} + +.upload_pkg: + extends: .standard_job + before_script: + - *ensure_rsync + - *add_ssh_keys + - *verify_ssh_host_keys + script: + - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST} + - cd scripts + - make ${MAKE_TARGET} + - pwd + - cd out + - time rsync -avP -e "ssh -p ${PKG_HOST_SSH_PORT}" depot@${PKG_HOST}:files/pkgs/RHEL/8/libpEpAdapter/SHA256SUMS || true + - if sha256sum --ignore-missing --check SHA256SUMS ; then echo "Package already exists... Exiting..." && exit ; else true ; fi + - sha256sum ./*.rpm | tee --append SHA256SUMS + - cat ./SHA256SUMS + - time rsync -azvP -e "ssh -p ${PKG_HOST_SSH_PORT}" --rsync-path="mkdir -p /home/depot/files/pkgs/RHEL/8/libpEpAdapter && rsync" ./*.rpm SHA256SUMS depot@${PKG_HOST}:files/pkgs/RHEL/8/libpEpAdapter/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dc8459..b61f6e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ include: stages: - build + - packages # Debian @@ -11,6 +12,7 @@ debian10:build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" CI_DISTRO_TARGET: "debian10" DEBIAN_FRONTEND: "noninteractive" rules: @@ -20,8 +22,52 @@ debian10:tagged-build: extends: .make_in_docker stage: build variables: + MAKE_TARGET: "build" CI_DISTRO_TARGET: "debian10" DEBIAN_FRONTEND: "noninteractive" TAGGED_BUILD: "true" rules: - if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + +# CentOS + +centos8:build: + extends: .make_in_docker + stage: build + variables: + MAKE_TARGET: "build" + CI_DISTRO_TARGET: "centos8" + rules: + - if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + +centos8:tagged-build: + extends: .make_in_docker + stage: build + variables: + MAKE_TARGET: "build" + CI_DISTRO_TARGET: "centos8" + TAGGED_BUILD: "true" + rules: + - if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + +centos8:rpm: + extends: .make_in_docker + stage: packages + needs: ["centos8:build"] + variables: + MAKE_TARGET: "rpm" + CI_DISTRO_TARGET: "centos8" + rules: + - if: '$CI_COMMIT_TAG !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' + + +centos8:rpm:tagged-build: + extends: .upload_pkg + stage: packages + needs: ["centos8:tagged-build"] + variables: + MAKE_TARGET: "rpm" + CI_DISTRO_TARGET: "centos8" + TAGGED_BUILD: "true" + rules: + - if: '$CI_COMMIT_TAG =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/' diff --git a/scripts/Makefile b/scripts/Makefile new file mode 100644 index 0000000..745f879 --- /dev/null +++ b/scripts/Makefile @@ -0,0 +1,56 @@ +include ../DEPENDENCIES +export +CURRENT_DISTRO=${CI_DISTRO_TARGET} +IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CI_DISTRO_TARGET)-libpepadapter +DOCKERFILE=libpEpAdapter.linux.Dockerfile +PKG_BUILD_IMAGE=${DOCKER_REGISTRY_HOST}/fpm-$(CURRENT_DISTRO) +PKG_INSTALL_PATH=/opt/pEp +PKG_NAME=libpEpAdapter +PKG_DESCRIPTION="p≡p libpEpAdapter Binary RPM Package" +PKG_DEPENDS=pEpEngine +PKG_INSTALL_PATH_STRING="/ /package/lib=${PKG_INSTALL_PATH} /package/include/pEp=${PKG_INSTALL_PATH}/include" +IS_TAGGED=${TAGGED_BUILD} +ifeq ($(IS_TAGGED), true) +# $CI_COMMIT_TAG is a predefined environment variable from Gitlab + LIBPEPADAPTER_VERSION=${CI_COMMIT_TAG} +else + LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD) +endif +all: build + +build: + -docker pull $(IMAGE_NAME):latest + cd ../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ + --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ + --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \ + --cache-from $(IMAGE_NAME):latest \ + --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \ + --tag=$(IMAGE_NAME):latest \ + -f scripts/common/$(DOCKERFILE) . + docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION} + docker push $(IMAGE_NAME):latest + +rpm: + -docker pull $(PKG_BUILD_IMAGE)-libpepadapter:latest + @docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ + --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \ + --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ + --build-arg PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \ + --cache-from $(PKG_BUILD_IMAGE)-engine:latest \ + --tag=$(PKG_BUILD_IMAGE)-libpepadapter:$(LIBPEPADAPTER_VERSION) \ + --tag=$(PKG_BUILD_IMAGE)-libpepadapter:latest \ + centos8/packages/rpm + @docker push $(PKG_BUILD_IMAGE)-libpepadapter:$(LIBPEPADAPTER_VERSION) + @docker push $(PKG_BUILD_IMAGE)-libpepadapter:latest + @docker run -e LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \ + -e PEP_MACHINE_DIR=$(PEP_MACHINE_DIR) \ + -e PKG_VERSION=$(LIBPEPADAPTER_VERSION) \ + -e PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \ + -e PKG_NAME=$(PKG_NAME) \ + -e PKG_DESCRIPTION=$(PKG_DESCRIPTION) \ + -e PKG_DEPENDS=$(PKG_DEPENDS) \ + -e PKG_INSTALL_PATH_STRING=$(PKG_INSTALL_PATH_STRING) \ + --rm -v $(shell pwd)/centos8/packages/rpm/create-libpepadapter-rpm.sh:/usr/bin/create-rpm.sh:ro \ + -v $(shell pwd)/out:/out \ + -w / $(PKG_BUILD_IMAGE)-libpepadapter:${LIBPEPADAPTER_VERSION} \ + /usr/bin/create-rpm.sh diff --git a/scripts/centos8/Makefile b/scripts/centos8/Makefile deleted file mode 100644 index 8e10078..0000000 --- a/scripts/centos8/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -CURRENT_DISTRO=$(shell basename $(shell pwd)) -LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD) -IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-libpepadapter -DOCKERFILE=libpEpAdapter.$(CURRENT_DISTRO).Dockerfile -all: - -docker pull $(IMAGE_NAME):latest - cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ - --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ - --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \ - --cache-from $(IMAGE_NAME):latest \ - --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \ - --tag=$(IMAGE_NAME):latest \ - -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) . - docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION} - docker push $(IMAGE_NAME):latest diff --git a/scripts/centos8/libpEpAdapter.centos8.Dockerfile b/scripts/centos8/libpEpAdapter.centos8.Dockerfile deleted file mode 100644 index a2fe8a3..0000000 --- a/scripts/centos8/libpEpAdapter.centos8.Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG DOCKER_REGISTRY_HOST -ARG CURRENT_DISTRO -ARG PEPENGINE_VERSION=latest -FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-engine:${PEPENGINE_VERSION} - -ENV BUILDROOT /build -ENV INSTPREFIX /install -ENV OUTDIR /out - -### Setup working directory -RUN mkdir ${BUILDROOT}/libpEpAdapter -COPY . ${BUILDROOT}/libpEpAdapter -USER root -RUN chown -R pep-builder:pep-builder ${BUILDROOT}/libpEpAdapter -USER pep-builder -WORKDIR ${BUILDROOT}/libpEpAdapter - -ARG LIBPEPADAPTER_VERSION -ARG CURRENT_DISTRO - -### Build libpEpAdapter -RUN sh ./scripts/${CURRENT_DISTRO}/build_libpEpAdapter.sh && \ - rm -rf ${BUILDROOT}/* diff --git a/scripts/centos8/packages/rpm/Dockerfile b/scripts/centos8/packages/rpm/Dockerfile new file mode 100644 index 0000000..a7a0164 --- /dev/null +++ b/scripts/centos8/packages/rpm/Dockerfile @@ -0,0 +1,18 @@ +ARG DOCKER_REGISTRY_HOST +ARG CURRENT_DISTRO +ARG LIBPEPADAPTER_VERSION +FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-libpepadapter:${LIBPEPADAPTER_VERSION} AS pEpBuild + +FROM ${DOCKER_REGISTRY_HOST}/fpm-${CURRENT_DISTRO}:latest +# whatever is required for building should be installed in this image; just like BuildRequires: for RPM specs +RUN yum -y install readline-devel epel-release && \ + yum -y install patchelf chrpath + +ARG PEP_MACHINE_DIR +COPY --from=pEpBuild /install /source + +COPY install.sh /usr/local/bin/install.sh + +ENV INSTPREFIX /source + +RUN /usr/local/bin/install.sh diff --git a/scripts/centos8/packages/rpm/create-libpepadapter-rpm.sh b/scripts/centos8/packages/rpm/create-libpepadapter-rpm.sh new file mode 100755 index 0000000..33f3e64 --- /dev/null +++ b/scripts/centos8/packages/rpm/create-libpepadapter-rpm.sh @@ -0,0 +1,27 @@ +#!/bin/bash -ex +# we should always set proper ownership before exiting, otherwise +# the created packages will have root:root ownership and we'll be unable +# to delete them from our host. +trap 'chown -R --reference /usr/bin/create-rpm.sh /out/' EXIT + +# the source directory is mounted read-only to prevent issues where the build +# could alter the source; we should copy it somewhere inside the container +cd /source/out +ls -alh +tree +INSTALL_TOP=/package +mkdir -p ${INSTALL_TOP}/lib +mkdir -p ${INSTALL_TOP}/include/pEp +cp -ar lib/* ${INSTALL_TOP}/lib/. +cp -ar include/pEp/* ${INSTALL_TOP}/include/pEp/. + + +cd /out + +#this would be the no-signature command line +fpm -t rpm -s dir \ + -n ${PKG_NAME} \ + --version ${PKG_VERSION} \ + --description "${PKG_DESCRIPTION}" \ + --depends ${PKG_DEPENDS} \ + -C ${PKG_INSTALL_PATH_STRING} diff --git a/scripts/centos8/packages/rpm/install.sh b/scripts/centos8/packages/rpm/install.sh new file mode 100755 index 0000000..4ffcdc2 --- /dev/null +++ b/scripts/centos8/packages/rpm/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -exuo pipefail + +# =========================== +# Distro +# =========================== + +echo 7 >"${INSTPREFIX}/D_REVISION" + +D_REV=$(cat ${INSTPREFIX}/D_REVISION) +D="" + +D=${INSTPREFIX}/out + +mkdir -p ${INSTPREFIX}/out +rm -rf ${INSTPREFIX}/out/* +mkdir -p "$D"/{bin,ld,lib/pEp,share/pEp,include/pEp} + +# libpEpAdapter +cp -a ${INSTPREFIX}/lib/libpEpAdapter.a "$D"/lib +cp -av ${INSTPREFIX}/include/pEp/{status_to_string.hh,message_cache.hh,call_with_lock.hh,Adapter.hh,callback_dispatcher.hh,locked_queue.hh,slurp.hh,constant_time_algo.hh,Semaphore.hh,pc_container.hh,passphrase_cache.hh,pEpLog.hh,Adapter.hxx,passphrase_cache.hxx} "$D"/include/pEp + + +# versions +cp -a ${INSTPREFIX}/*.ver "$D" + +find "$D"/lib -maxdepth 1 -type f -print -exec patchelf --set-rpath '$ORIGIN/pEp:$ORIGIN' {} \; +find "$D"/lib/pEp -type f -print -exec patchelf --set-rpath '$ORIGIN' {} \; +find "$D"/bin -type f -print -exec patchelf --set-rpath '$ORIGIN/../lib/pEp:$ORIGIN/../lib' {} \; + +ls -lh "$D"/* +du -sch "$D" diff --git a/scripts/centos8/build_libpEpAdapter.sh b/scripts/common/build_libpEpAdapter.sh similarity index 100% rename from scripts/centos8/build_libpEpAdapter.sh rename to scripts/common/build_libpEpAdapter.sh diff --git a/scripts/debian10/libpEpAdapter.debian10.Dockerfile b/scripts/common/libpEpAdapter.linux.Dockerfile similarity index 89% rename from scripts/debian10/libpEpAdapter.debian10.Dockerfile rename to scripts/common/libpEpAdapter.linux.Dockerfile index a2fe8a3..913fad3 100644 --- a/scripts/debian10/libpEpAdapter.debian10.Dockerfile +++ b/scripts/common/libpEpAdapter.linux.Dockerfile @@ -19,5 +19,5 @@ ARG LIBPEPADAPTER_VERSION ARG CURRENT_DISTRO ### Build libpEpAdapter -RUN sh ./scripts/${CURRENT_DISTRO}/build_libpEpAdapter.sh && \ +RUN sh ./scripts/common/build_libpEpAdapter.sh && \ rm -rf ${BUILDROOT}/* diff --git a/scripts/debian10/Makefile b/scripts/debian10/Makefile deleted file mode 100644 index 6d59421..0000000 --- a/scripts/debian10/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -include ../../DEPENDENCIES -export -PEPENGINE_VERSION=${pEpEngine} -CURRENT_DISTRO=$(shell basename $(shell pwd)) -IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-libpepadapter -DOCKERFILE=libpEpAdapter.$(CURRENT_DISTRO).Dockerfile -IS_TAGGED=${TAGGED_BUILD} -ifeq ($(IS_TAGGED), true) -# $CI_COMMIT_TAG is a predefined environment variable from Gitlab - LIBPEPADAPTER_VERSION=${CI_COMMIT_TAG} -else - LIBPEPADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD) -endif -all: - -docker pull $(IMAGE_NAME):latest - cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \ - --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \ - --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \ - --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \ - --cache-from $(IMAGE_NAME):latest \ - --tag=$(IMAGE_NAME):$(LIBPEPADAPTER_VERSION) \ - --tag=$(IMAGE_NAME):${LIBPEPADAPTER_VERSION}_engine-${PEPENGINE_VERSION} \ - --tag=$(IMAGE_NAME):latest \ - -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) . - docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION} - docker push $(IMAGE_NAME):${LIBPEPADAPTER_VERSION}_engine-${PEPENGINE_VERSION} - docker push $(IMAGE_NAME):latest diff --git a/scripts/debian10/build_libpEpAdapter.sh b/scripts/debian10/build_libpEpAdapter.sh deleted file mode 100755 index 1f4a6c5..0000000 --- a/scripts/debian10/build_libpEpAdapter.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -set -exo - -echo "ENGINE_LIB_PATH=${INSTPREFIX}/lib" >> local.conf -echo "ENGINE_INC_PATH=${INSTPREFIX}/include" >> local.conf - -make -make install PREFIX="${INSTPREFIX}"