CI: build CentOS/RedHat 8 RPM packages

This commit cleans up the existing CI scripts a bit while adding jobs to
build RPM packages. The directory structure allows for easily adding
more
package targets.

RPM outputs will only be uploaded on tagged commits.

.gitlab-ci-files/common-prepare.yml

@@ -8,11 +8,22 @@
   - 'which rsync || ( sudo apt-get update -y && sudo apt-get install rsync -y )'
   - 'which make || ( sudo apt-get update -y && sudo apt-get install make -y )'
 
+.add_ssh_keys: &add_ssh_keys
+  # Add the SSH key (stored in the SSH_PRIVATE_KEY variable) to the agent.
+  - eval $(ssh-agent -s)
+  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+
+.verify_ssh_host_keys: &verify_ssh_host_keys
+  # Verify SSH host keys
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - echo "${SSH_KNOWN_HOSTS}" >> ~/.ssh/known_hosts
+  - chmod 644 ~/.ssh/known_hosts
+
 .standard_job:
   tags: [kvm]
   before_script:
     - *ensure_docker
-    - *ensure_rsync
 
 .make_in_docker:
   extends: .standard_job
@@ -20,3 +31,21 @@
     - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
     - cd scripts/${CI_DISTRO_TARGET}
     - make ${MAKE_TARGET}
+
+.upload_pkg:
+  extends: .standard_job
+  before_script:
+    - *ensure_rsync
+    - *add_ssh_keys
+    - *verify_ssh_host_keys
+  script:
+    - docker login -u ${DOCKER_REGISTRY_USER} -p ${DOCKER_REGISTRY_PASS} ${DOCKER_REGISTRY_HOST}
+    - cd scripts/${CI_DISTRO_TARGET}
+    - make ${MAKE_TARGET}
+    - pwd
+    - cd out
+    - time rsync -avP -e "ssh -p ${PKG_HOST_SSH_PORT}" depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpJNIAdapter/SHA256SUMS || true
+    - if sha256sum --ignore-missing --check SHA256SUMS ; then echo "Package already exists... Exiting..." && exit ; else true ; fi
+    - sha256sum ./*.rpm | tee --append SHA256SUMS
+    - cat ./SHA256SUMS
+    - time rsync -azvP -e "ssh -p ${PKG_HOST_SSH_PORT}" --rsync-path="mkdir -p /home/depot/files/pkgs/RHEL/8/pEpJNIAdapter && rsync" ./*.rpm SHA256SUMS depot@${PKG_HOST}:files/pkgs/RHEL/8/pEpJNIAdapter/

.gitlab-ci.yml

@@ -3,6 +3,8 @@ include:
 
 stages:
   - build
+  - packages
+
 
 # Debian
 
@@ -10,6 +12,7 @@ debian10:build:
   extends: .make_in_docker
   stage: build
   variables:
+    MAKE_TARGET: "build"
     CI_DISTRO_TARGET: "debian10"
     DEBIAN_FRONTEND: "noninteractive"
   rules:
@@ -19,6 +22,7 @@ debian10:tagged-build:
   extends: .make_in_docker
   stage: build
   variables:
+    MAKE_TARGET: "build"
     CI_DISTRO_TARGET: "debian10"
     DEBIAN_FRONTEND: "noninteractive"
     TAGGED_BUILD: "true"
@@ -29,8 +33,8 @@ debian10-java8:build:
   extends: .make_in_docker
   stage: build
   variables:
-    CI_DISTRO_TARGET: "debian10"
     MAKE_TARGET: "java8"
+    CI_DISTRO_TARGET: "debian10"
     DEBIAN_FRONTEND: "noninteractive"
   rules:
     - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
@@ -45,3 +49,46 @@ debian10-java8:tagged-build:
     TAGGED_BUILD: "true"
   rules:
     - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+# CentOS
+
+centos8:build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    MAKE_TARGET: "build"
+    CI_DISTRO_TARGET: "centos8"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+centos8:tagged-build:
+  extends: .make_in_docker
+  stage: build
+  variables:
+    MAKE_TARGET: "build"
+    CI_DISTRO_TARGET: "centos8"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+centos8:rpm:
+  extends: .make_in_docker
+  stage: packages
+  needs: ["centos8:build"]
+  variables:
+    MAKE_TARGET: "rpm"
+    CI_DISTRO_TARGET: "centos8"
+  rules:
+    - if: '$CI_COMMIT_TAG  !~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'
+
+
+centos8:rpm:tagged-build:
+  extends: .upload_pkg
+  stage: packages
+  needs: ["centos8:tagged-build"]
+  variables:
+    MAKE_TARGET: "rpm"
+    CI_DISTRO_TARGET: "centos8"
+    TAGGED_BUILD: "true"
+  rules:
+    - if: '$CI_COMMIT_TAG  =~ /^Release_[0-9]+\.[0-9]+\.[0-9]+$/'

scripts/centos8/JNIAdapter.centos8.Dockerfile

@@ -0,0 +1,30 @@
+ARG DOCKER_REGISTRY_HOST
+ARG CURRENT_DISTRO
+ARG LIBPEPADAPTER_VERSION
+FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-libpepadapter:${LIBPEPADAPTER_VERSION}
+
+ENV BUILDROOT /build
+ENV INSTPREFIX /install
+ENV OUTDIR /out
+
+ARG JNIADAPTER_VERSION
+
+## Install system dependencies
+USER root
+RUN yum -y install time java-1.8.0-openjdk java-1.8.0-openjdk-devel && \
+    yum clean all
+
+## Build and install pEpJNIAdapter
+### Setup working directory
+RUN mkdir ${BUILDROOT}/pEpJNIAdapter
+COPY . ${BUILDROOT}/pEpJNIAdapter
+RUN chown -R pep-builder:pep-builder ${BUILDROOT}/pEpJNIAdapter
+USER pep-builder
+WORKDIR ${BUILDROOT}/pEpJNIAdapter
+
+RUN sh ./scripts/common/build_pEpJNIAdapter.sh && \
+    install -m 644 -t ${INSTPREFIX}/lib dist/libpEpJNI.a && \
+    install -m 755 -t ${INSTPREFIX}/lib dist/libpEpJNI.so && \
+    install -m 644 -t ${INSTPREFIX}/lib dist/pEp.jar && \
+    echo "${pepjni_ver}">${INSTPREFIX}/pEp_JNI.ver && \
+    rm -rf ${BUILDROOT}/*

scripts/centos8/Makefile

@@ -0,0 +1,79 @@
+include ../../DEPENDENCIES
+export
+LIBPEPADAPTER_VERSION=${libpEpAdapter}
+PEPENGINE_VERSION=${pEpEngine}
+SEQUOIA_VERSION=${sequoia}
+CURRENT_DISTRO=${CI_DISTRO_TARGET}
+IMAGE_NAME=${DOCKER_REGISTRY_HOST}/pep-$(CURRENT_DISTRO)-jniadapter
+DOCKERFILE=JNIAdapter.$(CURRENT_DISTRO).Dockerfile
+PKG_BUILD_IMAGE=${DOCKER_REGISTRY_HOST}/fpm-$(CURRENT_DISTRO)
+PKG_INSTALL_PATH=/opt/pEp
+PKG_NAME=pEpJNIAdapter
+PKG_DESCRIPTION="p≡p JNI Adapter Binary RPM Package"
+PKG_DEPENDS="libpEpAdapter --depends java-1.8.0-openjdk"
+PKG_INSTALL_PATH_STRING="/ /package/lib=${PKG_INSTALL_PATH}"
+IS_TAGGED=${TAGGED_BUILD}
+ifeq ($(IS_TAGGED), true)
+# $CI_COMMIT_TAG is a predefined environment variable from Gitlab
+	PEPJNIADAPTER_VERSION=${CI_COMMIT_TAG}
+else
+	PEPJNIADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD)
+endif
+all: build
+
+build:
+	-docker pull $(IMAGE_NAME):latest
+	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		      --build-arg PEPJNIADAPTER_VERSION=$(PEPJNIADAPTER_VERSION) \
+		      --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
+		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
+		      --build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \
+		      --cache-from $(IMAGE_NAME):latest \
+		      --tag=$(IMAGE_NAME):$(PEPJNIADAPTER_VERSION) \
+		      --tag=$(IMAGE_NAME):$(PEPJNIADAPTER_VERSION)_libpepadapter-$(LIBPEPADAPTER_VERSION)_engine-$(PEPENGINE_VERSION) \
+		      --tag=$(IMAGE_NAME):latest \
+		      -f scripts/${CURRENT_DISTRO}/$(DOCKERFILE) .
+	docker push $(IMAGE_NAME):$(PEPJNIADAPTER_VERSION)
+	docker push $(IMAGE_NAME):$(PEPJNIADAPTER_VERSION)_libpepadapter-$(LIBPEPADAPTER_VERSION)_engine-$(PEPENGINE_VERSION)
+	docker push $(IMAGE_NAME):latest
+
+java8:
+	-docker pull $(IMAGE_NAME)-java8:latest
+	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		      --build-arg PEPJNIADAPTER_VERSION=$(PEPJNIADAPTER_VERSION) \
+		      --build-arg LIBPEPADAPTER_VERSION=$(LIBPEPADAPTER_VERSION) \
+		      --build-arg PEPENGINE_VERSION=$(PEPENGINE_VERSION) \
+		      --build-arg SEQUOIA_VERSION=$(SEQUOIA_VERSION) \
+		      --cache-from $(IMAGE_NAME):latest \
+		      --tag=$(IMAGE_NAME)-java8:$(PEPJNIADAPTER_VERSION) \
+		      --tag=$(IMAGE_NAME)-java8:$(PEPJNIADAPTER_VERSION)_libpepadapter-$(LIBPEPADAPTER_VERSION)_engine-$(PEPENGINE_VERSION) \
+		      --tag=$(IMAGE_NAME)-java8:latest \
+		      -f scripts/${CURRENT_DISTRO}/java8.$(DOCKERFILE) .
+	docker push $(IMAGE_NAME)-java8:$(PEPJNIADAPTER_VERSION)
+	docker push $(IMAGE_NAME)-java8:$(PEPJNIADAPTER_VERSION)_libpepadapter-$(LIBPEPADAPTER_VERSION)_engine-$(PEPENGINE_VERSION)
+	docker push $(IMAGE_NAME)-java8:latest
+rpm:
+	-docker pull $(PKG_BUILD_IMAGE)-jniadapter:latest
+	@docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
+		--build-arg PEPJNIADAPTER_VERSION=$(PEPJNIADAPTER_VERSION) \
+		--build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \
+		--build-arg PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
+		--cache-from $(PKG_BUILD_IMAGE)-jniadapter:latest \
+		--tag=$(PKG_BUILD_IMAGE)-jniadapter:$(PEPJNIADAPTER_VERSION) \
+		--tag=$(PKG_BUILD_IMAGE)-jniadapter:latest \
+		packages/rpm
+	@docker push $(PKG_BUILD_IMAGE)-jniadapter:$(PEPJNIADAPTER_VERSION)
+	@docker push $(PKG_BUILD_IMAGE)-jniadapter:latest
+	@docker run -e PEPJNIADAPTER_VERSION=$(PEPJNIADAPTER_VERSION) \
+		-e PKG_VERSION=$(LIBPEPADAPTER_VERSION) \
+		-e PKG_INSTALL_PATH=$(PKG_INSTALL_PATH) \
+		-e PKG_NAME=$(PKG_NAME) \
+		-e PKG_DESCRIPTION=$(PKG_DESCRIPTION) \
+		-e PKG_DEPENDS=$(PKG_DEPENDS) \
+		-e PKG_INSTALL_PATH_STRING=$(PKG_INSTALL_PATH_STRING) \
+		--rm -v $(shell pwd)/packages/rpm/create-jniadapter-rpm.sh:/usr/bin/create-rpm.sh:ro  \
+		-v $(shell pwd)/out:/out \
+		-w / $(PKG_BUILD_IMAGE)-jniadapter:${PEPJNIADAPTER_VERSION} \
+		/usr/bin/create-rpm.sh

scripts/centos8/packages/rpm/Dockerfile

@@ -0,0 +1,18 @@
+ARG DOCKER_REGISTRY_HOST
+ARG CURRENT_DISTRO
+ARG PEPJNIADAPTER_VERSION
+FROM ${DOCKER_REGISTRY_HOST}/pep-${CURRENT_DISTRO}-jniadapter:${PEPJNIADAPTER_VERSION} AS pEpBuild
+
+FROM ${DOCKER_REGISTRY_HOST}/fpm-${CURRENT_DISTRO}:latest
+# whatever is required for building should be installed in this image; just like BuildRequires: for RPM specs
+RUN yum -y install readline-devel epel-release && \
+    yum -y install patchelf chrpath
+
+ARG PEP_MACHINE_DIR
+COPY --from=pEpBuild /install /source
+
+COPY install.sh /usr/local/bin/install.sh
+
+ENV INSTPREFIX /source
+
+RUN /usr/local/bin/install.sh

scripts/centos8/packages/rpm/create-jniadapter-rpm.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -ex
+# we should always set proper ownership before exiting, otherwise
+# the created packages will have root:root ownership and we'll be unable
+# to delete them from our host.
+trap 'chown -R --reference /usr/bin/create-rpm.sh /out/' EXIT
+
+# the source directory is mounted read-only to prevent issues where the build
+# could alter the source; we should copy it somewhere inside the container
+cd /source/out
+ls -alh
+tree
+INSTALL_TOP=/package
+mkdir -p ${INSTALL_TOP}/lib
+cp -ar lib/* ${INSTALL_TOP}/lib/.
+
+
+cd /out
+
+#this would be the no-signature command line
+fpm -t rpm -s dir \
+	-n ${PKG_NAME} \
+	--version ${PKG_VERSION} \
+	--description "${PKG_DESCRIPTION}" \
+	--depends ${PKG_DEPENDS} \
+	-C ${PKG_INSTALL_PATH_STRING}

scripts/centos8/packages/rpm/install.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -exuo pipefail
+
+# ===========================
+# Distro
+# ===========================
+
+echo 7 >"${INSTPREFIX}/D_REVISION"
+
+D_REV=$(cat ${INSTPREFIX}/D_REVISION)
+D=""
+
+D=${INSTPREFIX}/out
+
+mkdir -p ${INSTPREFIX}/out
+rm -rf ${INSTPREFIX}/out/*
+mkdir -p "$D"/{bin,ld,lib/pEp,share/pEp,include/pEp}
+
+tree ${INSTPREFIX}
+
+# pEpJNIAdapter
+cp -a ${INSTPREFIX}/lib/libpEpJNI.a "$D"/lib
+cp -a ${INSTPREFIX}/lib/libpEpJNI.so "$D"/lib
+cp -a ${INSTPREFIX}/lib/pEp.jar "$D"/lib
+
+# versions
+cp -a ${INSTPREFIX}/*.ver "$D"
+
+find "$D"/lib -maxdepth 1 -type f -print -exec patchelf --set-rpath '$ORIGIN/pEp:$ORIGIN' {} \;
+find "$D"/lib/pEp         -type f -print -exec patchelf --set-rpath '$ORIGIN' {} \;
+find "$D"/bin -type f -print -exec patchelf --set-rpath '$ORIGIN/../lib/pEp:$ORIGIN/../lib' {} \;
+
+ls -lh "$D"/*
+du -sch "$D"

scripts/debian10/Makefile

@@ -13,7 +13,9 @@ ifeq ($(IS_TAGGED), true)
 else
 	PEPJNIADAPTER_VERSION=$(shell git rev-parse --short=8 HEAD)
 endif
-all:
+all: build
+
+build:
 	-docker pull $(IMAGE_NAME):latest
 	cd ../../ && docker build --build-arg CURRENT_DISTRO=$(CURRENT_DISTRO) \
 		      --build-arg DOCKER_REGISTRY_HOST=${DOCKER_REGISTRY_HOST} \

scripts/debian10/java8.pEpJNIAdapter.debian10.Dockerfile

@@ -49,7 +49,7 @@ ARG PEPJNIADAPTER_VERSION
 ARG CURRENT_DISTRO
 
 ### Build pEpJNIAdapter
-RUN sh ./scripts/${CURRENT_DISTRO}/build_pEpJNIAdapter.sh && \
+RUN sh ./scripts/common/build_pEpJNIAdapter.sh && \
     install -m 644 -t ${INSTPREFIX}/lib dist/libpEpJNI.a && \
     install -m 755 -t ${INSTPREFIX}/lib dist/libpEpJNI.so && \
     install -m 644 -t ${INSTPREFIX}/lib dist/pEp.jar

scripts/debian10/pEpJNIAdapter.debian10.Dockerfile

@@ -43,7 +43,7 @@ ARG PEPJNIADAPTER_VERSION
 ARG CURRENT_DISTRO
 
 ### Build pEpJNIAdapter
-RUN sh ./scripts/${CURRENT_DISTRO}/build_pEpJNIAdapter.sh && \
+RUN sh ./scripts/common/build_pEpJNIAdapter.sh && \
     install -m 644 -t ${INSTPREFIX}/lib dist/libpEpJNI.a && \
     install -m 755 -t ${INSTPREFIX}/lib dist/libpEpJNI.so && \
     install -m 644 -t ${INSTPREFIX}/lib dist/pEp.jar