Introduce C++ into project together with conan dependencies (918) (#985)

Author: intuibase

.ci/Jenkinsfile

@@ -31,7 +31,6 @@ pipeline {
     choice(name: 'AGENT_LOG_LEVEL', choices: LOG_LEVELS, description: "Agent's log level")
     choice(name: 'TESTS_LOG_LEVEL', choices: LOG_LEVELS, description: "Tests' log level")
     booleanParam(name: 'INCLUDE_TESTING', defaultValue: true, description: 'Should the testing stages be included?')
-    booleanParam(name: 'ADD_LIBUNWIND_DEPENDENCY', defaultValue: false, description: 'Should Alpine (.apk) build have dependency on libunwind (to be able to log C call stack on crash)?')
   }
   stages {
     stage('Initializing'){
@@ -84,7 +83,7 @@ pipeline {
             }
           }
         }
-        stage('Build binaries and basic testing') {
+        stage('Build binaries') {
           options { skipDefaultCheckout() }
           when {
             beforeAgent true
@@ -94,50 +93,116 @@ pipeline {
           matrix {
             agent { label 'ubuntu-18.04 && immutable' }
             axes {
+              axis {
+                name 'BUILD_ARCHITECTURE'
+                values 'linux-x86-64', 'linuxmusl-x86-64'
+              }
+            }
+
+            stages {
+              stage('Build PHP extension') {
+                steps {
+                  initWorkspace(context: "Build-${BUILD_ARCHITECTURE}") {
+                    sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} make -f .ci/Makefile build", label: 'build'
+                  }
+                  withGithubNotify(context: "Build-${BUILD_ARCHITECTURE}") {
+                    dir("${BASE_DIR}"){
+                      stash includes: "agent/native/_build/${BUILD_ARCHITECTURE}-release/ext/elastic_apm-*", name: "built-extensions-${BUILD_ARCHITECTURE}"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('PHP tests of extension') {
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression { return env.ONLY_DOCS == "false" }
+          }
+          failFast false
+          matrix {
+            agent { label 'ubuntu-18.04 && immutable' }
+            axes {
+              axis {
+                name 'BUILD_ARCHITECTURE'
+                values 'linux-x86-64', 'linuxmusl-x86-64'
+              }
               axis {
                 name 'PHP_VERSION'
                 // Make sure list of PHP versions supported by the Elastic APM PHP Agent is in sync.
                 // See the comment in .ci/shared.sh
                 values '7.2', '7.3', '7.4', '8.0', '8.1', '8.2'
               }
-              axis {
-                name 'DOCKERFILE'
-                values 'Dockerfile', 'Dockerfile.alpine'
-              }
             }
+
             stages {
-              stage('Build PHP extension') {
+              stage('Execute phpt tests') {
+                when {
+                  beforeAgent true
+                  expression { return params.INCLUDE_TESTING }
+                }
                 steps {
-                  echo "params.INCLUDE_TESTING: ${params.INCLUDE_TESTING}"
-                  echo "params.ADD_LIBUNWIND_DEPENDENCY: ${params.ADD_LIBUNWIND_DEPENDENCY}"
-                  initWorkspace(context: "Build-${PHP_VERSION}") {
+                  initWorkspace(context: "PHPT-${PHP_VERSION}", tab: "tests") {
                     // When running in the CI with multiple parallel stages
                     // the access could be considered as a DDOS attack.
-                    retryWithSleep(retries: 3, seconds: 5, backoff: true) {
-                        sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} ADD_LIBUNWIND_DEPENDENCY=${params.ADD_LIBUNWIND_DEPENDENCY} make -f .ci/Makefile prepare", label: 'prepare docker image'
+                    retryWithSleep(retries: 3, seconds: 45, backoff: true) {
+                        sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} PHP_VERSION=${PHP_VERSION} make -f .ci/Makefile prepare", label: 'prepare docker image'
                     }
-                    sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile build", label: 'build'
+                    unstash "built-extensions-${BUILD_ARCHITECTURE}"
+                    sh script: "BUILD_ARCHITECTURE=${BUILD_ARCHITECTURE} PHP_VERSION=${PHP_VERSION} make -f .ci/Makefile run-phpt-tests", label: 'run-phpt-tests'
                   }
                 }
               }
+            }
+          }
+        }
+        stage('Static analysis and tests') {
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression { return env.ONLY_DOCS == "false" }
+          }
+          failFast false
+          matrix {
+            agent { label 'ubuntu-18.04 && immutable' }
+            axes {
+              axis {
+                name 'PHP_VERSION'
+                // Make sure list of PHP versions supported by the Elastic APM PHP Agent is in sync.
+                // See the comment in .ci/shared.sh
+                values '7.2', '7.3', '7.4', '8.0', '8.1', '8.2'
+              } 
+              axis {
+                name 'DOCKERFILE'
+                values 'Dockerfile', 'Dockerfile.alpine'
+              }
+            }
+            stages {
               stage('Static analysis and unit tests') {
                 when {
                   beforeAgent true
                   expression { return params.INCLUDE_TESTING }
                 }
                 steps {
+                  echo "params.INCLUDE_TESTING: ${params.INCLUDE_TESTING}"
                   echo "params.AGENT_LOG_LEVEL: ${params.AGENT_LOG_LEVEL}"
                   echo "params.AGENT_LOG_LEVEL == null: " + (params.AGENT_LOG_LEVEL == null)
                   echo "params.TESTS_LOG_LEVEL: ${params.TESTS_LOG_LEVEL}"
                   echo "params.TESTS_LOG_LEVEL == null: " + (params.TESTS_LOG_LEVEL == null)
                   echo "addEnvVarsFromParams([]): " + addEnvVarsFromParams([])
-                  withGithubNotify(context: "Static-Check-Unit-Tests-${PHP_VERSION}", tab: 'tests') {
+
+                  initWorkspace(context: "Static-Check-Unit-Tests-${PHP_VERSION}", tab: "tests") {
                     withEnv(addEnvVarsFromParams([])) {
                       echo "env.ELASTIC_APM_LOG_LEVEL: ${env.ELASTIC_APM_LOG_LEVEL}"
                       echo "env.TESTS_LOG_LEVEL: ${env.TESTS_LOG_LEVEL}"
-                      dir("${BASE_DIR}"){
-                        sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile static-check-unit-test", label: 'static-check-unit-test'
+
+                      retryWithSleep(retries: 3, seconds: 45, backoff: true) {
+                          sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile prepare", label: 'prepare docker image'
                       }
+
+                      sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} make -f .ci/Makefile static-check-unit-test", label: 'static-check-unit-test'
                     }
                   }
                 }
@@ -147,16 +212,6 @@ pipeline {
                   }
                 }
               }
-              stage('Build parts for packages') {
-                steps {
-                  withGithubNotify(context: "Generate-For-Package-${PHP_VERSION}") {
-                    dir("${BASE_DIR}"){
-                      sh script: "PHP_VERSION=${PHP_VERSION} DOCKERFILE=${DOCKERFILE} ADD_LIBUNWIND_DEPENDENCY=${params.ADD_LIBUNWIND_DEPENDENCY} make -f .ci/Makefile generate-for-package", label: 'generate-for-package'
-                      stash includes: 'src/ext/modules/*.so', name: "generate-for-package-${PHP_VERSION}-${DOCKERFILE}"
-                    }
-                  }
-                }
-              }
             }
           }
         }
@@ -374,10 +429,9 @@ pipeline {
 */
 def packageGeneration(def args = [:]) {
     def local = args.get('local', false)
-    args.versions.each { version ->
-      unstash "generate-for-package-${version}-Dockerfile"
-      unstash "generate-for-package-${version}-Dockerfile.alpine"
-    }
+    unstash "built-extensions-linux-x86-64"
+    unstash "built-extensions-linuxmusl-x86-64"
+
     if (local) {
       // VERSION=1.0.0 is needed to override the current version.
       // current version is >1.0.0, and this is the way we can
@@ -387,7 +441,8 @@ def packageGeneration(def args = [:]) {
       sh script: "mv build/packages build/local", label: 'prepare-local-upgrade-agent'
     } else {
       // Archive the so files to be downloaded if possible.
-      archiveArtifacts(allowEmptyArchive: true, artifacts: 'src/ext/modules/*.so')
+      archiveArtifacts(allowEmptyArchive: true, artifacts: 'agent/native/_build/*-release/ext/elastic_apm-*')
+
       sh script: "make -C packaging package", label: 'package'
       sh script: "make -C packaging info", label: 'package info'
       // checksum files are regenerated by the signing component in the internal-ci instance.
@@ -419,8 +474,10 @@ def packageWorkspace(def args = [:], Closure body){
       unstash (args.shouldUseSignedBinaries ? env.SIGNED_ARTIFACTS : 'package')
       // When running in the CI sometimes the docker build might fail for
       // some environmental issues, let's retry
-      retryWithSleep(retries: 3, seconds: 5, backoff: true) {
-        sh script: "PHP_VERSION=${PHP_VERSION} make -C packaging ${args.prepareGoal}", label: "${args.prepareGoal} for ${PHP_VERSION}"
+      retryWithSleep(retries: 3, seconds: 45, backoff: true) {
+        if (args.prepareGoal != null) {
+          sh script: "PHP_VERSION=${PHP_VERSION} make -C packaging ${args.prepareGoal}", label: "${args.prepareGoal} for ${PHP_VERSION}"
+        }
       }
       body()
     }
@@ -546,7 +603,6 @@ def addEnvVarsFromParams(def withEnvList) {
     if (params.TESTS_LOG_LEVEL != null && !params.TESTS_LOG_LEVEL.equals(LOG_LEVEL_NOT_SET)) {
       withEnvList.add('ELASTIC_APM_PHP_TESTS_LOG_LEVEL=' + params.TESTS_LOG_LEVEL)
     }
-    withEnvList.add('ADD_LIBUNWIND_DEPENDENCY=' + params.ADD_LIBUNWIND_DEPENDENCY)
     return withEnvList
 }
 
@@ -574,7 +630,6 @@ def buildLabel(def args = [:]) {
 def lifecycleTesting(def args = [:]) {
   runTestingCommand(
     args + [
-      prepareGoal: "prepare-${args.linuxPackageType}",
       testingCommand: "make -C packaging ${args.linuxPackageType}-lifecycle-testing"
     ]
   )
@@ -583,7 +638,6 @@ def lifecycleTesting(def args = [:]) {
 def lifecycleTestingOnProdServerKind(def args = [:]) {
   runTestingCommand(
     args + [
-      prepareGoal: "prepare-${args.linuxPackageType}-${args.prodServerKind}",
       testingCommand: "make -C packaging ${args.linuxPackageType}-lifecycle-testing-in-${args.prodServerKind}",
     ]
   )
@@ -592,7 +646,6 @@ def lifecycleTestingOnProdServerKind(def args = [:]) {
 def phpUpgradeTesting(def args = [:]) {
   runTestingCommand(
     args + [
-      prepareGoal: "prepare-${args.linuxPackageType}",
       testingCommand: "PHP_VERSION=${args.phpVersion} make -C packaging ${args.linuxPackageType}-php-upgrade-testing"
     ]
   )

.ci/Makefile

@@ -1,3 +1,10 @@
+ifeq ($(DOCKERFILE),)
+	ifeq ($(BUILD_ARCHITECTURE), linuxmusl-x86-64)
+		DOCKERFILE := Dockerfile.alpine
+	else
+		DOCKERFILE := Dockerfile
+	endif
+endif
 
 SHELL=/bin/bash -o pipefail
 MAKEFLAGS += --no-print-directory
@@ -6,15 +13,20 @@ PHP_VERSION ?= 7.2
 DOCKERFILE ?= Dockerfile
 LOOPS ?= 50
 SUFFIX :=
-ADD_LIBUNWIND_DEPENDENCY_BUILD_ARG_OPT:=
 ifeq ($(DOCKERFILE), Dockerfile.alpine)
     ## This is only required to tag the docker images used for building/testing this project
     SUFFIX := -alpine
-    ADD_LIBUNWIND_DEPENDENCY_BUILD_ARG_OPT:=--build-arg ADD_LIBUNWIND_DEPENDENCY=${ADD_LIBUNWIND_DEPENDENCY}
+endif
+
+CONAN_USER_HOME:=/tmp/conan_user_home
+MAP_CONAN_HOME_TO_DOCKER_HOST_CMD_LINE_ARG:=
+ifdef MAP_CONAN_HOME_TO_DOCKER_HOST
+    MAP_CONAN_HOME_TO_DOCKER_HOST_CMD_LINE_ARG:=-v "${MAP_CONAN_HOME_TO_DOCKER_HOST}:$(CONAN_USER_HOME)"
 endif
 
 CURRENT_UID := $(shell id -u)
 CURRENT_GID := $(shell id -g)
+CURRENT_USER_HOME := $(shell echo ~)
 
 .PHONY: help
 .DEFAULT_GOAL := help
@@ -26,19 +38,24 @@ prepare:  ## Build docker image for building and testing the project
 	@echo "::group::$@"  # Helping to group logs in GitHub actions
 	docker build \
 		--build-arg PHP_VERSION=${PHP_VERSION} \
-		${ADD_LIBUNWIND_DEPENDENCY_BUILD_ARG_OPT} \
 		--tag $(IMAGE):${PHP_VERSION}$(SUFFIX) \
 		-f ${DOCKERFILE} .
 	@echo "::endgroup::"
 
 .PHONY: build
-build: prepare  ## Build the project
+build:
 	@echo "::group::$@"  # Helping to group logs in GitHub actions
 	# docker as the current user
 	docker run --rm -t \
 		-u $(CURRENT_UID):$(CURRENT_GID) \
-		-v $(PWD):/app \
-		$(IMAGE):${PHP_VERSION}$(SUFFIX)
+		-v $(PWD):/source \
+		${MAP_CONAN_HOME_TO_DOCKER_HOST_CMD_LINE_ARG} \
+		-w /source/agent/native \
+		-e CONAN_USER_HOME=$(CONAN_USER_HOME) \
+		elasticobservability/apm-agent-php-dev:native-build-gcc-12.2.0-$(BUILD_ARCHITECTURE)-0.0.2 \
+		sh -c "cmake --preset $(BUILD_ARCHITECTURE)-release \
+				&& cmake --build --preset $(BUILD_ARCHITECTURE)-release \
+				&& /source/agent/native/_build/$(BUILD_ARCHITECTURE)-release/ext/unit_tests/unit_tests"
 	@echo "::endgroup::"
 
 .PHONY: static-check-unit-test
@@ -52,15 +69,16 @@ static-check-unit-test: prepare  ## Test the static check and unit tests
 		/app/.ci/static-check-unit-test.sh
 	@echo "::endgroup::"
 
-.PHONY: generate-for-package
-generate-for-package: prepare  ## Generate the agent extension for the package
+.PHONY: run-phpt-tests
+run-phpt-tests: prepare  ## Runs phpt tests
 	@echo "::group::$@"  # Helping to group logs in GitHub actions
 	# docker as the current user
 	docker run --rm -t \
+		-e CHOWN_RESULTS_UID=$(CURRENT_UID) -e CHOWN_RESULTS_GID=$(CURRENT_GID) \
+		-e BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \
 		-v $(PWD):/app \
-		-u $(CURRENT_UID):$(CURRENT_GID) \
 		$(IMAGE):${PHP_VERSION}$(SUFFIX) \
-		/app/.ci/generate-for-package.sh
+		/app/.ci/run-phpt-tests.sh
 	@echo "::endgroup::"
 
 .PHONY: interactive
@@ -75,6 +93,7 @@ component-test: prepare  ## Run component-test
 	@echo "::group::$@"  # Helping to group logs in GitHub actions
 	# docker as root to install the extension
 	docker run -t --rm \
+		-e BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \
 		-v $(PWD):/app \
 		$(IMAGE):${PHP_VERSION}$(SUFFIX) \
 		sh -c '/app/.ci/component-test.sh'

.ci/bump_version.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -xe
 
-sed -i.bck "s#\(VERSION = \).*;#\1'${VERSION}';#g" src/ElasticApm/ElasticApm.php
-sed -i.bck "s#\(PHP_ELASTIC_APM_VERSION\).*#\1 \"${VERSION}\"#g" src/ext/elastic_apm_version.h
+sed -i.bck "s#\(VERSION = \).*;#\1'${VERSION}';#g" agent/php/ElasticApm/ElasticApm.php
+sed -i.bck "s#\(PHP_ELASTIC_APM_VERSION\).*#\1 \"${VERSION}\"#g" agent/native/ext/elastic_apm_version.h
 
-git add src/ElasticApm/ElasticApm.php src/ext/elastic_apm_version.h
+git add agent/php/ElasticApm/ElasticApm.php agent/native/ext/elastic_apm_version.h

.ci/component-test.sh

@@ -4,12 +4,32 @@ set -xe -o pipefail
 # Disable Elastic APM for any process outside the component tests to prevent noise in the logs
 export ELASTIC_APM_ENABLED=false
 
+APP_FOLDER=/app
+PHP_API=$(php -i | grep -i 'PHP API' | sed -e 's#.* =>##g' | awk '{print $1}')
+PHP_EXECUTABLE=$(which php)
+
+echo "BUILD ARCHITECTURE: $BUILD_ARCHITECTURE"
+
+if [ -z "${BUILD_ARCHITECTURE}" ]
+then
+      echo "\$BUILD_ARCHITECTURE is not specified, assuming linux-x86-64"
+      BUILD_ARCHITECTURE=linux-x86-64
+fi
+echo "BUILD ARCHITECTURE: $BUILD_ARCHITECTURE"
+
+
+
+AGENT_EXTENSION_DIR=$APP_FOLDER/agent/native/_build/$BUILD_ARCHITECTURE-release/ext
+AGENT_EXTENSION=$AGENT_EXTENSION_DIR/elastic_apm-$PHP_API.so
+
+mkdir -p $APP_FOLDER/build
+
+
 PHP_INI=/usr/local/etc/php/php.ini
-make install
-echo 'extension=elastic_apm.so' > ${PHP_INI}
-echo 'elastic_apm.bootstrap_php_part_file=/app/src/bootstrap_php_part.php' >> ${PHP_INI}
+echo "extension=${AGENT_EXTENSION}" > ${PHP_INI}
+echo "elastic_apm.bootstrap_php_part_file=${APP_FOLDER}/agent/php/bootstrap_php_part.php" >> ${PHP_INI}
 php -m
-cd /app
+cd "${APP_FOLDER}"
 
 # Install 3rd party dependencies
 composer install
@@ -28,5 +48,5 @@ fi
 
 # Run component tests
 mkdir -p ./build/
-composer run-script run_component_tests 2>&1 | tee /app/build/run_component_tests_output.txt
+composer run-script run_component_tests 2>&1 | tee ${APP_FOLDER}/build/run_component_tests_output.txt