From 276ec8a1006423a6390af01f7ee719d545b97a5c Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Tue, 6 Feb 2024 13:05:59 +1100 Subject: [PATCH 01/17] Updated the CI scripts to follow our dev processes better --- .drone.jsonnet | 37 +++++--------- Scripts/drone-static-upload.sh | 93 ++++++++++++++++++++-------------- Scripts/drone-upload-exists.sh | 33 ++++++++++++ 3 files changed, 102 insertions(+), 61 deletions(-) create mode 100755 Scripts/drone-upload-exists.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index b2546206e..ca7dd8c5b 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -71,12 +71,13 @@ local update_cocoapods_cache = { [ - // Unit tests + // Unit tests (PRs only) { kind: 'pipeline', type: 'exec', name: 'Unit Tests', platform: { os: 'darwin', arch: 'amd64' }, + trigger: { event: { exclude: [ 'push' ] } }, steps: [ clone_submodules, load_cocoapods_cache, @@ -91,39 +92,27 @@ local update_cocoapods_cache = { update_cocoapods_cache ], }, - // Simulator build + // Validate build artifact was created by the direct branch push (PRs only) { kind: 'pipeline', type: 'exec', - name: 'Simulator Build', + name: 'Check Build Artifact Existence', platform: { os: 'darwin', arch: 'amd64' }, - trigger: { event: { exclude: [ 'pull_request' ] } }, + trigger: { event: { exclude: [ 'push' ] } }, steps: [ - clone_submodules, - load_cocoapods_cache, - install_cocoapods, { - name: 'Build', + name: 'Poll for build artifact existence', commands: [ - 'mkdir build', - 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' - ], - }, - update_cocoapods_cache, - { - name: 'Upload artifacts', - environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, - commands: [ - './Scripts/drone-static-upload.sh' + './Scripts/drone-upload-exists.sh' ] - }, - ], + } + ] }, - // AppStore build (generate an archive to be signed later) + // Simulator build (non-PRs only) { kind: 'pipeline', type: 'exec', - name: 'AppStore Build', + name: 'Simulator Build', platform: { os: 'darwin', arch: 'amd64' }, trigger: { event: { exclude: [ 'pull_request' ] } }, steps: [ @@ -134,7 +123,7 @@ local update_cocoapods_cache = { name: 'Build', commands: [ 'mkdir build', - 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -configuration "App Store Release" -sdk iphoneos -archivePath ./build/Session.xcarchive -destination "generic/platform=iOS" -allowProvisioningUpdates CODE_SIGNING_ALLOWED=NO | ./Pods/xcbeautify/xcbeautify --is-ci' + 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' ], }, update_cocoapods_cache, @@ -146,5 +135,5 @@ local update_cocoapods_cache = { ] }, ], - }, + } ] \ No newline at end of file diff --git a/Scripts/drone-static-upload.sh b/Scripts/drone-static-upload.sh index 681e58994..0f2dee2c0 100755 --- a/Scripts/drone-static-upload.sh +++ b/Scripts/drone-static-upload.sh @@ -2,20 +2,29 @@ # Script used with Drone CI to upload build artifacts (because specifying all this in # .drone.jsonnet is too painful). +# +# Passing an argument of 'false' will result in just the upload location being output +# instead of attepting to actually upload set -o errexit -if [ -z "$SSH_KEY" ]; then - echo -e "\n\n\n\e[31;1mUnable to upload artifact: SSH_KEY not set\e[0m" - # Just warn but don't fail, so that this doesn't trigger a build failure for untrusted builds - exit 0 -fi +if [[ -z "$1" || "$0" == "true" ]]; then + if [ -z "$SSH_KEY" ]; then + echo -e "\n\n\n\e[31;1mUnable to upload artifact: SSH_KEY not set\e[0m" + # Just warn but don't fail, so that this doesn't trigger a build failure for untrusted builds + exit 0 + fi + + echo "$SSH_KEY" >ssh_key -echo "$SSH_KEY" >ssh_key + set -o xtrace # Don't start tracing until *after* we write the ssh key -set -o xtrace # Don't start tracing until *after* we write the ssh key + chmod 600 ssh_key -chmod 600 ssh_key + should_upload=true +else + should_upload=$1 +fi # Define the output paths prod_path="build/Session.xcarchive" @@ -25,7 +34,7 @@ sim_path="build/Session_sim.xcarchive/Products/Applications/Session.app" if [ -d $prod_path ]; then suffix="store" target_path=$prod_path -elif [ -d $sim_path ]; then +elif [[ -d $sim_path || "$should_upload" == "false" ]]; then suffix="sim" target_path=$sim_path else @@ -36,42 +45,52 @@ fi if [ -n "$DRONE_TAG" ]; then # For a tag build use something like `session-ios-v1.2.3` base="session-ios-$DRONE_TAG-$suffix" +elif [ "$should_upload" == "false" ]; then + # Instead of the datetime include a wildcard in the base name as it'll differ in the actual build: + # session-ios-.*-2fba13878 + # base="session-ios-.*-${DRONE_COMMIT:0:9}-$suffix" + base="session-ios-.*-2fba13878-$suffix" else # Otherwise build a length name from the datetime and commit hash, such as: - # session-ios-20200522T212342Z-04d7dcc54 + # session-ios-20200522T212342Z-2fba13878 base="session-ios-$(date --date=@$DRONE_BUILD_CREATED +%Y%m%dT%H%M%SZ)-${DRONE_COMMIT:0:9}-$suffix" fi -# Copy over the build products -mkdir -vp "$base" -mkdir -p build -cp -av $target_path "$base" - -# tar dat shiz up yo archive="$base.tar.xz" -tar cJvf "$archive" "$base" - -upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" - -# sftp doesn't have any equivalent to mkdir -p, so we have to split the above up into a chain of -# -mkdir a/, -mkdir a/b/, -mkdir a/b/c/, ... commands. The leading `-` allows the command to fail -# without error. -upload_dirs=(${upload_to//\// }) -put_debug= -mkdirs= -dir_tmp="" -for p in "${upload_dirs[@]}"; do - dir_tmp="$dir_tmp$p/" - mkdirs="$mkdirs --mkdir $dir_tmp" -done +#upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" +upload_to="oxen.rocks/mpretty-cyro/session-ios/feature/groups-rebuild" + +if [ "$should_upload" == "true" ]; then + # Copy over the build products + mkdir -vp "$base" + mkdir -p build + cp -av $target_path "$base" + + # tar dat shiz up yo + tar cJvf "$archive" "$base" + + # sftp doesn't have any equivalent to mkdir -p, so we have to split the above up into a chain of + # -mkdir a/, -mkdir a/b/, -mkdir a/b/c/, ... commands. The leading `-` allows the command to fail + # without error. + upload_dirs=(${upload_to//\// }) + put_debug= + mkdirs= + dir_tmp="" + for p in "${upload_dirs[@]}"; do + dir_tmp="$dir_tmp$p/" + mkdirs="$mkdirs + -mkdir $dir_tmp" + done sftp -i ssh_key -b - -o StrictHostKeyChecking=off drone@oxen.rocks < Date: Tue, 6 Feb 2024 13:07:54 +1100 Subject: [PATCH 02/17] Removed debug code --- Scripts/drone-static-upload.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Scripts/drone-static-upload.sh b/Scripts/drone-static-upload.sh index 0f2dee2c0..a337c14d0 100755 --- a/Scripts/drone-static-upload.sh +++ b/Scripts/drone-static-upload.sh @@ -48,8 +48,7 @@ if [ -n "$DRONE_TAG" ]; then elif [ "$should_upload" == "false" ]; then # Instead of the datetime include a wildcard in the base name as it'll differ in the actual build: # session-ios-.*-2fba13878 - # base="session-ios-.*-${DRONE_COMMIT:0:9}-$suffix" - base="session-ios-.*-2fba13878-$suffix" + base="session-ios-.*-${DRONE_COMMIT:0:9}-$suffix" else # Otherwise build a length name from the datetime and commit hash, such as: # session-ios-20200522T212342Z-2fba13878 @@ -57,8 +56,7 @@ else fi archive="$base.tar.xz" -#upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" -upload_to="oxen.rocks/mpretty-cyro/session-ios/feature/groups-rebuild" +upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" if [ "$should_upload" == "true" ]; then # Copy over the build products From 6e3c48bd5f8e7f386f84a41192143f9157292963 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Tue, 6 Feb 2024 14:10:06 +1100 Subject: [PATCH 03/17] Tweaks to the polling script --- Scripts/drone-upload-exists.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index fe2c59fdf..7fcf30a0a 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -7,10 +7,12 @@ upload_url=$("${current_dir}/drone-static-upload.sh" false) upload_dir="$(dirname "${upload_url}")" target_file_pattern="$(basename "${upload_url}")" +echo "Starting to poll ${upload_dir} every 10s to check for a build matching '${target_file_pattern}'" + # Loop indefinitely the CI can timeout the script if it takes too long while true; do # Need to add the trailing '/' or else we get a '301' response - build_artifacts_html=$(curl -X GET "${upload_dir}/") + build_artifacts_html=$(curl -s "${upload_dir}/") if [ $? != 0 ]; then echo "Failed to retrieve build artifact list" From 18bc37a8a6247b17524ddd9b3b4454516024cad5 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Tue, 6 Feb 2024 14:13:39 +1100 Subject: [PATCH 04/17] Added a timeout for the polling --- Scripts/drone-upload-exists.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index 7fcf30a0a..d61c40099 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -10,6 +10,9 @@ target_file_pattern="$(basename "${upload_url}")" echo "Starting to poll ${upload_dir} every 10s to check for a build matching '${target_file_pattern}'" # Loop indefinitely the CI can timeout the script if it takes too long +total_poll_duration=0 +max_poll_duration=(30 * 60) # Poll for a maximum of 30 mins + while true; do # Need to add the trailing '/' or else we get a '301' response build_artifacts_html=$(curl -s "${upload_dir}/") @@ -32,4 +35,9 @@ while true; do # Sleep for 10 seconds before checking again sleep 10 + total_poll_duration=$((total_poll_duration + 10)) + + if [ $total_poll_duration -gt $max_poll_duration ]; then + echo -e "\n\n\n\n\e[31;1mCould not find existing build artifact after polling for 30 minutes\e[0m\n\n\n" + fi done From 22433ad3580e2b98c97fbd55e5cb28a2612a84ab Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Tue, 6 Feb 2024 15:01:18 +1100 Subject: [PATCH 05/17] Fixed an incorrect env var, added an exit in the poller --- Scripts/drone-static-upload.sh | 5 +++-- Scripts/drone-upload-exists.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Scripts/drone-static-upload.sh b/Scripts/drone-static-upload.sh index a337c14d0..040cde913 100755 --- a/Scripts/drone-static-upload.sh +++ b/Scripts/drone-static-upload.sh @@ -56,9 +56,10 @@ else fi archive="$base.tar.xz" -upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" if [ "$should_upload" == "true" ]; then + upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" + # Copy over the build products mkdir -vp "$base" mkdir -p build @@ -90,5 +91,5 @@ SFTP echo -e "\n\n\n\n\e[32;1mUploaded to https://${upload_to}/${archive}\e[0m\n\n\n" else - echo -e "https://${upload_to}/${archive}" + echo -e "https://oxen.rocks/${DRONE_REPO// /_}/${DRONE_SOURCE_BRANCH// /_}/${archive}" fi \ No newline at end of file diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index d61c40099..2b1d286e0 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -39,5 +39,6 @@ while true; do if [ $total_poll_duration -gt $max_poll_duration ]; then echo -e "\n\n\n\n\e[31;1mCould not find existing build artifact after polling for 30 minutes\e[0m\n\n\n" + exit 1 fi done From 9211766fc5f0d80e0e9b77b86d7e3fd63effee4f Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Tue, 6 Feb 2024 18:07:53 +1100 Subject: [PATCH 06/17] Updated the 'drone-upload-exists' to be a standalone script --- Scripts/drone-static-upload.sh | 94 ++++++++++++++-------------------- Scripts/drone-upload-exists.sh | 36 +++++++++---- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/Scripts/drone-static-upload.sh b/Scripts/drone-static-upload.sh index 040cde913..6a0120433 100755 --- a/Scripts/drone-static-upload.sh +++ b/Scripts/drone-static-upload.sh @@ -2,29 +2,20 @@ # Script used with Drone CI to upload build artifacts (because specifying all this in # .drone.jsonnet is too painful). -# -# Passing an argument of 'false' will result in just the upload location being output -# instead of attepting to actually upload set -o errexit -if [[ -z "$1" || "$0" == "true" ]]; then - if [ -z "$SSH_KEY" ]; then - echo -e "\n\n\n\e[31;1mUnable to upload artifact: SSH_KEY not set\e[0m" - # Just warn but don't fail, so that this doesn't trigger a build failure for untrusted builds - exit 0 - fi - - echo "$SSH_KEY" >ssh_key +if [ -z "$SSH_KEY" ]; then + echo -e "\n\n\n\e[31;1mUnable to upload artifact: SSH_KEY not set\e[0m" + # Just warn but don't fail, so that this doesn't trigger a build failure for untrusted builds + exit 0 +fi - set -o xtrace # Don't start tracing until *after* we write the ssh key +echo "$SSH_KEY" >ssh_key - chmod 600 ssh_key +set -o xtrace # Don't start tracing until *after* we write the ssh key - should_upload=true -else - should_upload=$1 -fi +chmod 600 ssh_key # Define the output paths prod_path="build/Session.xcarchive" @@ -32,9 +23,11 @@ sim_path="build/Session_sim.xcarchive/Products/Applications/Session.app" # Validate the paths exist if [ -d $prod_path ]; then + suffix="-store" suffix="store" target_path=$prod_path -elif [[ -d $sim_path || "$should_upload" == "false" ]]; then +elif [ -d $sim_path ]; then + suffix="-sim" suffix="sim" target_path=$sim_path else @@ -45,51 +38,42 @@ fi if [ -n "$DRONE_TAG" ]; then # For a tag build use something like `session-ios-v1.2.3` base="session-ios-$DRONE_TAG-$suffix" -elif [ "$should_upload" == "false" ]; then - # Instead of the datetime include a wildcard in the base name as it'll differ in the actual build: - # session-ios-.*-2fba13878 - base="session-ios-.*-${DRONE_COMMIT:0:9}-$suffix" else # Otherwise build a length name from the datetime and commit hash, such as: - # session-ios-20200522T212342Z-2fba13878 + # session-ios-20200522T212342Z-04d7dcc54 base="session-ios-$(date --date=@$DRONE_BUILD_CREATED +%Y%m%dT%H%M%SZ)-${DRONE_COMMIT:0:9}-$suffix" fi -archive="$base.tar.xz" +# Copy over the build products +mkdir -vp "$base" +mkdir -p build +cp -av $target_path "$base" -if [ "$should_upload" == "true" ]; then - upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" - - # Copy over the build products - mkdir -vp "$base" - mkdir -p build - cp -av $target_path "$base" - - # tar dat shiz up yo - tar cJvf "$archive" "$base" - - # sftp doesn't have any equivalent to mkdir -p, so we have to split the above up into a chain of - # -mkdir a/, -mkdir a/b/, -mkdir a/b/c/, ... commands. The leading `-` allows the command to fail - # without error. - upload_dirs=(${upload_to//\// }) - put_debug= - mkdirs= - dir_tmp="" - for p in "${upload_dirs[@]}"; do - dir_tmp="$dir_tmp$p/" - mkdirs="$mkdirs - -mkdir $dir_tmp" - done +# tar dat shiz up yo +archive="$base.tar.xz" +tar cJvf "$archive" "$base" + +upload_to="oxen.rocks/${DRONE_REPO// /_}/${DRONE_BRANCH// /_}" + +# sftp doesn't have any equivalent to mkdir -p, so we have to split the above up into a chain of +# -mkdir a/, -mkdir a/b/, -mkdir a/b/c/, ... commands. The leading `-` allows the command to fail +# without error. +upload_dirs=(${upload_to//\// }) +put_debug= +mkdirs= +dir_tmp="" +for p in "${upload_dirs[@]}"; do + dir_tmp="$dir_tmp$p/" + mkdirs="$mkdirs +-mkdir $dir_tmp" +done sftp -i ssh_key -b - -o StrictHostKeyChecking=off drone@oxen.rocks < Date: Wed, 7 Feb 2024 08:39:17 +1100 Subject: [PATCH 07/17] Fixed an incorrect url --- Scripts/drone-upload-exists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index 2a479f232..5dd50f1a4 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -13,8 +13,8 @@ prefix="session-ios-" suffix="-${DRONE_COMMIT:0:9}-sim.tar.xz" # Extracting head.label using string manipulation -echo "Extracting repo information for 'https://api.github.com/repos/oxen-io/${DRONE_REPO}/pulls/${DRONE_PULL_REQUEST}'" -pr_info=$(curl -s https://api.github.com/repos/oxen-io/${DRONE_REPO}/pulls/${DRONE_PULL_REQUEST}) +echo "Extracting repo information for 'https://api.github.com/repos/${DRONE_REPO}/pulls/${DRONE_PULL_REQUEST}'" +pr_info=$(curl -s https://api.github.com/repos/${DRONE_REPO}/pulls/${DRONE_PULL_REQUEST}) pr_info_clean=$(echo "$pr_info" | tr -d '[:space:]') head_info=$(echo "$pr_info_clean" | sed -n 's/.*"head"\(.*\)"base".*/\1/p') fork_repo=$(echo "$head_info" | grep -o '"full_name":"[^"]*' | sed 's/"full_name":"//') From 518818a8b0c9be6cced54c41d454b98b3f69fdb1 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 08:58:57 +1100 Subject: [PATCH 08/17] Removed a debug line --- Scripts/drone-upload-exists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index 5dd50f1a4..64fe91c98 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -22,7 +22,7 @@ fork_branch=$(echo "$head_info" | grep -o '"ref":"[^"]*' | sed 's/"ref":"//') upload_dir="https://oxen.rocks/${fork_repo}/${fork_branch}" echo "Starting to poll ${upload_dir} every 10s to check for a build matching '${prefix}.*${suffix}'" -exit 1 + # Loop indefinitely the CI can timeout the script if it takes too long total_poll_duration=0 max_poll_duration=$((30 * 60)) # Poll for a maximum of 30 mins From a05963f6ac2a917159b8417dd40bc071ef383c2b Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 09:37:57 +1100 Subject: [PATCH 09/17] Added some debugging code for the upload exists script --- Scripts/drone-static-upload.sh | 2 -- Scripts/drone-upload-exists.sh | 5 ++++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Scripts/drone-static-upload.sh b/Scripts/drone-static-upload.sh index 6a0120433..1363a4b83 100755 --- a/Scripts/drone-static-upload.sh +++ b/Scripts/drone-static-upload.sh @@ -23,11 +23,9 @@ sim_path="build/Session_sim.xcarchive/Products/Applications/Session.app" # Validate the paths exist if [ -d $prod_path ]; then - suffix="-store" suffix="store" target_path=$prod_path elif [ -d $sim_path ]; then - suffix="-sim" suffix="sim" target_path=$sim_path else diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index 64fe91c98..cbea38e8e 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -32,13 +32,16 @@ while true; do build_artifacts_html=$(curl -s "${upload_dir}/") if [ $? != 0 ]; then - echo "Failed to retrieve build artifact list" + echo -e "\n\n\n\n\e[31;1mFailed to retrieve build artifact list\e[0m\n\n\n" exit 1 fi # Extract 'session-ios...' titles using grep and awk current_build_artifacts=$(echo "$build_artifacts_html" | grep -o 'href="${prefix}[^"]*' | sed 's/href="//') + echo -e "\n\n\n\nDebug - Existing build artifacts:" + echo -e "${current_build_artifacts}" + # Use grep to check for the combination target_file=$(echo "$current_build_artifacts" | grep -o "${prefix}.*${suffix}" | tail -n 1) From 0cbfbadef4e72e8ccf63ccaebb1a512026b536a5 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 10:35:57 +1100 Subject: [PATCH 10/17] Fixed an incorrect grep format --- Scripts/drone-upload-exists.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index cbea38e8e..7035d9597 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -21,7 +21,7 @@ fork_repo=$(echo "$head_info" | grep -o '"full_name":"[^"]*' | sed 's/"full_name fork_branch=$(echo "$head_info" | grep -o '"ref":"[^"]*' | sed 's/"ref":"//') upload_dir="https://oxen.rocks/${fork_repo}/${fork_branch}" -echo "Starting to poll ${upload_dir} every 10s to check for a build matching '${prefix}.*${suffix}'" +echo "Starting to poll ${upload_dir}/ every 10s to check for a build matching '${prefix}.*${suffix}'" # Loop indefinitely the CI can timeout the script if it takes too long total_poll_duration=0 @@ -37,7 +37,7 @@ while true; do fi # Extract 'session-ios...' titles using grep and awk - current_build_artifacts=$(echo "$build_artifacts_html" | grep -o 'href="${prefix}[^"]*' | sed 's/href="//') + current_build_artifacts=$(echo "$build_artifacts_html" | grep -o "href=\"${prefix}[^\"]*" | sed 's/href="//') echo -e "\n\n\n\nDebug - Existing build artifacts:" echo -e "${current_build_artifacts}" From 4aa5eb1bbc36be60a2361a03ad1dd8666d854473 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 11:29:02 +1100 Subject: [PATCH 11/17] Remove debug lines --- Scripts/drone-upload-exists.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Scripts/drone-upload-exists.sh b/Scripts/drone-upload-exists.sh index 7035d9597..a810b77a3 100755 --- a/Scripts/drone-upload-exists.sh +++ b/Scripts/drone-upload-exists.sh @@ -36,13 +36,8 @@ while true; do exit 1 fi - # Extract 'session-ios...' titles using grep and awk + # Extract 'session-ios...' titles using grep and awk then look for the target file current_build_artifacts=$(echo "$build_artifacts_html" | grep -o "href=\"${prefix}[^\"]*" | sed 's/href="//') - - echo -e "\n\n\n\nDebug - Existing build artifacts:" - echo -e "${current_build_artifacts}" - - # Use grep to check for the combination target_file=$(echo "$current_build_artifacts" | grep -o "${prefix}.*${suffix}" | tail -n 1) if [ -n "$target_file" ]; then From d324f36c536f1048a4948b1b091ed2cc0902e71c Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 13:27:42 +1100 Subject: [PATCH 12/17] Updated to the latest CI config (few optimisations) --- .drone.jsonnet | 78 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index ca7dd8c5b..80aa7b8fe 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,3 +1,13 @@ +// Log a bunch of version information to make it easier for debugging +local version_info = { + name: 'Version Information', + commands: [ + 'git --version', + 'pod --version', + 'xcodebuild -version' + ] +}; + // Intentionally doing a depth of 2 as libSession-util has it's own submodules (and libLokinet likely will as well) local clone_submodules = { name: 'Clone Submodules', @@ -15,7 +25,10 @@ local install_cocoapods = { name: 'Install CocoaPods', commands: [' LANG=en_US.UTF-8 pod install || rm -rf ./Pods && LANG=en_US.UTF-8 pod install - '] + '], + depends_on: [ + 'Load CocoaPods Cache' + ] }; // Load from the cached CocoaPods directory (to speed up the build) @@ -40,11 +53,14 @@ local load_cocoapods_cache = { fi |||, 'rm -f /Users/drone/.cocoapods_cache.lock' + ], + depends_on: [ + 'Clone Submodules' ] }; // Override the cached CocoaPods directory (to speed up the next build) -local update_cocoapods_cache = { +local update_cocoapods_cache(depends_on) = { name: 'Update CocoaPods Cache', commands: [ ||| @@ -66,7 +82,19 @@ local update_cocoapods_cache = { fi |||, 'rm -f /Users/drone/.cocoapods_cache.lock' - ] + ], + depends_on: depends_on, +}; + +// Run specified unit tests +local run_tests(testName, testBuildStepName) = { + name: 'Run ' + testName, + commands: [ + 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test-without-building -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -test-timeouts-enabled YES -maximum-test-execution-time-allowance 10 -only-testing ' + testName + ' -collect-test-diagnostics never 2>&1 | ./Pods/xcbeautify/xcbeautify --is-ci', + ], + depends_on: [ + testBuildStepName + ], }; @@ -79,17 +107,49 @@ local update_cocoapods_cache = { platform: { os: 'darwin', arch: 'amd64' }, trigger: { event: { exclude: [ 'push' ] } }, steps: [ + version_info, clone_submodules, load_cocoapods_cache, install_cocoapods, { - name: 'Run Unit Tests', + name: 'Reset Simulators', + commands: [ + 'xcrun simctl shutdown all', + 'xcrun simctl erase all' + ], + depends_on: [ + 'Install CocoaPods' + ] + }, + { + name: 'Build For Testing', commands: [ 'mkdir build', - 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -destination "platform=iOS Simulator,name=iPhone 14 Pro Max" -parallel-testing-enabled YES -test-timeouts-enabled YES -maximum-test-execution-time-allowance 2 -collect-test-diagnostics never 2>&1 | ./Pods/xcbeautify/xcbeautify --is-ci --report junit --report-path ./build/reports --junit-report-filename junit2.xml' + 'xcodebuild build-for-testing -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -destination "platform=iOS Simulator,name=iPhone 14" | ./Pods/xcbeautify/xcbeautify --is-ci', + ], + depends_on: [ + 'Install CocoaPods' ], }, - update_cocoapods_cache + run_tests('SessionTests', 'Build For Testing'), + run_tests('SessionMessagingKitTests', 'Build For Testing'), + run_tests('SessionSnodeKitTests', 'Build For Testing'), + run_tests('SessionUtilitiesKitTests', 'Build For Testing'), + { + name: 'Shutdown Simulators', + commands: [ 'xcrun simctl shutdown all' ], + depends_on: [ + 'Build For Testing', + 'Run SessionTests', + 'Run SessionMessagingKitTests', + 'Run SessionSnodeKitTests', + 'Run SessionUtilitiesKitTests' + ], + when: { + status: ['failure', 'success'] + } + }, + update_cocoapods_cache(['Build For Testing']) ], }, // Validate build artifact was created by the direct branch push (PRs only) @@ -116,6 +176,7 @@ local update_cocoapods_cache = { platform: { os: 'darwin', arch: 'amd64' }, trigger: { event: { exclude: [ 'pull_request' ] } }, steps: [ + version_info, clone_submodules, load_cocoapods_cache, install_cocoapods, @@ -126,12 +187,15 @@ local update_cocoapods_cache = { 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' ], }, - update_cocoapods_cache, + update_cocoapods_cache(['Build']), { name: 'Upload artifacts', environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, commands: [ './Scripts/drone-static-upload.sh' + ], + depends_on: [ + 'Build' ] }, ], From d816b621beae798c6c0cda5b3c218385e3121cb8 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 13:31:45 +1100 Subject: [PATCH 13/17] Added a missing dependency --- .drone.jsonnet | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 80aa7b8fe..b1ac82f91 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -184,7 +184,10 @@ local run_tests(testName, testBuildStepName) = { name: 'Build', commands: [ 'mkdir build', - 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' + 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' + ], + depends_on: [ + 'Install CocoaPods' ], }, update_cocoapods_cache(['Build']), From 86957b32de82a3ddfd7b3202467f1f03a4855e51 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 13:48:32 +1100 Subject: [PATCH 14/17] Removed test target which doesn't exist yet --- .drone.jsonnet | 2 -- 1 file changed, 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index b1ac82f91..ad72e1ef1 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -133,7 +133,6 @@ local run_tests(testName, testBuildStepName) = { }, run_tests('SessionTests', 'Build For Testing'), run_tests('SessionMessagingKitTests', 'Build For Testing'), - run_tests('SessionSnodeKitTests', 'Build For Testing'), run_tests('SessionUtilitiesKitTests', 'Build For Testing'), { name: 'Shutdown Simulators', @@ -142,7 +141,6 @@ local run_tests(testName, testBuildStepName) = { 'Build For Testing', 'Run SessionTests', 'Run SessionMessagingKitTests', - 'Run SessionSnodeKitTests', 'Run SessionUtilitiesKitTests' ], when: { From 4670d7c22edaf9af47b6354b8f0420cc3e0d6ead Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 14:23:28 +1100 Subject: [PATCH 15/17] Debugging failing unit tests --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index ad72e1ef1..50722f682 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -90,7 +90,7 @@ local update_cocoapods_cache(depends_on) = { local run_tests(testName, testBuildStepName) = { name: 'Run ' + testName, commands: [ - 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test-without-building -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -test-timeouts-enabled YES -maximum-test-execution-time-allowance 10 -only-testing ' + testName + ' -collect-test-diagnostics never 2>&1 | ./Pods/xcbeautify/xcbeautify --is-ci', + 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test-without-building -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -test-timeouts-enabled YES -maximum-test-execution-time-allowance 10 -only-testing ' + testName, ], depends_on: [ testBuildStepName From e23d28c5700cb8e50b7246ecc6687aec955d1625 Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 14:48:49 +1100 Subject: [PATCH 16/17] Removed the xcbeautify dependency (installed on CI directly) --- .drone.jsonnet | 6 +++--- Podfile | 3 --- Podfile.lock | 6 +----- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 50722f682..e7bb8a51b 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -90,7 +90,7 @@ local update_cocoapods_cache(depends_on) = { local run_tests(testName, testBuildStepName) = { name: 'Run ' + testName, commands: [ - 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test-without-building -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -test-timeouts-enabled YES -maximum-test-execution-time-allowance 10 -only-testing ' + testName, + 'NSUnbufferedIO=YES set -o pipefail && xcodebuild test-without-building -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -destination "platform=iOS Simulator,name=iPhone 14" -test-timeouts-enabled YES -maximum-test-execution-time-allowance 10 -only-testing ' + testName + ' -collect-test-diagnostics never 2>&1 | xcbeautify --is-ci', ], depends_on: [ testBuildStepName @@ -125,7 +125,7 @@ local run_tests(testName, testBuildStepName) = { name: 'Build For Testing', commands: [ 'mkdir build', - 'xcodebuild build-for-testing -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -destination "platform=iOS Simulator,name=iPhone 14" | ./Pods/xcbeautify/xcbeautify --is-ci', + 'xcodebuild build-for-testing -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -destination "platform=iOS Simulator,name=iPhone 14" | xcbeautify --is-ci', ], depends_on: [ 'Install CocoaPods' @@ -182,7 +182,7 @@ local run_tests(testName, testBuildStepName) = { name: 'Build', commands: [ 'mkdir build', - 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | ./Pods/xcbeautify/xcbeautify --is-ci' + 'xcodebuild archive -workspace Session.xcworkspace -scheme Session -derivedDataPath ./build/derivedData -parallelizeTargets -configuration "App Store Release" -sdk iphonesimulator -archivePath ./build/Session_sim.xcarchive -destination "generic/platform=iOS Simulator" | xcbeautify --is-ci' ], depends_on: [ 'Install CocoaPods' diff --git a/Podfile b/Podfile index 201db8853..69fbc174f 100644 --- a/Podfile +++ b/Podfile @@ -5,9 +5,6 @@ inhibit_all_warnings! install! 'cocoapods', :warn_for_unused_master_specs_repo => false -# CI Dependencies -pod 'xcbeautify' - # Dependencies to be included in the app and all extensions/frameworks abstract_target 'GlobalDependencies' do # FIXME: If https://github.com/jedisct1/swift-sodium/pull/249 gets resolved then revert this back to the standard pod diff --git a/Podfile.lock b/Podfile.lock index f04dbd43b..2cd1ec047 100644 --- a/Podfile.lock +++ b/Podfile.lock @@ -45,7 +45,6 @@ PODS: - SQLCipher/common - SwiftProtobuf (1.5.0) - WebRTC-lib (114.0.0) - - xcbeautify (0.17.0) - YapDatabase/SQLCipher (3.1.1): - YapDatabase/SQLCipher/Core (= 3.1.1) - YapDatabase/SQLCipher/Extensions (= 3.1.1) @@ -128,7 +127,6 @@ DEPENDENCIES: - SQLCipher (~> 4.5.3) - SwiftProtobuf (~> 1.5.0) - WebRTC-lib - - xcbeautify - YapDatabase/SQLCipher (from `https://github.com/oxen-io/session-ios-yap-database.git`, branch `signal-release`) - YYImage/libwebp (from `https://github.com/signalapp/YYImage`) @@ -149,7 +147,6 @@ SPEC REPOS: - libwebp - Nimble - Quick - - xcbeautify EXTERNAL SOURCES: Curve25519Kit: @@ -202,10 +199,9 @@ SPEC CHECKSUMS: SQLCipher: 57fa9f863fa4a3ed9dd3c90ace52315db8c0fdca SwiftProtobuf: 241400280f912735c1e1b9fe675fdd2c6c4d42e2 WebRTC-lib: d83df8976fa608b980f1d85796b3de66d60a1953 - xcbeautify: 6e2f57af5c3a86d490376d5758030a8dcc201c1b YapDatabase: b418a4baa6906e8028748938f9159807fd039af4 YYImage: f1ddd15ac032a58b78bbed1e012b50302d318331 -PODFILE CHECKSUM: dd814a5a92577bb2a94dac6a1cc482f193721cdf +PODFILE CHECKSUM: f56c28baefe3077effcb3a2ea5941b52c4cc6e86 COCOAPODS: 1.14.3 From 260003e9f45106a1a28e3eea42f9a05bde47e84f Mon Sep 17 00:00:00 2001 From: Morgan Pretty Date: Wed, 7 Feb 2024 14:57:30 +1100 Subject: [PATCH 17/17] Added multiple jobs for cloning submodules to speed it up a little --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index e7bb8a51b..f51504353 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -11,7 +11,7 @@ local version_info = { // Intentionally doing a depth of 2 as libSession-util has it's own submodules (and libLokinet likely will as well) local clone_submodules = { name: 'Clone Submodules', - commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=2'] + commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=2 --jobs=4'] }; // cmake options for static deps mirror