Unify build number between App Store and DMG releases (#1929)

Task/Issue URL: https://app.asana.com/0/1201037661562251/1206111130949152/f

Description:
* DMG target now uses the same build number as App Store target.
* To support internal releases using the same marketing version, DMG filenames now include build number.
  * appcastManager and archive.sh were updated to support it.
* set_version.sh has been deleted and replaced by fastlane lane:
  bundle exec fastlane mac set_version version:1.70.0
* New fastlane lane was added for creating subsequent internal releases:
  bundle exec fastlane mac bump_internal_release

.swiftlint.yml

@@ -76,3 +76,4 @@ excluded:
   - UITests
   - vendor
   - DerivedData
+  - .ruby-lsp

Configuration/App/AppTargetsBase.xcconfig

@@ -28,8 +28,6 @@ CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES
 CODE_SIGN_STYLE[sdk=*] = Manual
 CODE_SIGN_STYLE[config=Debug][sdk=*] = Automatic
 
-CURRENT_PROJECT_VERSION = $(MARKETING_VERSION)
-
 ENABLE_HARDENED_RUNTIME = YES
 
 INFOPLIST_FILE = DuckDuckGo/Info.plist

Configuration/AppStore.xcconfig

@@ -14,7 +14,7 @@
 //  limitations under the License.
 //
 
-#include "AppStoreBuildNumber.xcconfig"
+#include "BuildNumber.xcconfig"
 
 MAIN_BUNDLE_IDENTIFIER_PREFIX = com.duckduckgo.mobile.ios
 

Configuration/Common.xcconfig

@@ -14,6 +14,7 @@
 //
 
 #include "Version.xcconfig"
+#include "BuildNumber.xcconfig"
 
 COMBINE_HIDPI_IMAGES = YES
 

Configuration/Extensions/ExtensionBase.xcconfig

@@ -20,8 +20,6 @@ CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES
 
 CODE_SIGN_STYLE[sdk=*] = Manual
 
-CURRENT_PROJECT_VERSION = $(MARKETING_VERSION)
-
 ENABLE_HARDENED_RUNTIME = YES
 
 PRODUCT_BUNDLE_IDENTIFIER[sdk=*] = $(BUNDLE_IDENTIFIER_PREFIX)

DuckDuckGo.xcodeproj/project.pbxproj

@@ -3214,7 +3214,7 @@
 		3767190128E724B2003A2A15 /* DuckPlayerURLExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DuckPlayerURLExtension.swift; sourceTree = "<group>"; };
 		376C4DB828A1A48A00CC0F5B /* FirePopoverViewModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FirePopoverViewModelTests.swift; sourceTree = "<group>"; };
 		376CC8B4296EB630006B63A7 /* AppStore.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = AppStore.xcconfig; sourceTree = "<group>"; };
-		376CC8B5296EBA8F006B63A7 /* AppStoreBuildNumber.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = AppStoreBuildNumber.xcconfig; sourceTree = "<group>"; };
+		376CC8B5296EBA8F006B63A7 /* BuildNumber.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = BuildNumber.xcconfig; sourceTree = "<group>"; };
 		37717E66296B5A20002FAEDF /* Global.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Global.xcconfig; sourceTree = "<group>"; };
 		3775912C29AAC72700E26367 /* SyncPreferences.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncPreferences.swift; sourceTree = "<group>"; };
 		3775913529AB9A1C00E26367 /* SyncManagementDialogViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncManagementDialogViewController.swift; sourceTree = "<group>"; };
@@ -4757,8 +4757,8 @@
 				37717E66296B5A20002FAEDF /* Global.xcconfig */,
 				378B5888295CF2A4002C0CC0 /* Common.xcconfig */,
 				378B5887295CF2A4002C0CC0 /* Version.xcconfig */,
+				376CC8B5296EBA8F006B63A7 /* BuildNumber.xcconfig */,
 				376CC8B4296EB630006B63A7 /* AppStore.xcconfig */,
-				376CC8B5296EBA8F006B63A7 /* AppStoreBuildNumber.xcconfig */,
 				7B5291892A169BC90022E406 /* DeveloperID.xcconfig */,
 				378C76D8296842FD0092E949 /* App */,
 				4B4D604E2A0B293C00BCD287 /* Extensions */,

Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
 gem 'fastlane', '2.217.0'
+gem 'httparty'

Gemfile.lock

@@ -146,13 +146,17 @@ GEM
     highline (2.0.3)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
+    httparty (0.21.0)
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
     httpclient (2.8.3)
     jmespath (1.6.2)
     json (2.6.3)
     jwt (2.7.1)
     mini_magick (4.12.0)
     mini_mime (1.1.5)
     multi_json (1.15.0)
+    multi_xml (0.6.0)
     multipart-post (2.3.0)
     nanaimo (0.3.0)
     naturally (2.2.1)
@@ -208,6 +212,7 @@ PLATFORMS
 
 DEPENDENCIES
   fastlane (= 2.217.0)
+  httparty
 
 BUNDLED WITH
    2.3.26

fastlane/Fastfile

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+fastlane_require 'httparty'
+fastlane_require 'rexml/document'
+
 UI.abort_with_message!('Please run fastlane via `bundle exec`') unless FastlaneCore::Helper.bundler?
 
 ########################################################################
@@ -8,8 +11,9 @@ UI.abort_with_message!('Please run fastlane via `bundle exec`') unless FastlaneC
 DEFAULT_BRANCH = 'main'
 RELEASE_BRANCH = 'release'
 PROJECT_ROOT_FOLDER = File.dirname(File.expand_path(__dir__))
+INFO_PLIST = File.join(PROJECT_ROOT_FOLDER, 'DuckDuckGo/Info.plist')
 VERSION_CONFIG_PATH = File.join(PROJECT_ROOT_FOLDER, 'Configuration/Version.xcconfig')
-APP_STORE_BUILD_NUMBER_CONFIG_PATH = File.join(PROJECT_ROOT_FOLDER, 'Configuration/AppStoreBuildNumber.xcconfig')
+BUILD_NUMBER_CONFIG_PATH = File.join(PROJECT_ROOT_FOLDER, 'Configuration/BuildNumber.xcconfig')
 VERSION_CONFIG_DEFINITION = 'MARKETING_VERSION'
 BUILD_NUMBER_CONFIG_DEFINITION = 'CURRENT_PROJECT_VERSION'
 UPGRADABLE_EMBEDDED_FILES = [
@@ -99,23 +103,27 @@ platform :mac do
   #
   # - Cuts a new release branch
   # - Updates submodules and embedded files
+  # - Pushes changes to remote
   #
   # @option [String] version (default: nil) Marketing version string
-  # @option [Boolean] skip_confirm (default: false) If true, avoids any interactive prompt
   # @option [Boolean] resume (default: false) If true, the lane can run from a release/ branch and will run dedicated prechecks.
+  # @option [Boolean] force (default: false) Don't ask for confirmation.
   #
   desc 'Executes the release preparation work in the repository'
   lane :code_freeze do |options|
     begin
       options[:resume]? macos_codefreeze_resume_prechecks : macos_codefreeze_prechecks
       new_version = validate_new_version(options)
-      app_store_build_number = increment_app_store_build_number(options)
+      build_number = increment_current_build_number(options)
       macos_create_release_branch(version: new_version) unless options[:resume]
       macos_update_embedded_files
       macos_update_version_and_build_number_config(
         version: new_version,
-        build_number: app_store_build_number
+        build_number: build_number,
+        force: options[:force]
       )
+      sh('git', 'push')
+
     rescue => exception
       if exception.message == "Tests have failed"
         UI.user_error! %{Tests have failed.
@@ -131,6 +139,75 @@ platform :mac do
     end
   end
 
+  # Bumps build number for the current version and updates embedded files.
+  # Pushes changes to remote.
+  #
+  # - Should be called on an existing internal release branch.
+  # - Also runs unit tests after updating embedded files.
+  #
+  # @option [Boolean] force (default: false) Don't ask for confirmation.
+  #
+  desc 'Prepares new internal release on top of an existing one'
+  lane :bump_internal_release do |options|
+    begin
+      unless git_branch.start_with?(RELEASE_BRANCH)
+        UI.abort_with_message!("Incorrect branch. Branch name must start with '#{RELEASE_BRANCH}/'.")
+      end
+
+      force = options[:force].nil? ? false : options[:force]
+      current_version = macos_current_version
+      current_build_number = macos_current_build_number
+      build_number = increment_current_build_number(options)
+
+      UI.important("Current version is #{current_version} (#{current_build_number}).")
+      UI.important("Will update to #{current_version} (#{build_number}).")
+
+      unless force
+        unless UI.confirm("Do you want to continue?")
+          UI.abort_with_message!('Aborted by user.')
+        end
+      end
+
+      macos_update_embedded_files
+      macos_update_version_and_build_number_config(
+        version: current_version,
+        build_number: build_number,
+        force: true
+      )
+      sh('git', 'push')
+
+    rescue => exception
+      if exception.message == "Tests have failed"
+        UI.user_error! %{Tests have failed.
+* If you believe the failing test is flaky, please retry the same fastlane command.
+* If the failure looks legitimate, try to fix it, commit the fix (be sure to only
+  include the files you've changed while making a fix and leave other changed files
+  unmodified), and run the command again appending `resume:true`.
+        }
+      else
+        raise exception
+      end
+    end
+  end
+
+  # Updates marketing version to the specified one and increments build number by 1.
+  #
+  # @option [String] version Marketing version string.
+  # @option [Boolean] force (default: false) Don't ask for confirmation.
+  #
+  desc 'Executes the release preparation work in the repository'
+  lane :set_version do |options|
+    unless options[:version]
+      UI.user_error! 'You must provide a version.'
+    end
+    new_version = validate_new_version(options)
+    build_number = increment_current_build_number(options)
+    macos_update_version_and_build_number_config(
+      version: new_version,
+      build_number: build_number
+    )
+  end
+
   #################################################
   # Helper functions
   #################################################
@@ -229,30 +306,52 @@ platform :mac do
     end
   end
 
-  # Calculates the new version or validate the provided one, if it exists
+  # Calculates the new version or validates the provided one, if it exists
   # and prompts the user to confirm
   #
   # @option [String] version (default: nil) Marketing version string
+  # @option [Boolean] force (default: false) Don't ask for confirmation.
   #
   private_lane :validate_new_version do |options|
     current_version = macos_current_version
     user_version = format_user_version(options[:version])
     new_version = user_version.nil? ? macos_bump_minor_version(current_version) : user_version
-    unless UI.confirm(
-      "Current version is #{current_version}.\nNew version is #{new_version}.\nDo you want to continue?"
-    )
-    UI.abort_with_message!('Aborted by user.')
+    force = options[:force].nil? ? false : options[:force]
+    unless force
+      unless UI.confirm(
+        "Current version is #{current_version}.\nNew version is #{new_version}.\nDo you want to continue?"
+      )
+      UI.abort_with_message!('Aborted by user.')
+      end
     end
     new_version
   end
 
   # Checks current build number and increments it by 1.
   #
-  desc 'Increment App Store build number'
-  private_lane :increment_app_store_build_number do
+  desc 'Increment build number'
+  private_lane :increment_current_build_number do
+    macos_current_build_number = [fetch_testflight_build_number, fetch_appcast_build_number].max
     macos_current_build_number + 1
   end
 
+  private_lane :fetch_testflight_build_number do |options|
+    build_number = latest_testflight_build_number(
+      api_key: get_api_key,
+      username: get_username(options),
+      platform: 'osx'
+    )
+    build_number
+  end
+
+  private_lane :fetch_appcast_build_number do |options|
+    url = sh("plutil -extract SUFeedURL raw #{INFO_PLIST}").chomp
+    xml = HTTParty.get(url).body
+    xml_data = REXML::Document.new(xml)
+    versions = xml_data.get_elements('//rss/channel/item/sparkle:version').map { |e| e.text.split('.')[0].to_i }
+    versions.max
+  end
+
   # Checks out a new branch from the current commit and pushes it
   #
   # @option [String] version (default: nil) Marketing version string
@@ -279,7 +378,7 @@ platform :mac do
   # - Runs automated tests
   # - Commits and pushes
   #
-  private_lane :macos_update_embedded_files do
+  private_lane :macos_update_embedded_files do |options|
     sh("cd #{PROJECT_ROOT_FOLDER} && ./scripts/update_embedded.sh")
 
     # Verify no unexpected files were modified
@@ -300,36 +399,35 @@ platform :mac do
     modified_files.each { |modified_file| sh('git', 'add', modified_file.to_s) }
     sh('git', 'commit', '-m', 'Update embedded files')
     ensure_git_status_clean
-    sh('git', 'push')
   end
 
   # Updates version in the config file
   #
-  # @option [String] version (default: nil) Marketing version string
+  # @option [String] version Marketing version string
+  # @option [String] build_number Build number
   #
   private_lane :macos_update_version_and_build_number_config do |options|
     version = options[:version]
     build_number = options[:build_number]
     File.write(VERSION_CONFIG_PATH, "#{VERSION_CONFIG_DEFINITION} = #{version}\n")
-    File.write(APP_STORE_BUILD_NUMBER_CONFIG_PATH, "#{BUILD_NUMBER_CONFIG_DEFINITION} = #{build_number}\n")
+    File.write(BUILD_NUMBER_CONFIG_PATH, "#{BUILD_NUMBER_CONFIG_DEFINITION} = #{build_number}\n")
     git_commit(
       path: [
         VERSION_CONFIG_PATH,
-        APP_STORE_BUILD_NUMBER_CONFIG_PATH
+        BUILD_NUMBER_CONFIG_PATH
       ],
       message: "Bump version to #{version} (#{build_number})"
     )
-    sh('git', 'push')
   end
 
-  # Reads App Store build number from the config file
+  # Reads build number from the config file
   #
   # @return [String] build number read from the file, or nil in case of failure
   #
   def macos_current_build_number
     current_build_number = 0
 
-    file_data = File.read(APP_STORE_BUILD_NUMBER_CONFIG_PATH).split("\n")
+    file_data = File.read(BUILD_NUMBER_CONFIG_PATH).split("\n")
     file_data.each do |line|
       current_build_number = line.split('=')[1].strip.to_i if line.start_with?(BUILD_NUMBER_CONFIG_DEFINITION)
     end

fastlane/README.md

@@ -55,6 +55,22 @@ Updates App Store metadata
 
 Executes the release preparation work in the repository
 
+### mac bump_internal_release
+
+```sh
+[bundle exec] fastlane mac bump_internal_release
+```
+
+Prepares new internal release on top of an existing one
+
+### mac set_version
+
+```sh
+[bundle exec] fastlane mac set_version
+```
+
+Executes the release preparation work in the repository
+
 ----
 
 This README.md is auto-generated and will be re-generated every time [_fastlane_](https://fastlane.tools) is run.