Merge pull request opf#15054 from opf/impl/batch_copy_project_job

Introduces the use of `GoodJob::Batch` for CopyProjectJob in order to support async managed remote folder copy.

Author: mereghost

Diff for: app/services/projects/copy_service.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #-- copyright
 # OpenProject is an open source project management software.
 # Copyright (C) 2012-2024 the OpenProject GmbH
@@ -41,12 +43,22 @@ def self.copy_dependencies
         ::Projects::Copy::QueriesDependentService,
         ::Projects::Copy::BoardsDependentService,
         ::Projects::Copy::OverviewDependentService,
-        ::Projects::Copy::StoragesDependentService,
-        ::Projects::Copy::StorageProjectFoldersDependentService,
-        ::Projects::Copy::FileLinksDependentService
+        ::Projects::Copy::StoragesDependentService
       ]
     end
 
+    # Project Folders and File Links aren't dependent services anymore,
+    #  so we need to amend the services for the form Representer
+    def self.copyable_dependencies
+      super + [{ identifier: "storage_project_folders",
+                 name_source: -> { I18n.t(:label_project_storage_project_folder) },
+                 count_source: ->(source, _) { source.storages.count } },
+
+               { identifier: "file_links",
+                 name_source: -> { I18n.t("projects.copy.work_package_file_links") },
+                 count_source: ->(source, _) { source.work_packages.joins(:file_links).count("file_links.id") } }]
+    end
+
     protected
 
     ##

Diff for: app/services/projects/enqueue_copy_service.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #-- copyright
 # OpenProject is an open source project management software.
 # Copyright (C) 2012-2024 the OpenProject GmbH
@@ -60,11 +62,13 @@ def test_copy(params)
     ##
     # Schedule the project copy job
     def schedule_copy_job(params)
-      CopyProjectJob.perform_later(user_id: user.id,
-                                   source_project_id: source.id,
-                                   target_project_params: params[:target_project_params],
-                                   associations_to_copy: params[:only].to_a,
-                                   send_mails: ActiveRecord::Type::Boolean.new.cast(params[:send_notifications]))
+      job = nil
+      GoodJob::Batch.enqueue(on_finish: SendCopyProjectStatusEmailJob, user:, source_project: source) do
+        job = CopyProjectJob.perform_later(target_project_params: params[:target_project_params],
+                                           associations_to_copy: params[:only].to_a,
+                                           send_mails: ActiveRecord::Type::Boolean.new.cast(params[:send_notifications]))
+      end
+      job
     end
   end
 end

Diff for: app/workers/copy_project_job.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #-- copyright
 # OpenProject is an open source project management software.
 # Copyright (C) 2012-2024 the OpenProject GmbH
@@ -27,71 +29,51 @@
 #++
 
 class CopyProjectJob < ApplicationJob
-  queue_with_priority :above_normal
   include OpenProject::LocaleHelper
+  include GoodJob::ActiveJobExtensions::Batches
 
-  attr_reader :user_id,
-              :source_project_id,
-              :target_project_params,
-              :target_project_name,
-              :target_project,
-              :errors,
-              :associations_to_copy,
-              :send_mails
-
-  def perform(user_id:,
-              source_project_id:,
-              target_project_params:,
-              associations_to_copy:,
-              send_mails: false)
-    # Needs refactoring after moving to activejob
-
-    @user_id               = user_id
-    @source_project_id     = source_project_id
-    @target_project_params = target_project_params.with_indifferent_access
-    @associations_to_copy  = associations_to_copy
-    @send_mails            = send_mails
+  queue_with_priority :above_normal
 
+  # Again error handling pushing the branch costs up
+  def perform(target_project_params:, associations_to_copy:, send_mails: false)
     User.current = user
-    @target_project_name = target_project_params[:name]
+    target_project_params = target_project_params.with_indifferent_access
 
-    @target_project, @errors = with_locale_for(user) do
-      create_project_copy
+    target_project, errors = with_locale_for(user) do
+      create_project_copy(target_project_params, associations_to_copy, send_mails)
     end
 
+    update_batch(target_project:, errors:, target_project_name: target_project_params[:name])
+
     if target_project
-      successful_status_update
-      ProjectMailer.copy_project_succeeded(user, source_project, target_project, errors).deliver_later
+      successful_status_update(target_project, errors)
     else
-      failure_status_update
-      ProjectMailer.copy_project_failed(user, source_project, target_project_name, errors).deliver_later
+      failure_status_update(errors)
     end
   rescue StandardError => e
     logger.error { "Failed to finish copy project job: #{e} #{e.message}" }
     errors = [I18n.t("copy_project.failed_internal")]
-    failure_status_update
-    ProjectMailer.copy_project_failed(user, source_project, target_project_name, errors).deliver_later
+    update_batch(errors:)
+    failure_status_update(errors)
   end
 
-  def store_status?
-    true
-  end
+  def store_status? = true
 
-  def updates_own_status?
-    true
-  end
+  def updates_own_status? = true
 
   protected
 
-  def title
-    I18n.t(:label_copy_project)
-  end
+  def title = I18n.t(:label_copy_project)
 
   private
 
-  def successful_status_update
-    payload = redirect_payload(url_helpers.project_url(target_project))
-      .merge(hal_links(target_project))
+  def update_batch(hash)
+    batch.properties.merge!(hash)
+    batch.save
+  end
+
+  def successful_status_update(target_project, errors)
+    payload = redirect_payload(url_helpers.project_url(target_project)).merge(hal_links(target_project))
 
     if errors.any?
       payload[:errors] = errors
@@ -102,7 +84,7 @@ def successful_status_update
                   payload:
   end
 
-  def failure_status_update
+  def failure_status_update(errors)
     message = I18n.t("copy_project.failed", source_project_name: source_project.name)
 
     if errors
@@ -123,60 +105,66 @@ def hal_links(project)
     }
   end
 
-  def user
-    @user ||= User.find user_id
-  end
+  def user = batch.properties[:user]
 
-  def source_project
-    @source_project ||= Project.find source_project_id
-  end
+  def source_project = batch.properties[:source_project]
 
-  def create_project_copy
+  # rubocop:disable Metrics/AbcSize
+  # Most of the cost is from handling errors, we need to check what can be moved around / removed
+  def create_project_copy(target_project_params, associations_to_copy, send_mails)
     errors = []
 
     ProjectMailer.with_deliveries(send_mails) do
-      service_call = copy_project
-      target_project = service_call.result
-      errors = service_call.errors.full_messages
+      service_result = copy_project(target_project_params, associations_to_copy, send_mails)
+      target_project = service_result.result
+      errors = service_result.errors.full_messages
 
       # We assume the copying worked "successfully" if the project was saved
-      unless target_project&.persisted?
-        target_project = nil
+      if target_project&.persisted?
+        return target_project, errors
+      else
         logger.error("Copying project fails with validation errors: #{errors.join("\n")}")
+        return nil, errors
       end
-
-      return target_project, errors
     end
   rescue ActiveRecord::RecordNotFound => e
     logger.error("Entity missing: #{e.message} #{e.backtrace.join("\n")}")
+    raise e
   rescue StandardError => e
     logger.error("Encountered an error when trying to copy project " \
-                 "'#{source_project_id}' : #{e.message} #{e.backtrace.join("\n")}")
+                 "'#{source_project.id}' : #{e.message} #{e.backtrace.join("\n")}")
+    raise e
   ensure
-    unless errors.empty?
+    if errors.any?
       logger.error("Encountered an errors while trying to copy related objects for " \
-                   "project '#{source_project_id}': #{errors.inspect}")
+                   "project '#{source_project.id}': #{errors.inspect}")
     end
   end
+  # rubocop:enable Metrics/AbcSize
 
-  def copy_project
-    ::Projects::CopyService
-      .new(source: source_project, user:)
-      .call(copy_project_params)
-  end
+  def copy_project(target_project_params, associations_to_copy, send_notifications)
+    copy_service = ::Projects::CopyService.new(source: source_project, user:)
+    result = copy_service.call({ target_project_params:, send_notifications:, only: Array(associations_to_copy) })
 
-  def copy_project_params
-    params = { target_project_params:, send_notifications: send_mails }
-    params[:only] = associations_to_copy if associations_to_copy.present?
+    enqueue_copy_project_folder_jobs(copy_service.state.copied_project_storages,
+                                     copy_service.state.work_package_id_lookup,
+                                     associations_to_copy)
 
-    params
+    result
   end
 
-  def logger
-    Rails.logger
-  end
+  def enqueue_copy_project_folder_jobs(copied_storages, work_packages_map, only)
+    return unless only.intersect?(%w[file_links storage_project_folders])
 
-  def url_helpers
-    @url_helpers ||= OpenProject::StaticRouting::StaticUrlHelpers.new
+    Array(copied_storages).each do |storage_pair|
+      batch.enqueue do
+        Storages::CopyProjectFoldersJob
+          .perform_later(source: storage_pair[:source], target: storage_pair[:target], work_packages_map:)
+      end
+    end
   end
+
+  def logger = OpenProject.logger
+
+  def url_helpers = OpenProject::StaticRouting::StaticUrlHelpers.new
 end

Diff for: app/workers/notifications/schedule_date_alerts_notifications_job.rb

@@ -23,7 +23,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
-# See docs/COPYRIGHT.rdoc for more details.
+# See COPYRIGHT and LICENSE files for more details.
 #++
 
 module Notifications

Diff for: app/workers/notifications/schedule_reminder_mails_job.rb

@@ -23,7 +23,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
-# See docs/COPYRIGHT.rdoc for more details.
+# See COPYRIGHT and LICENSE files for more details.
 #++
 
 module Notifications

Diff for: app/workers/send_copy_project_status_email_job.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) 2012-2024 the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+class SendCopyProjectStatusEmailJob < ApplicationJob
+  # Job is to be used as a callback to the CopyProjectJob batch
+
+  def perform(batch, _args)
+    if copy_job_succeeded?(batch) && batch.properties[:target_project]
+      send_success_email(batch)
+    else
+      send_failure_email(batch)
+    end
+  end
+
+  private
+
+  def copy_job_succeeded?(batch)
+    job = batch.active_jobs.find { |batch_job| batch_job.instance_of?(CopyProjectJob) }
+
+    job.job_status.success?
+  end
+
+  def send_failure_email(batch)
+    ProjectMailer.copy_project_failed(
+      batch.properties[:user],
+      batch.properties[:source_project],
+      batch.properties[:target_project_name],
+      batch.properties[:errors]
+    ).deliver_later
+  end
+
+  def send_success_email(batch)
+    ProjectMailer.copy_project_succeeded(
+      batch.properties[:user],
+      batch.properties[:source_project],
+      batch.properties[:target_project],
+      batch.properties[:errors]
+    ).deliver_later
+  end
+end

Diff for: modules/storages/app/common/storages/errors.rb

@@ -34,6 +34,8 @@ class BaseError < StandardError; end
 
     class ResolverStandardError < BaseError; end
 
+    class PollingRequired < BaseError; end
+
     class MissingContract < ResolverStandardError; end
 
     class OperationNotSupported < ResolverStandardError; end

Diff for: modules/storages/app/common/storages/peripherals/nextcloud.rb renamed to modules/storages/app/common/storages/peripherals/nextcloud_registry.rb

@@ -30,7 +30,7 @@
 
 module Storages
   module Peripherals
-    Nextcloud = Dry::Container::Namespace.new("nextcloud") do
+    NextcloudRegistry = Dry::Container::Namespace.new("nextcloud") do
       namespace("queries") do
         register(:auth_check, StorageInteraction::Nextcloud::AuthCheckQuery)
         register(:download_link, StorageInteraction::Nextcloud::DownloadLinkQuery)

Diff for: modules/storages/app/common/storages/peripherals/one_drive.rb renamed to modules/storages/app/common/storages/peripherals/one_drive_registry.rb

@@ -30,7 +30,7 @@
 
 module Storages
   module Peripherals
-    OneDrive = Dry::Container::Namespace.new("one_drive") do
+    OneDriveRegistry = Dry::Container::Namespace.new("one_drive") do
       namespace("queries") do
         register(:auth_check, StorageInteraction::OneDrive::AuthCheckQuery)
         register(:download_link, StorageInteraction::OneDrive::DownloadLinkQuery)
@@ -44,6 +44,7 @@ module Peripherals
       end
 
       namespace("commands") do
+        register(:copy_template_folder, StorageInteraction::OneDrive::CopyTemplateFolderCommand)
         register(:create_folder, StorageInteraction::OneDrive::CreateFolderCommand)
         register(:delete_folder, StorageInteraction::OneDrive::DeleteFolderCommand)
         register(:rename_file, StorageInteraction::OneDrive::RenameFileCommand)

Diff for: modules/storages/app/common/storages/peripherals/registry.rb

@@ -44,7 +44,7 @@ def call(container, key)
       config.resolver = Resolver.new
     end
 
-    Registry.import Nextcloud
-    Registry.import OneDrive
+    Registry.import NextcloudRegistry
+    Registry.import OneDriveRegistry
   end
 end